diff --git a/.buildkite/packer_cache.sh b/.buildkite/packer_cache.sh
index 01e1ad5cd7823..e4a80d439741d 100755
--- a/.buildkite/packer_cache.sh
+++ b/.buildkite/packer_cache.sh
@@ -30,5 +30,7 @@ for branch in "${branches[@]}"; do
 
   export JAVA_HOME="$HOME/.java/$ES_BUILD_JAVA"
   "checkout/${branch}/gradlew" --project-dir "$CHECKOUT_DIR" --parallel -s resolveAllDependencies -Dorg.gradle.warning.mode=none -DisCI --max-workers=4
+  "checkout/${branch}/gradlew" --stop
+  pkill -f '.*GradleDaemon.*'
   rm -rf "checkout/${branch}"
 done
diff --git a/.buildkite/scripts/cloud-deploy.sh b/.buildkite/scripts/cloud-deploy.sh
index 2b98aa224406b..045b05ce16dee 100755
--- a/.buildkite/scripts/cloud-deploy.sh
+++ b/.buildkite/scripts/cloud-deploy.sh
@@ -2,11 +2,11 @@
 
 set -euo pipefail
 
-.ci/scripts/run-gradle.sh buildCloudDockerImage
+.ci/scripts/run-gradle.sh buildCloudEssDockerImage
 
 ES_VERSION=$(grep 'elasticsearch' build-tools-internal/version.properties | awk '{print $3}')
-DOCKER_TAG="docker.elastic.co/elasticsearch-ci/elasticsearch-cloud:${ES_VERSION}-${BUILDKITE_COMMIT:0:7}"
-docker tag elasticsearch-cloud:test "$DOCKER_TAG"
+DOCKER_TAG="docker.elastic.co/elasticsearch-ci/elasticsearch-cloud-ess:${ES_VERSION}-${BUILDKITE_COMMIT:0:7}"
+docker tag elasticsearch-cloud-ess:test "$DOCKER_TAG"
 
 echo "$DOCKER_REGISTRY_PASSWORD" | docker login -u "$DOCKER_REGISTRY_USERNAME" --password-stdin docker.elastic.co
 unset DOCKER_REGISTRY_USERNAME DOCKER_REGISTRY_PASSWORD
diff --git a/benchmarks/build.gradle b/benchmarks/build.gradle
index 3a03cbe2d934d..f3ced9f16d327 100644
--- a/benchmarks/build.gradle
+++ b/benchmarks/build.gradle
@@ -40,15 +40,15 @@ dependencies {
     // us to invoke the JMH uberjar as usual.
     exclude group: 'net.sf.jopt-simple', module: 'jopt-simple'
   }
-  api(project(':libs:elasticsearch-h3'))
+  api(project(':libs:h3'))
   api(project(':modules:aggregations'))
   api(project(':x-pack:plugin:esql-core'))
   api(project(':x-pack:plugin:esql'))
   api(project(':x-pack:plugin:esql:compute'))
-  implementation project(path: ':libs:elasticsearch-simdvec')
+  implementation project(path: ':libs:simdvec')
   expression(project(path: ':modules:lang-expression', configuration: 'zip'))
   painless(project(path: ':modules:lang-painless', configuration: 'zip'))
-  nativeLib(project(':libs:elasticsearch-native'))
+  nativeLib(project(':libs:native'))
   api "org.openjdk.jmh:jmh-core:$versions.jmh"
   annotationProcessor "org.openjdk.jmh:jmh-generator-annprocess:$versions.jmh"
   // Dependencies of JMH
diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/PublishPluginFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/PublishPluginFuncTest.groovy
index 99d451116dbe7..6e403c85a23f4 100644
--- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/PublishPluginFuncTest.groovy
+++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/PublishPluginFuncTest.groovy
@@ -18,7 +18,7 @@ class PublishPluginFuncTest extends AbstractGradleFuncTest {
 
     def setup() {
         // required for JarHell to work
-        subProject(":libs:elasticsearch-core") << "apply plugin:'java'"
+        subProject(":libs:core") << "apply plugin:'java'"
 
         configurationCacheCompatible = false
     }
diff --git a/build-tools-internal/src/main/groovy/elasticsearch.ide.gradle b/build-tools-internal/src/main/groovy/elasticsearch.ide.gradle
index 67878181a005d..86b48f744e16e 100644
--- a/build-tools-internal/src/main/groovy/elasticsearch.ide.gradle
+++ b/build-tools-internal/src/main/groovy/elasticsearch.ide.gradle
@@ -161,7 +161,7 @@ if (providers.systemProperty('idea.active').getOrNull() == 'true') {
       ':server:generateModulesList',
       ':server:generatePluginsList',
       ':generateProviderImpls',
-      ':libs:elasticsearch-native:elasticsearch-native-libraries:extractLibs',
+      ':libs:native:native-libraries:extractLibs',
       ':x-pack:libs:es-opensaml-security-api:shadowJar'].collect { elasticsearchProject.right()?.task(it) ?: it })
   }
 
diff --git a/build-tools-internal/src/main/groovy/elasticsearch.stable-api.gradle b/build-tools-internal/src/main/groovy/elasticsearch.stable-api.gradle
index 0148caf8983ef..1fab4d035177a 100644
--- a/build-tools-internal/src/main/groovy/elasticsearch.stable-api.gradle
+++ b/build-tools-internal/src/main/groovy/elasticsearch.stable-api.gradle
@@ -33,12 +33,12 @@ BuildParams.bwcVersions.withIndexCompatible({ it.onOrAfter(Version.fromString(ex
     if (unreleasedVersion) {
       // For unreleased snapshot versions, build them from source
       "oldJar${baseName}"(files(project(unreleasedVersion.gradleProjectPath).tasks.named(buildBwcTaskName(project.name))))
-    } else if(bwcVersion.onOrAfter('8.7.0') && project.name.endsWith("elasticsearch-logging")==false) {
+    } else if(bwcVersion.onOrAfter('8.7.0') && project.name.endsWith("logging")==false) {
       //there was a package rename in 8.7.0, except for es-logging
-      "oldJar${baseName}"("org.elasticsearch.plugin:${project.name}:${bwcVersion}")
+      "oldJar${baseName}"("org.elasticsearch.plugin:elasticsearch-${project.name}:${bwcVersion}")
     } else {
       // For released versions, download it
-      "oldJar${baseName}"("org.elasticsearch:${project.name}:${bwcVersion}")
+      "oldJar${baseName}"("org.elasticsearch:elasticsearch-${project.name}:${bwcVersion}")
     }
   }
 
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java
index 0535f0bdc3cc8..3e0a47a8f453c 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java
@@ -24,7 +24,7 @@ public enum DockerBase {
     // Chainguard based wolfi image with latest jdk
     // This is usually updated via renovatebot
     // spotless:off
-    WOLFI("docker.elastic.co/wolfi/chainguard-base:latest@sha256:bf163e1977002301f7b9fd28fe6837a8cb2dd5c83e4cd45fb67fb28d15d5d40f",
+    WOLFI("docker.elastic.co/wolfi/chainguard-base:latest@sha256:973431347ad45f40e01afbbd010bf9de929c088a63382239b90dd84f39618bc8",
         "-wolfi",
         "apk"
     ),
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavaBasePlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavaBasePlugin.java
index 5913339e32f47..05b7af83aa8e4 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavaBasePlugin.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavaBasePlugin.java
@@ -177,7 +177,7 @@ public static void configureInputNormalization(Project project) {
     }
 
     private static void configureNativeLibraryPath(Project project) {
-        String nativeProject = ":libs:elasticsearch-native:elasticsearch-native-libraries";
+        String nativeProject = ":libs:native:native-libraries";
         Configuration nativeConfig = project.getConfigurations().create("nativeLibs");
         nativeConfig.defaultDependencies(deps -> {
             deps.add(project.getDependencies().project(Map.of("path", nativeProject, "configuration", "default")));
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPlugin.java
index 90b9c0d395f43..fcf286ed471dd 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPlugin.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPlugin.java
@@ -165,7 +165,12 @@ private static void configureBwcProject(
             DistributionProjectArtifact stableAnalysisPluginProjectArtifact = new DistributionProjectArtifact(
                 new File(
                     checkoutDir.get(),
-                    relativeDir + "/build/distributions/" + stableApiProject.getName() + "-" + bwcVersion.get() + "-SNAPSHOT.jar"
+                    relativeDir
+                        + "/build/distributions/elasticsearch-"
+                        + stableApiProject.getName()
+                        + "-"
+                        + bwcVersion.get()
+                        + "-SNAPSHOT.jar"
                 ),
                 null
             );
@@ -275,7 +280,7 @@ private static List<DistributionProject> resolveArchiveProjects(File checkoutDir
     }
 
     private static List<Project> resolveStableProjects(Project project) {
-        Set<String> stableProjectNames = Set.of("elasticsearch-logging", "elasticsearch-plugin-api", "elasticsearch-plugin-analysis-api");
+        Set<String> stableProjectNames = Set.of("logging", "plugin-api", "plugin-analysis-api");
         return project.findProject(":libs")
             .getSubprojects()
             .stream()
@@ -312,7 +317,9 @@ static void createBuildBwcTask(
                 c.getOutputs().files(expectedOutputFile);
             }
             c.getOutputs().doNotCacheIf("BWC distribution caching is disabled for local builds", task -> BuildParams.isCi() == false);
-            c.getArgs().add(projectPath.replace('/', ':') + ":" + assembleTaskName);
+            c.getArgs().add("-p");
+            c.getArgs().add(projectPath);
+            c.getArgs().add(assembleTaskName);
             if (project.getGradle().getStartParameter().isBuildCacheEnabled()) {
                 c.getArgs().add("--build-cache");
             }
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/JarHellPrecommitPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/JarHellPrecommitPlugin.java
index 56434cf1f4eda..0a22a2b61c953 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/JarHellPrecommitPlugin.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/JarHellPrecommitPlugin.java
@@ -21,11 +21,11 @@ public class JarHellPrecommitPlugin extends PrecommitPlugin {
     public TaskProvider<? extends Task> createTask(Project project) {
         project.getPluginManager().apply(JarHellPlugin.class);
 
-        if (project.getPath().equals(":libs:elasticsearch-core") == false) {
+        if (project.getPath().equals(":libs:core") == false) {
             // ideally we would configure this as a default dependency. But Default dependencies do not work correctly
             // with gradle project dependencies as they're resolved to late in the build and don't setup according task
             // dependencies properly
-            var elasticsearchCoreProject = project.findProject(":libs:elasticsearch-core");
+            var elasticsearchCoreProject = project.findProject(":libs:core");
             if (elasticsearchCoreProject != null) {
                 project.getDependencies().add("jarHell", elasticsearchCoreProject);
             }
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditPrecommitPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditPrecommitPlugin.java
index f0eefe1f81a8c..80cece6074ab7 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditPrecommitPlugin.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditPrecommitPlugin.java
@@ -27,7 +27,7 @@
 public class ThirdPartyAuditPrecommitPlugin extends PrecommitPlugin {
 
     public static final String JDK_JAR_HELL_CONFIG_NAME = "jdkJarHell";
-    public static final String LIBS_ELASTICSEARCH_CORE_PROJECT_PATH = ":libs:elasticsearch-core";
+    public static final String LIBS_ELASTICSEARCH_CORE_PROJECT_PATH = ":libs:core";
 
     @Override
     public TaskProvider<? extends Task> createTask(Project project) {
diff --git a/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy b/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy
index 567fb048fad54..d3d06b2de3575 100644
--- a/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy
+++ b/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy
@@ -56,7 +56,7 @@ abstract class AbstractGradleFuncTest extends Specification {
         propertiesFile <<
             "org.gradle.java.installations.fromEnv=JAVA_HOME,RUNTIME_JAVA_HOME,JAVA15_HOME,JAVA14_HOME,JAVA13_HOME,JAVA12_HOME,JAVA11_HOME,JAVA8_HOME"
 
-        def nativeLibsProject = subProject(":libs:elasticsearch-native:elasticsearch-native-libraries")
+        def nativeLibsProject = subProject(":libs:native:native-libraries")
         nativeLibsProject << """
             plugins {
                 id 'base'
diff --git a/client/rest/build.gradle b/client/rest/build.gradle
index 6006fae1c2d84..003c251186510 100644
--- a/client/rest/build.gradle
+++ b/client/rest/build.gradle
@@ -79,7 +79,7 @@ tasks.named('forbiddenApisTest').configure {
 }
 
 // JarHell is part of es server, which we don't want to pull in
-// TODO: Not anymore. Now in :libs:elasticsearch-core
+// TODO: Not anymore. Now in :libs:core
 tasks.named("jarHell").configure {
   enabled = false
 }
diff --git a/client/sniffer/build.gradle b/client/sniffer/build.gradle
index 901917c7b25f8..f6f26c8f7c0d5 100644
--- a/client/sniffer/build.gradle
+++ b/client/sniffer/build.gradle
@@ -73,7 +73,7 @@ tasks.named("dependencyLicenses").configure {
 }
 
 // JarHell is part of es server, which we don't want to pull in
-// TODO: Not anymore. Now in :libs:elasticsearch-core
+// TODO: Not anymore. Now in :libs:core
 tasks.named("jarHell").configure { enabled = false }
 
 tasks.named("testTestingConventions").configure {
diff --git a/client/test/build.gradle b/client/test/build.gradle
index 3a3a9e3c03264..8de6b3dbf92be 100644
--- a/client/test/build.gradle
+++ b/client/test/build.gradle
@@ -54,7 +54,7 @@ tasks.named('forbiddenApisTest').configure {
 tasks.named("thirdPartyAudit").configure { enabled = false }
 
 // JarHell is part of es server, which we don't want to pull in
-// TODO: Not anymore. Now in :libs:elasticsearch-core
+// TODO: Not anymore. Now in :libs:core
 tasks.named("jarHell").configure { enabled = false }
 
 // TODO: should we have licenses for our test deps?
diff --git a/distribution/build.gradle b/distribution/build.gradle
index 72dea714fdcdb..f7b6f7bc1c7d0 100644
--- a/distribution/build.gradle
+++ b/distribution/build.gradle
@@ -275,7 +275,7 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
     }
     all {
       resolutionStrategy.dependencySubstitution {
-        substitute module("org.apache.logging.log4j:log4j-core") using project(":libs:elasticsearch-log4j") because "patched to remove JndiLookup clas"}
+        substitute module("org.apache.logging.log4j:log4j-core") using project(":libs:log4j") because "patched to remove JndiLookup clas"}
     }
   }
 
@@ -291,7 +291,7 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
     libsKeystoreCli project(path: ':distribution:tools:keystore-cli')
     libsSecurityCli project(':x-pack:plugin:security:cli')
     libsGeoIpCli project(':distribution:tools:geoip-cli')
-    libsNative project(':libs:elasticsearch-native:elasticsearch-native-libraries')
+    libsNative project(':libs:native:native-libraries')
   }
 
   project.ext {
diff --git a/distribution/tools/entitlement-agent/build.gradle b/distribution/tools/entitlement-agent/build.gradle
index 3fa9d0f5ef83a..d3e7ae10dcc6d 100644
--- a/distribution/tools/entitlement-agent/build.gradle
+++ b/distribution/tools/entitlement-agent/build.gradle
@@ -22,7 +22,7 @@ configurations {
 
 dependencies {
   entitlementBridge project(":distribution:tools:entitlement-bridge")
-  compileOnly project(":libs:elasticsearch-core")
+  compileOnly project(":libs:core")
   compileOnly project(":distribution:tools:entitlement-runtime")
   testImplementation project(":test:framework")
   testImplementation project(":distribution:tools:entitlement-bridge")
diff --git a/distribution/tools/entitlement-runtime/build.gradle b/distribution/tools/entitlement-runtime/build.gradle
index 55471272c1b5f..aaeee76d8bc57 100644
--- a/distribution/tools/entitlement-runtime/build.gradle
+++ b/distribution/tools/entitlement-runtime/build.gradle
@@ -10,8 +10,8 @@ apply plugin: 'elasticsearch.build'
 apply plugin: 'elasticsearch.publish'
 
 dependencies {
-  compileOnly project(':libs:elasticsearch-core') // For @SuppressForbidden
-  compileOnly project(":libs:elasticsearch-x-content") // for parsing policy files
+  compileOnly project(':libs:core') // For @SuppressForbidden
+  compileOnly project(":libs:x-content") // for parsing policy files
   compileOnly project(':server') // To access the main server module for special permission checks
   compileOnly project(':distribution:tools:entitlement-bridge')
   testImplementation project(":test:framework")
diff --git a/distribution/tools/geoip-cli/build.gradle b/distribution/tools/geoip-cli/build.gradle
index ee20d5e1bd88e..26af3bb4f9911 100644
--- a/distribution/tools/geoip-cli/build.gradle
+++ b/distribution/tools/geoip-cli/build.gradle
@@ -15,8 +15,8 @@ base {
 
 dependencies {
   compileOnly project(":server")
-  compileOnly project(":libs:elasticsearch-cli")
-  compileOnly project(":libs:elasticsearch-x-content")
+  compileOnly project(":libs:cli")
+  compileOnly project(":libs:x-content")
   testImplementation project(":test:framework")
   testImplementation "org.apache.commons:commons-compress:1.26.1"
   testImplementation "commons-io:commons-io:2.15.1"
diff --git a/distribution/tools/keystore-cli/build.gradle b/distribution/tools/keystore-cli/build.gradle
index 07aa92151171a..0140cd9d8eedf 100644
--- a/distribution/tools/keystore-cli/build.gradle
+++ b/distribution/tools/keystore-cli/build.gradle
@@ -11,7 +11,7 @@ apply plugin: 'elasticsearch.build'
 
 dependencies {
   compileOnly project(":server")
-  compileOnly project(":libs:elasticsearch-cli")
+  compileOnly project(":libs:cli")
   testImplementation project(":test:framework")
   testImplementation "com.google.jimfs:jimfs:${versions.jimfs}"
   testRuntimeOnly "com.google.guava:guava:${versions.jimfs_guava}"
diff --git a/distribution/tools/plugin-cli/build.gradle b/distribution/tools/plugin-cli/build.gradle
index 16932df96e223..ac8ade89c9014 100644
--- a/distribution/tools/plugin-cli/build.gradle
+++ b/distribution/tools/plugin-cli/build.gradle
@@ -21,9 +21,9 @@ tasks.named("dependencyLicenses").configure {
 
 dependencies {
   compileOnly project(":server")
-  compileOnly project(":libs:elasticsearch-cli")
-  implementation project(":libs:elasticsearch-plugin-api")
-  implementation project(":libs:elasticsearch-plugin-scanner")
+  compileOnly project(":libs:cli")
+  implementation project(":libs:plugin-api")
+  implementation project(":libs:plugin-scanner")
   // TODO: asm is picked up from the plugin scanner, we should consolidate so it is not defined twice
   implementation 'org.ow2.asm:asm:9.7'
   implementation 'org.ow2.asm:asm-tree:9.7'
diff --git a/distribution/tools/server-cli/build.gradle b/distribution/tools/server-cli/build.gradle
index e8f70e9053d7c..299d511ba5dbe 100644
--- a/distribution/tools/server-cli/build.gradle
+++ b/distribution/tools/server-cli/build.gradle
@@ -12,7 +12,7 @@ apply plugin: 'elasticsearch.build'
 
 dependencies {
   compileOnly project(":server")
-  compileOnly project(":libs:elasticsearch-cli")
+  compileOnly project(":libs:cli")
 
   testImplementation project(":test:framework")
 }
diff --git a/distribution/tools/windows-service-cli/build.gradle b/distribution/tools/windows-service-cli/build.gradle
index 77da0d407a40d..dcfaf244b7eec 100644
--- a/distribution/tools/windows-service-cli/build.gradle
+++ b/distribution/tools/windows-service-cli/build.gradle
@@ -2,7 +2,7 @@ apply plugin: 'elasticsearch.build'
 
 dependencies {
   compileOnly project(":server")
-  compileOnly project(":libs:elasticsearch-cli")
+  compileOnly project(":libs:cli")
   compileOnly project(":distribution:tools:server-cli")
 
   testImplementation project(":test:framework")
diff --git a/docs/changelog/115266.yaml b/docs/changelog/115266.yaml
new file mode 100644
index 0000000000000..1d7fb1368c0e8
--- /dev/null
+++ b/docs/changelog/115266.yaml
@@ -0,0 +1,6 @@
+pr: 115266
+summary: ES|QL CCS uses `skip_unavailable` setting for handling disconnected remote
+  clusters
+area: ES|QL
+type: enhancement
+issues: [ 114531 ]
diff --git a/docs/changelog/115807.yaml b/docs/changelog/115807.yaml
new file mode 100644
index 0000000000000..d17cabca4bd03
--- /dev/null
+++ b/docs/changelog/115807.yaml
@@ -0,0 +1,5 @@
+pr: 115807
+summary: "[Inference API] Improve chunked results error message"
+area: Machine Learning
+type: enhancement
+issues: []
diff --git a/docs/changelog/115823.yaml b/docs/changelog/115823.yaml
new file mode 100644
index 0000000000000..a6119e0fa56e4
--- /dev/null
+++ b/docs/changelog/115823.yaml
@@ -0,0 +1,5 @@
+pr: 115823
+summary: Add ECK Role Mapping Cleanup
+area: Security
+type: bug
+issues: []
diff --git a/docs/changelog/115831.yaml b/docs/changelog/115831.yaml
new file mode 100644
index 0000000000000..18442ec3b97e6
--- /dev/null
+++ b/docs/changelog/115831.yaml
@@ -0,0 +1,13 @@
+pr: 115831
+summary: Increase minimum threshold in shard balancer
+area: Allocation
+type: breaking
+issues: []
+breaking:
+  title: Minimum shard balancer threshold is now 1.0
+  area: Cluster and node setting
+  details: >-
+    Earlier versions of {es} accepted any non-negative value for `cluster.routing.allocation.balance.threshold`, but values smaller than
+    `1.0` do not make sense and have been ignored since version 8.6.1. From 9.0.0 these nonsensical values are now forbidden.
+  impact: Do not set `cluster.routing.allocation.balance.threshold` to a value less than `1.0`.
+  notable: false
diff --git a/docs/changelog/115834.yaml b/docs/changelog/115834.yaml
new file mode 100644
index 0000000000000..91f9e9a4e2e41
--- /dev/null
+++ b/docs/changelog/115834.yaml
@@ -0,0 +1,5 @@
+pr: 115834
+summary: Try to simplify geometries that fail with `TopologyException`
+area: Geo
+type: bug
+issues: []
diff --git a/docs/changelog/115836.yaml b/docs/changelog/115836.yaml
new file mode 100644
index 0000000000000..f6da638f1feff
--- /dev/null
+++ b/docs/changelog/115836.yaml
@@ -0,0 +1,5 @@
+pr: 115836
+summary: Catch and handle disconnect exceptions in search
+area: Search
+type: bug
+issues: []
diff --git a/docs/reference/landing-page.asciidoc b/docs/reference/landing-page.asciidoc
index f1b5ce8210996..1f2145a3aae82 100644
--- a/docs/reference/landing-page.asciidoc
+++ b/docs/reference/landing-page.asciidoc
@@ -128,7 +128,7 @@
       <a href="https://www.elastic.co/guide/en/cloud/current/ec-cloud-ingest-data.html">Adding data to Elasticsearch</a>
     </li>
     <li>
-      <a href="https://www.elastic.co/guide/en/enterprise-search/current/connectors.html">Connectors</a>
+      <a href="es-connectors.html">Connectors</a>
     </li>
     <li>
       <a href="https://www.elastic.co/guide/en/enterprise-search/current/crawler.html">Web crawler</a>
diff --git a/libs/build.gradle b/libs/build.gradle
index d0dfabd9c4fc5..efd2329ca2b5e 100644
--- a/libs/build.gradle
+++ b/libs/build.gradle
@@ -7,10 +7,42 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-configure(childProjects.values() - project('elasticsearch-log4j')) {
+configure(childProjects.values()) {
+
+  apply plugin: 'base'
+
   /*
-   * All subprojects are java projects using Elasticsearch's standard build
-   * tools.
+   * Although these libs are local to Elasticsearch, they can conflict with other similarly
+   * named libraries when downloaded into a single directory via maven. Here we set the
+   * name of all libs to begin with the "elasticsearch-" prefix. Additionally, subprojects
+   * of libs begin with their parents artifactId.
    */
-  apply plugin: 'elasticsearch.build'
+  def baseProject = project
+  def baseArtifactId = "elasticsearch-${it.name}"
+  base {
+    archivesName = baseArtifactId
+  }
+  subprojects {
+    apply plugin: 'base'
+
+    def subArtifactId = baseArtifactId
+    def currentProject = project
+    while (currentProject != baseProject) {
+      subArtifactId += "-${currentProject.name}"
+      currentProject = currentProject.parent
+    }
+    base {
+      archivesName = subArtifactId
+    }
+  }
+
+  // log4j is a hack, and not really a full elasticsearch built jar
+  if (project.name != 'log4j') {
+
+    /*
+     * All subprojects are java projects using Elasticsearch's standard build
+     * tools.
+     */
+    apply plugin: 'elasticsearch.build'
+  }
 }
diff --git a/libs/cli/build.gradle b/libs/cli/build.gradle
index b6ae962eaa603..d5842d4a2c59c 100644
--- a/libs/cli/build.gradle
+++ b/libs/cli/build.gradle
@@ -11,10 +11,10 @@ apply plugin: 'elasticsearch.publish'
 
 dependencies {
   api 'net.sf.jopt-simple:jopt-simple:5.0.2'
-  api project(':libs:elasticsearch-core')
+  api project(':libs:core')
 
   testImplementation(project(":test:framework")) {
-    exclude group: 'org.elasticsearch', module: 'elasticsearch-cli'
+    exclude group: 'org.elasticsearch', module: 'cli'
   }
 }
 
diff --git a/libs/core/build.gradle b/libs/core/build.gradle
index ebbeac141e4bd..e24417e09a53d 100644
--- a/libs/core/build.gradle
+++ b/libs/core/build.gradle
@@ -13,19 +13,19 @@ apply plugin: 'elasticsearch.mrjar'
 dependencies {
   // This dependency is used only by :libs:core for null-checking interop with other tools
   compileOnly "com.google.code.findbugs:jsr305:3.0.2"
-  compileOnly project(':libs:elasticsearch-logging')
+  compileOnly project(':libs:logging')
 
   testImplementation "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}"
   testImplementation "junit:junit:${versions.junit}"
   testImplementation "org.hamcrest:hamcrest:${versions.hamcrest}"
 
   testImplementation(project(":test:framework")) {
-    exclude group: 'org.elasticsearch', module: 'elasticsearch-core'
+    exclude group: 'org.elasticsearch', module: 'core'
   }
 }
 
 tasks.named('forbiddenApisMain').configure {
-  // :libs:elasticsearch-core does not depend on server
+  // :libs:core does not depend on server
   // TODO: Need to decide how we want to handle for forbidden signatures with the changes to server
   replaceSignatureFiles 'jdk-signatures'
 }
diff --git a/libs/dissect/build.gradle b/libs/dissect/build.gradle
index be2691bfd332f..f1a09cc0ba0e6 100644
--- a/libs/dissect/build.gradle
+++ b/libs/dissect/build.gradle
@@ -9,7 +9,7 @@
 
 dependencies {
   testImplementation(project(":test:framework")) {
-    exclude group: 'org.elasticsearch', module: 'elasticsearch-dissect'
+    exclude group: 'org.elasticsearch', module: 'dissect'
   }
   testImplementation "com.fasterxml.jackson.core:jackson-core:${versions.jackson}"
   testImplementation "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}"
diff --git a/libs/geo/build.gradle b/libs/geo/build.gradle
index 37dd65cb19262..c753ba814a5f9 100644
--- a/libs/geo/build.gradle
+++ b/libs/geo/build.gradle
@@ -12,7 +12,7 @@ apply plugin: 'elasticsearch.publish'
 
 dependencies {
   testImplementation(project(":test:framework")) {
-    exclude group: 'org.elasticsearch', module: 'elasticsearch-geo'
+    exclude group: 'org.elasticsearch', module: 'geo'
   }
 }
 
diff --git a/libs/grok/build.gradle b/libs/grok/build.gradle
index ce4be699953c7..2a74927fedd83 100644
--- a/libs/grok/build.gradle
+++ b/libs/grok/build.gradle
@@ -14,7 +14,7 @@ dependencies {
   api 'org.jruby.jcodings:jcodings:1.0.44'
 
   testImplementation(project(":test:framework")) {
-    exclude group: 'org.elasticsearch', module: 'elasticsearch-grok'
+    exclude group: 'org.elasticsearch', module: 'grok'
   }
 }
 
diff --git a/libs/h3/build.gradle b/libs/h3/build.gradle
index 0eb1aea09d49c..81a0d56ed4606 100644
--- a/libs/h3/build.gradle
+++ b/libs/h3/build.gradle
@@ -23,7 +23,7 @@ apply plugin: 'elasticsearch.publish'
 
 dependencies {
   testImplementation(project(":test:framework")) {
-    exclude group: 'org.elasticsearch', module: 'elasticsearch-geo'
+    exclude group: 'org.elasticsearch', module: 'geo'
   }
   // lucene topology library that uses spherical geometry
   testImplementation "org.apache.lucene:lucene-spatial3d:${versions.lucene}"
@@ -40,4 +40,4 @@ licenseFile.set(rootProject.file('licenses/APACHE-LICENSE-2.0.txt'))
 
 tasks.withType(LicenseHeadersTask.class).configureEach {
   approvedLicenses = ['Apache', 'Generated', 'Vendored']
-}
\ No newline at end of file
+}
diff --git a/libs/logging/build.gradle b/libs/logging/build.gradle
index 4222d89ebe2da..f52c2629176a7 100644
--- a/libs/logging/build.gradle
+++ b/libs/logging/build.gradle
@@ -14,12 +14,12 @@ tasks.named("loggerUsageCheck").configure {enabled = false }
 
 dependencies {
   testImplementation(project(":test:framework")) {
-    exclude group: 'org.elasticsearch', module: 'elasticsearch-logging'
+    exclude group: 'org.elasticsearch', module: 'logging'
   }
 }
 
 
 tasks.named('forbiddenApisMain').configure {
-  // :libs:elasticsearch-logging does not depend on server
+  // :libs:logging does not depend on server
   replaceSignatureFiles 'jdk-signatures'
 }
diff --git a/libs/logstash-bridge/build.gradle b/libs/logstash-bridge/build.gradle
index e4b2728f693a0..117bed1e98105 100644
--- a/libs/logstash-bridge/build.gradle
+++ b/libs/logstash-bridge/build.gradle
@@ -10,9 +10,9 @@ apply plugin: 'elasticsearch.build'
 
 dependencies {
   compileOnly project(':server')
-  compileOnly project(':libs:elasticsearch-core')
-  compileOnly project(':libs:elasticsearch-plugin-api')
-  compileOnly project(':libs:elasticsearch-x-content')
+  compileOnly project(':libs:core')
+  compileOnly project(':libs:plugin-api')
+  compileOnly project(':libs:x-content')
   compileOnly project(':modules:lang-painless')
   compileOnly project(':modules:lang-painless:spi')
   compileOnly project(':modules:lang-mustache')
diff --git a/libs/lz4/build.gradle b/libs/lz4/build.gradle
index d9f1175248121..72e1bb50187a7 100644
--- a/libs/lz4/build.gradle
+++ b/libs/lz4/build.gradle
@@ -10,10 +10,10 @@ apply plugin: 'elasticsearch.publish'
 
 dependencies {
   api 'org.lz4:lz4-java:1.8.0'
-  api project(':libs:elasticsearch-core')
+  api project(':libs:core')
 
   testImplementation(project(":test:framework")) {
-    exclude group: 'org.elasticsearch', module: 'elasticsearch-lz4'
+    exclude group: 'org.elasticsearch', module: 'lz4'
   }
 }
 
diff --git a/libs/native/build.gradle b/libs/native/build.gradle
index 0c889d47566fb..eff8f82434461 100644
--- a/libs/native/build.gradle
+++ b/libs/native/build.gradle
@@ -14,10 +14,10 @@ apply plugin: 'elasticsearch.build'
 apply plugin: 'elasticsearch.mrjar'
 
 dependencies {
-  api project(':libs:elasticsearch-core')
-  api project(':libs:elasticsearch-logging')
+  api project(':libs:core')
+  api project(':libs:logging')
   testImplementation(project(":test:framework")) {
-    exclude group: 'org.elasticsearch', module: 'elasticsearch-native'
+    exclude group: 'org.elasticsearch', module: 'native'
   }
 }
 
diff --git a/libs/plugin-analysis-api/build.gradle b/libs/plugin-analysis-api/build.gradle
index e240f18a88e0a..3f1670d76a0c1 100644
--- a/libs/plugin-analysis-api/build.gradle
+++ b/libs/plugin-analysis-api/build.gradle
@@ -18,12 +18,12 @@ tasks.named("loggerUsageCheck").configure {enabled = false }
 
 dependencies {
   api "org.apache.lucene:lucene-core:${versions.lucene}"
-  api project(':libs:elasticsearch-plugin-api')
+  api project(':libs:plugin-api')
 }
 
 
 tasks.named('forbiddenApisMain').configure {
-  // :libs:elasticsearch-logging does not depend on server
+  // :libs:logging does not depend on server
   replaceSignatureFiles 'jdk-signatures'
 }
 
diff --git a/libs/plugin-scanner/build.gradle b/libs/plugin-scanner/build.gradle
index b8cd224eba46a..d04af0624b3b1 100644
--- a/libs/plugin-scanner/build.gradle
+++ b/libs/plugin-scanner/build.gradle
@@ -16,16 +16,16 @@ tasks.named("dependencyLicenses").configure {
 }
 
 dependencies {
-  api project(':libs:elasticsearch-core')
-  api project(':libs:elasticsearch-plugin-api')
-  api project(":libs:elasticsearch-x-content")
+  api project(':libs:core')
+  api project(':libs:plugin-api')
+  api project(":libs:x-content")
 
   api 'org.ow2.asm:asm:9.7'
   api 'org.ow2.asm:asm-tree:9.7'
 
   testImplementation "junit:junit:${versions.junit}"
   testImplementation(project(":test:framework")) {
-    exclude group: 'org.elasticsearch', module: 'elasticsearch-plugin-scanner'
+    exclude group: 'org.elasticsearch', module: 'plugin-scanner'
   }
 }
 tasks.named('forbiddenApisMain').configure {
diff --git a/libs/secure-sm/build.gradle b/libs/secure-sm/build.gradle
index 5e35f3ac7126f..473a86215e91e 100644
--- a/libs/secure-sm/build.gradle
+++ b/libs/secure-sm/build.gradle
@@ -16,7 +16,7 @@ dependencies {
   testImplementation "org.hamcrest:hamcrest:${versions.hamcrest}"
 
   testImplementation(project(":test:framework")) {
-    exclude group: 'org.elasticsearch', module: 'elasticsearch-secure-sm'
+    exclude group: 'org.elasticsearch', module: 'secure-sm'
   }
 }
 
diff --git a/libs/simdvec/build.gradle b/libs/simdvec/build.gradle
index eee56be72d0bf..02f960130e690 100644
--- a/libs/simdvec/build.gradle
+++ b/libs/simdvec/build.gradle
@@ -15,12 +15,12 @@ apply plugin: 'elasticsearch.build'
 apply plugin: 'elasticsearch.mrjar'
 
 dependencies {
-  implementation project(':libs:elasticsearch-native')
-  implementation project(':libs:elasticsearch-logging')
+  implementation project(':libs:native')
+  implementation project(':libs:logging')
   implementation "org.apache.lucene:lucene-core:${versions.lucene}"
 
   testImplementation(project(":test:framework")) {
-    exclude group: 'org.elasticsearch', module: 'elasticsearch-native'
+    exclude group: 'org.elasticsearch', module: 'native'
   }
 }
 
diff --git a/libs/ssl-config/build.gradle b/libs/ssl-config/build.gradle
index 3c0eb7c440510..d63df95003ab6 100644
--- a/libs/ssl-config/build.gradle
+++ b/libs/ssl-config/build.gradle
@@ -9,10 +9,10 @@
 apply plugin: "elasticsearch.publish"
 
 dependencies {
-  api project(':libs:elasticsearch-core')
+  api project(':libs:core')
 
   testImplementation(project(":test:framework")) {
-    exclude group: 'org.elasticsearch', module: 'elasticsearch-ssl-config'
+    exclude group: 'org.elasticsearch', module: 'ssl-config'
   }
 
   testImplementation "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}"
diff --git a/libs/tdigest/build.gradle b/libs/tdigest/build.gradle
index 231eb845339aa..2713df701fb44 100644
--- a/libs/tdigest/build.gradle
+++ b/libs/tdigest/build.gradle
@@ -22,11 +22,11 @@ apply plugin: 'elasticsearch.build'
 apply plugin: 'elasticsearch.publish'
 
 dependencies {
-  api project(':libs:elasticsearch-core')
+  api project(':libs:core')
   api "org.apache.lucene:lucene-core:${versions.lucene}"
 
   testImplementation(project(":test:framework")) {
-    exclude group: 'org.elasticsearch', module: 'elasticsearch-tdigest'
+    exclude group: 'org.elasticsearch', module: 'tdigest'
   }
   testImplementation 'org.junit.jupiter:junit-jupiter:5.8.1'
 }
diff --git a/libs/x-content/build.gradle b/libs/x-content/build.gradle
index 7540bd0fb68f0..1cf18d46e7610 100644
--- a/libs/x-content/build.gradle
+++ b/libs/x-content/build.gradle
@@ -12,14 +12,14 @@ apply plugin: 'elasticsearch.publish'
 apply plugin: 'elasticsearch.embedded-providers'
 
 embeddedProviders {
-  impl 'x-content', project(':libs:elasticsearch-x-content:impl')
+  impl 'x-content', project(':libs:x-content:impl')
 }
 
 dependencies {
-  api project(':libs:elasticsearch-core')
+  api project(':libs:core')
 
   testImplementation(project(":test:framework")) {
-    exclude group: 'org.elasticsearch', module: 'elasticsearch-x-content'
+    exclude group: 'org.elasticsearch', module: 'x-content'
   }
 }
 
diff --git a/libs/x-content/impl/build.gradle b/libs/x-content/impl/build.gradle
index 753d2c3d5fe1e..35e122d336c68 100644
--- a/libs/x-content/impl/build.gradle
+++ b/libs/x-content/impl/build.gradle
@@ -16,8 +16,8 @@ base {
 String jacksonVersion = "2.17.2"
 
 dependencies {
-  compileOnly project(':libs:elasticsearch-core')
-  compileOnly project(':libs:elasticsearch-x-content')
+  compileOnly project(':libs:core')
+  compileOnly project(':libs:x-content')
   implementation "com.fasterxml.jackson.core:jackson-core:${jacksonVersion}"
   implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-smile:${jacksonVersion}"
   implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:${jacksonVersion}"
@@ -25,7 +25,7 @@ dependencies {
   implementation "org.yaml:snakeyaml:${versions.snakeyaml}"
 
   testImplementation(project(":test:framework")) {
-    exclude group: 'org.elasticsearch', module: 'elasticsearch-x-content'
+    exclude group: 'org.elasticsearch', module: 'x-content'
   }
 }
 
diff --git a/modules/ingest-common/build.gradle b/modules/ingest-common/build.gradle
index 98dacce01fba4..7cfdba4d33744 100644
--- a/modules/ingest-common/build.gradle
+++ b/modules/ingest-common/build.gradle
@@ -20,8 +20,8 @@ esplugin {
 
 dependencies {
   compileOnly project(':modules:lang-painless:spi')
-  api project(':libs:elasticsearch-grok')
-  api project(':libs:elasticsearch-dissect')
+  api project(':libs:grok')
+  api project(':libs:dissect')
   implementation "org.apache.httpcomponents:httpclient:${versions.httpclient}"
   implementation "org.apache.httpcomponents:httpcore:${versions.httpcore}"
 }
diff --git a/modules/reindex/build.gradle b/modules/reindex/build.gradle
index ac68b565a0fbe..14a6b1e3f5b82 100644
--- a/modules/reindex/build.gradle
+++ b/modules/reindex/build.gradle
@@ -38,7 +38,7 @@ testClusters.configureEach {
 
 dependencies {
   api project(":client:rest")
-  api project(":libs:elasticsearch-ssl-config")
+  api project(":libs:ssl-config")
   // for parent/child testing
   testImplementation project(':modules:parent-join')
   testImplementation project(':modules:rest-root')
diff --git a/modules/runtime-fields-common/build.gradle b/modules/runtime-fields-common/build.gradle
index 00bb17df8665e..e743939cbf79e 100644
--- a/modules/runtime-fields-common/build.gradle
+++ b/modules/runtime-fields-common/build.gradle
@@ -19,8 +19,8 @@ esplugin {
 
 dependencies {
   compileOnly project(':modules:lang-painless:spi')
-  api project(':libs:elasticsearch-grok')
-  api project(':libs:elasticsearch-dissect')
+  api project(':libs:grok')
+  api project(':libs:dissect')
 }
 
 tasks.named("yamlRestCompatTestTransform").configure({ task ->
diff --git a/modules/systemd/build.gradle b/modules/systemd/build.gradle
index 28fd36160936a..8eb48e1d5f638 100644
--- a/modules/systemd/build.gradle
+++ b/modules/systemd/build.gradle
@@ -13,6 +13,6 @@ esplugin {
 }
 
 dependencies {
-  implementation project(':libs:elasticsearch-native')
+  implementation project(':libs:native')
 }
 
diff --git a/modules/transport-netty4/build.gradle b/modules/transport-netty4/build.gradle
index d80b63bec53d8..8dc718a818cec 100644
--- a/modules/transport-netty4/build.gradle
+++ b/modules/transport-netty4/build.gradle
@@ -35,7 +35,7 @@ configurations {
 }
 
 dependencies {
-  api project(":libs:elasticsearch-ssl-config")
+  api project(":libs:ssl-config")
 
   // network stack
   api "io.netty:netty-buffer:${versions.netty}"
@@ -244,4 +244,4 @@ tasks.named("thirdPartyAudit").configure {
 
 tasks.named('forbiddenApisMain').configure {
   signaturesFiles += files('forbidden/netty-signatures.txt')
-}
\ No newline at end of file
+}
diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpPipeliningHandler.java b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpPipeliningHandler.java
index b08c93a4dc240..1a391a05add58 100644
--- a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpPipeliningHandler.java
+++ b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpPipeliningHandler.java
@@ -137,7 +137,10 @@ public void channelRead(final ChannelHandlerContext ctx, final Object msg) {
                         netty4HttpRequest = new Netty4HttpRequest(readSequence++, fullHttpRequest);
                         currentRequestStream = null;
                     } else {
-                        var contentStream = new Netty4HttpRequestBodyStream(ctx.channel());
+                        var contentStream = new Netty4HttpRequestBodyStream(
+                            ctx.channel(),
+                            serverTransport.getThreadPool().getThreadContext()
+                        );
                         currentRequestStream = contentStream;
                         netty4HttpRequest = new Netty4HttpRequest(readSequence++, request, contentStream);
                     }
diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpRequestBodyStream.java b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpRequestBodyStream.java
index 9a0dc09b7566c..238faa7a9237e 100644
--- a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpRequestBodyStream.java
+++ b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpRequestBodyStream.java
@@ -16,6 +16,7 @@
 import io.netty.handler.codec.http.HttpContent;
 import io.netty.handler.codec.http.LastHttpContent;
 
+import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.core.Releasables;
 import org.elasticsearch.http.HttpBody;
 import org.elasticsearch.transport.netty4.Netty4Utils;
@@ -34,14 +35,18 @@ public class Netty4HttpRequestBodyStream implements HttpBody.Stream {
     private final Channel channel;
     private final ChannelFutureListener closeListener = future -> doClose();
     private final List<ChunkHandler> tracingHandlers = new ArrayList<>(4);
+    private final ThreadContext threadContext;
     private ByteBuf buf;
     private boolean hasLast = false;
     private boolean requested = false;
     private boolean closing = false;
     private HttpBody.ChunkHandler handler;
+    private ThreadContext.StoredContext requestContext;
 
-    public Netty4HttpRequestBodyStream(Channel channel) {
+    public Netty4HttpRequestBodyStream(Channel channel, ThreadContext threadContext) {
         this.channel = channel;
+        this.threadContext = threadContext;
+        this.requestContext = threadContext.newStoredContext();
         Netty4Utils.addListener(channel.closeFuture(), closeListener);
         channel.config().setAutoRead(false);
     }
@@ -66,6 +71,7 @@ public void addTracingHandler(ChunkHandler chunkHandler) {
     public void next() {
         assert closing == false : "cannot request next chunk on closing stream";
         assert handler != null : "handler must be set before requesting next chunk";
+        requestContext = threadContext.newStoredContext();
         channel.eventLoop().submit(() -> {
             requested = true;
             if (buf == null) {
@@ -108,11 +114,6 @@ private void addChunk(ByteBuf chunk) {
         }
     }
 
-    // visible for test
-    Channel channel() {
-        return channel;
-    }
-
     // visible for test
     ByteBuf buf() {
         return buf;
@@ -129,10 +130,12 @@ private void send() {
         var bytesRef = Netty4Utils.toReleasableBytesReference(buf);
         requested = false;
         buf = null;
-        for (var tracer : tracingHandlers) {
-            tracer.onNext(bytesRef, hasLast);
+        try (var ignored = threadContext.restoreExistingContext(requestContext)) {
+            for (var tracer : tracingHandlers) {
+                tracer.onNext(bytesRef, hasLast);
+            }
+            handler.onNext(bytesRef, hasLast);
         }
-        handler.onNext(bytesRef, hasLast);
         if (hasLast) {
             channel.config().setAutoRead(true);
             channel.closeFuture().removeListener(closeListener);
@@ -150,11 +153,13 @@ public void close() {
 
     private void doClose() {
         closing = true;
-        for (var tracer : tracingHandlers) {
-            Releasables.closeExpectNoException(tracer);
-        }
-        if (handler != null) {
-            handler.close();
+        try (var ignored = threadContext.restoreExistingContext(requestContext)) {
+            for (var tracer : tracingHandlers) {
+                Releasables.closeExpectNoException(tracer);
+            }
+            if (handler != null) {
+                handler.close();
+            }
         }
         if (buf != null) {
             buf.release();
diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpRequestBodyStreamTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpRequestBodyStreamTests.java
index f495883631a4e..5ff5a27e2d551 100644
--- a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpRequestBodyStreamTests.java
+++ b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpRequestBodyStreamTests.java
@@ -19,24 +19,33 @@
 import io.netty.handler.flow.FlowControlHandler;
 
 import org.elasticsearch.common.bytes.ReleasableBytesReference;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.http.HttpBody;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Map;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static org.hamcrest.Matchers.hasEntry;
 
 public class Netty4HttpRequestBodyStreamTests extends ESTestCase {
 
-    EmbeddedChannel channel;
-    Netty4HttpRequestBodyStream stream;
+    private final ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
+    private EmbeddedChannel channel;
+    private Netty4HttpRequestBodyStream stream;
     static HttpBody.ChunkHandler discardHandler = (chunk, isLast) -> chunk.close();
 
     @Override
     public void setUp() throws Exception {
         super.setUp();
         channel = new EmbeddedChannel();
-        stream = new Netty4HttpRequestBodyStream(channel);
+        threadContext.putHeader("header1", "value1");
+        stream = new Netty4HttpRequestBodyStream(channel, threadContext);
         stream.setHandler(discardHandler); // set default handler, each test might override one
         channel.pipeline().addLast(new SimpleChannelInboundHandler<HttpContent>(false) {
             @Override
@@ -118,6 +127,60 @@ public void testReadFromChannel() {
         assertTrue("should receive last content", gotLast.get());
     }
 
+    public void testReadFromHasCorrectThreadContext() throws InterruptedException {
+        var gotLast = new AtomicBoolean(false);
+        AtomicReference<Map<String, String>> headers = new AtomicReference<>();
+        stream.setHandler(new HttpBody.ChunkHandler() {
+            @Override
+            public void onNext(ReleasableBytesReference chunk, boolean isLast) {
+                headers.set(threadContext.getHeaders());
+                gotLast.set(isLast);
+                chunk.close();
+            }
+
+            @Override
+            public void close() {
+                headers.set(threadContext.getHeaders());
+            }
+        });
+        channel.pipeline().addFirst(new FlowControlHandler()); // block all incoming messages, need explicit channel.read()
+        var chunkSize = 1024;
+
+        channel.writeInbound(randomContent(chunkSize));
+        channel.writeInbound(randomLastContent(chunkSize));
+
+        threadContext.putHeader("header2", "value2");
+        stream.next();
+
+        Thread thread = new Thread(() -> channel.runPendingTasks());
+        thread.start();
+        thread.join();
+
+        assertThat(headers.get(), hasEntry("header1", "value1"));
+        assertThat(headers.get(), hasEntry("header2", "value2"));
+
+        threadContext.putHeader("header3", "value3");
+        stream.next();
+
+        thread = new Thread(() -> channel.runPendingTasks());
+        thread.start();
+        thread.join();
+
+        assertThat(headers.get(), hasEntry("header1", "value1"));
+        assertThat(headers.get(), hasEntry("header2", "value2"));
+        assertThat(headers.get(), hasEntry("header3", "value3"));
+
+        assertTrue("should receive last content", gotLast.get());
+
+        headers.set(new HashMap<>());
+
+        stream.close();
+
+        assertThat(headers.get(), hasEntry("header1", "value1"));
+        assertThat(headers.get(), hasEntry("header2", "value2"));
+        assertThat(headers.get(), hasEntry("header3", "value3"));
+    }
+
     HttpContent randomContent(int size, boolean isLast) {
         var buf = Unpooled.wrappedBuffer(randomByteArrayOfLength(size));
         if (isLast) {
diff --git a/muted-tests.yml b/muted-tests.yml
index 4315f1283a347..ddb50c5a829f9 100644
--- a/muted-tests.yml
+++ b/muted-tests.yml
@@ -236,9 +236,6 @@ tests:
 - class: org.elasticsearch.xpack.inference.DefaultEndPointsIT
   method: testInferDeploysDefaultE5
   issue: https://github.com/elastic/elasticsearch/issues/115361
-- class: org.elasticsearch.reservedstate.service.FileSettingsServiceTests
-  method: testProcessFileChanges
-  issue: https://github.com/elastic/elasticsearch/issues/115280
 - class: org.elasticsearch.xpack.security.FileSettingsRoleMappingsRestartIT
   method: testFileSettingsReprocessedOnRestartWithoutVersionChange
   issue: https://github.com/elastic/elasticsearch/issues/115450
@@ -254,18 +251,12 @@ tests:
 - class: org.elasticsearch.oldrepos.OldRepositoryAccessIT
   method: testOldRepoAccess
   issue: https://github.com/elastic/elasticsearch/issues/115631
-- class: org.elasticsearch.index.get.GetResultTests
-  method: testToAndFromXContent
-  issue: https://github.com/elastic/elasticsearch/issues/115688
 - class: org.elasticsearch.action.update.UpdateResponseTests
   method: testToAndFromXContent
   issue: https://github.com/elastic/elasticsearch/issues/115689
 - class: org.elasticsearch.xpack.shutdown.NodeShutdownIT
   method: testStalledShardMigrationProperlyDetected
   issue: https://github.com/elastic/elasticsearch/issues/115697
-- class: org.elasticsearch.index.get.GetResultTests
-  method: testToAndFromXContentEmbedded
-  issue: https://github.com/elastic/elasticsearch/issues/115657
 - class: org.elasticsearch.xpack.spatial.search.GeoGridAggAndQueryConsistencyIT
   method: testGeoShapeGeoHash
   issue: https://github.com/elastic/elasticsearch/issues/115664
@@ -290,6 +281,9 @@ tests:
 - class: org.elasticsearch.search.StressSearchServiceReaperIT
   method: testStressReaper
   issue: https://github.com/elastic/elasticsearch/issues/115816
+- class: org.elasticsearch.reservedstate.service.FileSettingsServiceTests
+  method: testProcessFileChanges
+  issue: https://github.com/elastic/elasticsearch/issues/115280
 
 # Examples:
 #
@@ -328,4 +322,4 @@ tests:
 #    issue: "https://github.com/elastic/elasticsearch/..."
 #  - class: "org.elasticsearch.xpack.esql.**"
 #    method: "test {union_types.MultiIndexIpStringStatsInline *}"
-#    issue: "https://github.com/elastic/elasticsearch/..."
\ No newline at end of file
+#    issue: "https://github.com/elastic/elasticsearch/..."
diff --git a/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartIT.java b/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartIT.java
index 92a704f793dc2..fcca3f9a4700c 100644
--- a/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartIT.java
+++ b/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartIT.java
@@ -16,9 +16,11 @@
 import org.apache.http.util.EntityUtils;
 import org.elasticsearch.Build;
 import org.elasticsearch.client.Request;
+import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.client.RestClient;
+import org.elasticsearch.client.WarningsHandler;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.MetadataIndexStateService;
 import org.elasticsearch.common.Strings;
@@ -27,6 +29,7 @@
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.core.Booleans;
 import org.elasticsearch.core.CheckedFunction;
+import org.elasticsearch.core.UpdateForV10;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.IndexVersions;
@@ -72,6 +75,7 @@
 import static java.util.stream.Collectors.toList;
 import static org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.SYSTEM_INDEX_ENFORCEMENT_INDEX_VERSION;
 import static org.elasticsearch.cluster.routing.UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING;
+import static org.elasticsearch.cluster.routing.allocation.allocator.BalancedShardsAllocator.THRESHOLD_SETTING;
 import static org.elasticsearch.cluster.routing.allocation.decider.MaxRetryAllocationDecider.SETTING_ALLOCATION_MAX_RETRY;
 import static org.elasticsearch.test.MapMatcher.assertMap;
 import static org.elasticsearch.test.MapMatcher.matchesMap;
@@ -1949,4 +1953,35 @@ public static void assertNumHits(String index, int numHits, int totalShards) thr
         assertThat(XContentMapValues.extractValue("_shards.successful", resp), equalTo(totalShards));
         assertThat(extractTotalHits(resp), equalTo(numHits));
     }
+
+    @UpdateForV10(owner = UpdateForV10.Owner.DISTRIBUTED_COORDINATION) // this test is just about v8->v9 upgrades, remove it in v10
+    public void testBalancedShardsAllocatorThreshold() throws Exception {
+        assumeTrue("test only applies for v8->v9 upgrades", getOldClusterTestVersion().getMajor() == 8);
+
+        final var chosenValue = randomFrom("0", "0.1", "0.5", "0.999");
+
+        if (isRunningAgainstOldCluster()) {
+            final var request = newXContentRequest(
+                HttpMethod.PUT,
+                "/_cluster/settings",
+                (builder, params) -> builder.startObject("persistent").field(THRESHOLD_SETTING.getKey(), chosenValue).endObject()
+            );
+            request.setOptions(RequestOptions.DEFAULT.toBuilder().setWarningsHandler(WarningsHandler.PERMISSIVE));
+            assertOK(client().performRequest(request));
+        }
+
+        final var clusterSettingsResponse = ObjectPath.createFromResponse(
+            client().performRequest(new Request("GET", "/_cluster/settings"))
+        );
+
+        final var settingsPath = "persistent." + THRESHOLD_SETTING.getKey();
+        final var settingValue = clusterSettingsResponse.evaluate(settingsPath);
+
+        if (isRunningAgainstOldCluster()) {
+            assertEquals(chosenValue, settingValue);
+        } else {
+            assertNull(settingValue);
+            assertNotNull(clusterSettingsResponse.<String>evaluate("persistent.archived." + THRESHOLD_SETTING.getKey()));
+        }
+    }
 }
diff --git a/qa/logging-config/build.gradle b/qa/logging-config/build.gradle
index 78da8590660f7..4d65c4384afa1 100644
--- a/qa/logging-config/build.gradle
+++ b/qa/logging-config/build.gradle
@@ -10,7 +10,7 @@ apply plugin: 'elasticsearch.build'
 apply plugin: 'elasticsearch.legacy-java-rest-test'
 
 dependencies {
-  testImplementation project(":libs:elasticsearch-x-content")
+  testImplementation project(":libs:x-content")
   testImplementation project(":test:framework")
 
 }
diff --git a/qa/packaging/build.gradle b/qa/packaging/build.gradle
index 73b6507490185..f9a903223c88a 100644
--- a/qa/packaging/build.gradle
+++ b/qa/packaging/build.gradle
@@ -13,7 +13,7 @@ plugins {
 
 dependencies {
   testImplementation project(':server')
-  testImplementation project(':libs:elasticsearch-core')
+  testImplementation project(':libs:core')
   testImplementation(testArtifact(project(':x-pack:plugin:core')))
   testImplementation "junit:junit:${versions.junit}"
   testImplementation "org.hamcrest:hamcrest:${versions.hamcrest}"
diff --git a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/FileSettingsRoleMappingUpgradeIT.java b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/FileSettingsRoleMappingUpgradeIT.java
index 834d97f755dfb..4caf33feeeebb 100644
--- a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/FileSettingsRoleMappingUpgradeIT.java
+++ b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/FileSettingsRoleMappingUpgradeIT.java
@@ -23,19 +23,20 @@
 import org.junit.rules.TemporaryFolder;
 import org.junit.rules.TestRule;
 
-import java.io.IOException;
 import java.util.List;
+import java.util.Map;
 import java.util.function.Supplier;
 
+import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.hasItem;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.nullValue;
 
 public class FileSettingsRoleMappingUpgradeIT extends ParameterizedRollingUpgradeTestCase {
 
-    private static final String settingsJSON = """
+    private static final int ROLE_MAPPINGS_CLEANUP_MIGRATION_VERSION = 2;
+    private static final String SETTING_JSON = """
         {
              "metadata": {
                  "version": "1",
@@ -53,7 +54,6 @@ public class FileSettingsRoleMappingUpgradeIT extends ParameterizedRollingUpgrad
         }""";
 
     private static final TemporaryFolder repoDirectory = new TemporaryFolder();
-
     private static final ElasticsearchCluster cluster = ElasticsearchCluster.local()
         .distribution(DistributionType.DEFAULT)
         .version(getOldClusterTestVersion())
@@ -68,7 +68,7 @@ public String get() {
         .setting("xpack.security.enabled", "true")
         // workaround to avoid having to set up clients and authorization headers
         .setting("xpack.security.authc.anonymous.roles", "superuser")
-        .configFile("operator/settings.json", Resource.fromString(settingsJSON))
+        .configFile("operator/settings.json", Resource.fromString(SETTING_JSON))
         .build();
 
     @ClassRule
@@ -91,7 +91,30 @@ public void checkVersions() {
         );
     }
 
-    public void testRoleMappingsAppliedOnUpgrade() throws IOException {
+    private static void waitForSecurityMigrationCompletionIfIndexExists() throws Exception {
+        final Request request = new Request("GET", "_cluster/state/metadata/.security-7");
+        assertBusy(() -> {
+            Map<String, Object> indices = new XContentTestUtils.JsonMapView(entityAsMap(client().performRequest(request))).get(
+                "metadata.indices"
+            );
+            assertNotNull(indices);
+            // If the security index exists, migration needs to happen. There is a bug in pre cluster state role mappings code that tries
+            // to write file based role mappings before security index manager state is recovered, this makes it look like the security
+            // index is outdated (isIndexUpToDate == false). Because we can't rely on the index being there for old versions, this check
+            // is needed.
+            if (indices.containsKey(".security-7")) {
+                // JsonMapView doesn't support . prefixed indices (splits on .)
+                @SuppressWarnings("unchecked")
+                String responseVersion = new XContentTestUtils.JsonMapView((Map<String, Object>) indices.get(".security-7")).get(
+                    "migration_version.version"
+                );
+                assertNotNull(responseVersion);
+                assertTrue(Integer.parseInt(responseVersion) >= ROLE_MAPPINGS_CLEANUP_MIGRATION_VERSION);
+            }
+        });
+    }
+
+    public void testRoleMappingsAppliedOnUpgrade() throws Exception {
         if (isOldCluster()) {
             Request clusterStateRequest = new Request("GET", "/_cluster/state/metadata");
             List<Object> roleMappings = new XContentTestUtils.JsonMapView(entityAsMap(client().performRequest(clusterStateRequest))).get(
@@ -107,11 +130,10 @@ public void testRoleMappingsAppliedOnUpgrade() throws IOException {
             ).get("metadata.role_mappings.role_mappings");
             assertThat(clusterStateRoleMappings, is(not(nullValue())));
             assertThat(clusterStateRoleMappings.size(), equalTo(1));
-
+            waitForSecurityMigrationCompletionIfIndexExists();
             assertThat(
                 entityAsMap(client().performRequest(new Request("GET", "/_security/role_mapping"))).keySet(),
-                // TODO change this to `contains` once the clean-up migration work is merged
-                hasItem("everyone_kibana-read-only-operator-mapping")
+                contains("everyone_kibana-read-only-operator-mapping")
             );
         }
     }
diff --git a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/LogsdbIndexingRollingUpgradeIT.java b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/LogsdbIndexingRollingUpgradeIT.java
index 226cb3dda2ba1..9cb91438e09c0 100644
--- a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/LogsdbIndexingRollingUpgradeIT.java
+++ b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/LogsdbIndexingRollingUpgradeIT.java
@@ -28,10 +28,7 @@
 import static org.elasticsearch.upgrades.LogsIndexModeRollingUpgradeIT.enableLogsdbByDefault;
 import static org.elasticsearch.upgrades.LogsIndexModeRollingUpgradeIT.getWriteBackingIndex;
 import static org.elasticsearch.upgrades.TsdbIT.formatInstant;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.greaterThan;
-import static org.hamcrest.Matchers.greaterThanOrEqualTo;
-import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.*;
 
 public class LogsdbIndexingRollingUpgradeIT extends AbstractRollingUpgradeTestCase {
 
@@ -240,7 +237,7 @@ protected static void startTrial() throws IOException {
         } catch (ResponseException e) {
             var responseBody = entityAsMap(e.getResponse());
             String error = ObjectPath.evaluate(responseBody, "error_message");
-            assertThat(error, equalTo("Trial was already activated."));
+            assertThat(error, containsString("Trial was already activated."));
         }
     }
 
diff --git a/renovate.json b/renovate.json
index 293a2bb262375..c1637ae651c1c 100644
--- a/renovate.json
+++ b/renovate.json
@@ -4,6 +4,9 @@
     "github>elastic/renovate-config:only-chainguard",
     ":disableDependencyDashboard"
   ],
+  "schedule": [
+    "after 1pm on tuesday"
+  ],
   "labels": [">non-issue", ":Delivery/Packaging", "Team:Delivery"],
   "baseBranches": ["main", "8.x"],
   "packageRules": [
diff --git a/rest-api-spec/build.gradle b/rest-api-spec/build.gradle
index 6cc2028bffa39..b9064ab1d79ad 100644
--- a/rest-api-spec/build.gradle
+++ b/rest-api-spec/build.gradle
@@ -59,6 +59,10 @@ tasks.named("yamlRestCompatTestTransform").configure ({ task ->
   task.replaceValueInMatch("profile.shards.0.dfs.knn.0.query.0.description", "DocAndScoreQuery[0,...][0.009673266,...],0.009673266", "dfs knn vector profiling with vector_operations_count")
   task.skipTest("indices.sort/10_basic/Index Sort", "warning does not exist for compatibility")
   task.skipTest("search/330_fetch_fields/Test search rewrite", "warning does not exist for compatibility")
+  task.skipTest("indices.create/20_synthetic_source/object with dynamic override", "temporary until backported")
+  task.skipTest("indices.create/20_synthetic_source/object with unmapped fields", "temporary until backported")
+  task.skipTest("indices.create/20_synthetic_source/empty object with unmapped fields", "temporary until backported")
+  task.skipTest("indices.create/20_synthetic_source/nested object with unmapped fields", "temporary until backported")
   task.skipTest("indices.create/21_synthetic_source_stored/object param - nested object with stored array", "temporary until backported")
   task.skipTest("cat.aliases/10_basic/Deprecated local parameter", "CAT APIs not covered by compatibility policy")
 })
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml
index a871d2ac0ae15..258dfeb57e00c 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml
@@ -1,6 +1,6 @@
 object with unmapped fields:
   - requires:
-      cluster_features: ["mapper.track_ignored_source"]
+      cluster_features: ["mapper.track_ignored_source", "mapper.bwc_workaround_9_0"]
       reason: requires tracking ignored source
 
   - do:
@@ -41,13 +41,13 @@ object with unmapped fields:
   - match: { hits.hits.0._source.some_string: AaAa }
   - match: { hits.hits.0._source.some_int: 1000 }
   - match: { hits.hits.0._source.some_double: 123.456789 }
-  - match: { hits.hits.0._source.a.very.deeply.nested.field: AAAA }
+  - match: { hits.hits.0._source.a: { very.deeply.nested.field: AAAA } }
   - match: { hits.hits.0._source.some_bool: true }
   - match: { hits.hits.1._source.name: bbbb }
   - match: { hits.hits.1._source.some_string: BbBb }
   - match: { hits.hits.1._source.some_int: 2000 }
   - match: { hits.hits.1._source.some_double: 321.987654 }
-  - match: { hits.hits.1._source.a.very.deeply.nested.field: BBBB }
+  - match: { hits.hits.1._source.a: { very.deeply.nested.field: BBBB } }
 
 
 ---
@@ -100,7 +100,7 @@ unmapped arrays:
 ---
 nested object with unmapped fields:
   - requires:
-      cluster_features: ["mapper.track_ignored_source"]
+      cluster_features: ["mapper.track_ignored_source", "mapper.bwc_workaround_9_0"]
       reason: requires tracking ignored source
 
   - do:
@@ -143,16 +143,16 @@ nested object with unmapped fields:
   - match: { hits.total.value: 2 }
   - match: { hits.hits.0._source.path.to.name: aaaa }
   - match: { hits.hits.0._source.path.to.surname: AaAa }
-  - match: { hits.hits.0._source.path.some.other.name: AaAaAa }
+  - match: { hits.hits.0._source.path.some.other\.name: AaAaAa }
   - match: { hits.hits.1._source.path.to.name: bbbb }
   - match: { hits.hits.1._source.path.to.surname: BbBb }
-  - match: { hits.hits.1._source.path.some.other.name: BbBbBb }
+  - match: { hits.hits.1._source.path.some.other\.name: BbBbBb }
 
 
 ---
 empty object with unmapped fields:
   - requires:
-      cluster_features: ["mapper.track_ignored_source"]
+      cluster_features: ["mapper.track_ignored_source", "mapper.bwc_workaround_9_0"]
       reason: requires tracking ignored source
 
   - do:
@@ -191,7 +191,7 @@ empty object with unmapped fields:
 
   - match: { hits.total.value: 1 }
   - match: { hits.hits.0._source.path.to.surname: AaAa }
-  - match: { hits.hits.0._source.path.some.other.name: AaAaAa }
+  - match: { hits.hits.0._source.path.some.other\.name: AaAaAa }
 
 
 ---
@@ -434,7 +434,7 @@ mixed disabled and enabled objects:
 ---
 object with dynamic override:
   - requires:
-      cluster_features: ["mapper.ignored_source.dont_expand_dots"]
+      cluster_features: ["mapper.ignored_source.dont_expand_dots", "mapper.bwc_workaround_9_0"]
       reason: requires tracking ignored source
 
   - do:
@@ -475,7 +475,7 @@ object with dynamic override:
   - match: { hits.hits.0._source.path_no.to: { a.very.deeply.nested.field: A } }
   - match: { hits.hits.0._source.path_runtime.name: bar }
   - match: { hits.hits.0._source.path_runtime.some_int: 20 }
-  - match: { hits.hits.0._source.path_runtime.to.a.very.deeply.nested.field: B }
+  - match: { hits.hits.0._source.path_runtime.to: { a.very.deeply.nested.field: B } }
 
 
 ---
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml
index 6a4e92f694220..f3545bb0a3f0e 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml
@@ -1249,3 +1249,52 @@ index param - nested object with stored array:
   - match: { hits.hits.1._source.nested.0.b.1.c: 300 }
   - match: { hits.hits.1._source.nested.1.b.0.c: 40 }
   - match: { hits.hits.1._source.nested.1.b.1.c: 400 }
+
+
+---
+index param - flattened fields:
+  - requires:
+      cluster_features: ["mapper.synthetic_source_keep", "mapper.bwc_workaround_9_0"]
+      reason: requires keeping array source
+
+  - do:
+      indices.create:
+        index: test
+        body:
+          settings:
+            index:
+              mapping:
+                synthetic_source_keep: arrays
+          mappings:
+            _source:
+              mode: synthetic
+            properties:
+              name:
+                type: keyword
+              outer:
+                properties:
+                  inner:
+                    type: object
+
+  - do:
+      bulk:
+        index: test
+        refresh: true
+        body:
+          - '{ "create": { } }'
+          - '{ "name": "A", "outer": { "inner": [ { "a.b": "AA", "a.c": "AAA" } ] } }'
+          - '{ "create": { } }'
+          - '{ "name": "B", "outer": { "inner": [ { "a.x.y.z": "BB", "a.z.y.x": "BBB" } ] } }'
+
+
+  - match: { errors: false }
+
+  - do:
+      search:
+        index: test
+        sort: name
+  - match: { hits.total.value: 2 }
+  - match: { hits.hits.0._source.name: A }
+  - match: { hits.hits.0._source.outer.inner: [{ a.b: AA, a.c: AAA }] }
+  - match: { hits.hits.1._source.name: B }
+  - match: { hits.hits.1._source.outer.inner: [{ a.x.y.z: BB, a.z.y.x: BBB }] }
diff --git a/server/build.gradle b/server/build.gradle
index 963b3cfb2e747..e8493751cb327 100644
--- a/server/build.gradle
+++ b/server/build.gradle
@@ -28,17 +28,17 @@ base {
 
 dependencies {
 
-  api project(':libs:elasticsearch-core')
-  api project(':libs:elasticsearch-logging')
-  api project(':libs:elasticsearch-secure-sm')
-  api project(':libs:elasticsearch-x-content')
-  api project(":libs:elasticsearch-geo")
-  api project(":libs:elasticsearch-lz4")
-  api project(":libs:elasticsearch-plugin-api")
-  api project(":libs:elasticsearch-plugin-analysis-api")
-  api project(':libs:elasticsearch-grok')
-  api project(":libs:elasticsearch-tdigest")
-  implementation project(":libs:elasticsearch-simdvec")
+  api project(':libs:core')
+  api project(':libs:logging')
+  api project(':libs:secure-sm')
+  api project(':libs:x-content')
+  api project(":libs:geo")
+  api project(":libs:lz4")
+  api project(":libs:plugin-api")
+  api project(":libs:plugin-analysis-api")
+  api project(':libs:grok')
+  api project(":libs:tdigest")
+  implementation project(":libs:simdvec")
 
   // lucene
   api "org.apache.lucene:lucene-core:${versions.lucene}"
@@ -56,7 +56,7 @@ dependencies {
   api "org.apache.lucene:lucene-suggest:${versions.lucene}"
 
   // utilities
-  api project(":libs:elasticsearch-cli")
+  api project(":libs:cli")
   implementation 'com.carrotsearch:hppc:0.8.1'
 
   // precentil ranks aggregation
@@ -67,7 +67,7 @@ dependencies {
   api "org.apache.logging.log4j:log4j-core:${versions.log4j}"
 
   // access to native functions
-  implementation project(':libs:elasticsearch-native')
+  implementation project(':libs:native')
 
   api "co.elastic.logging:log4j2-ecs-layout:${versions.ecsLogging}"
   api "co.elastic.logging:ecs-logging-core:${versions.ecsLogging}"
diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java
index 7bf3204b7e1a6..ea3e649de9ef8 100644
--- a/server/src/main/java/org/elasticsearch/TransportVersions.java
+++ b/server/src/main/java/org/elasticsearch/TransportVersions.java
@@ -185,6 +185,7 @@ static TransportVersion def(int id) {
     public static final TransportVersion INDEX_REQUEST_REMOVE_METERING = def(8_780_00_0);
     public static final TransportVersion CPU_STAT_STRING_PARSING = def(8_781_00_0);
     public static final TransportVersion QUERY_RULES_RETRIEVER = def(8_782_00_0);
+    public static final TransportVersion ESQL_CCS_EXEC_INFO_WITH_FAILURES = def(8_783_00_0);
 
     /*
      * STOP! READ THIS FIRST! No, really,
diff --git a/server/src/main/java/org/elasticsearch/action/bulk/IncrementalBulkService.java b/server/src/main/java/org/elasticsearch/action/bulk/IncrementalBulkService.java
index 2e7c87301b2f6..6ce198260ba3c 100644
--- a/server/src/main/java/org/elasticsearch/action/bulk/IncrementalBulkService.java
+++ b/server/src/main/java/org/elasticsearch/action/bulk/IncrementalBulkService.java
@@ -17,7 +17,6 @@
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
-import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.Releasable;
 import org.elasticsearch.core.Releasables;
@@ -43,12 +42,10 @@ public class IncrementalBulkService {
     private final Client client;
     private final AtomicBoolean enabledForTests = new AtomicBoolean(true);
     private final IndexingPressure indexingPressure;
-    private final ThreadContext threadContext;
 
-    public IncrementalBulkService(Client client, IndexingPressure indexingPressure, ThreadContext threadContext) {
+    public IncrementalBulkService(Client client, IndexingPressure indexingPressure) {
         this.client = client;
         this.indexingPressure = indexingPressure;
-        this.threadContext = threadContext;
     }
 
     public Handler newBulkRequest() {
@@ -58,7 +55,7 @@ public Handler newBulkRequest() {
 
     public Handler newBulkRequest(@Nullable String waitForActiveShards, @Nullable TimeValue timeout, @Nullable String refresh) {
         ensureEnabled();
-        return new Handler(client, threadContext, indexingPressure, waitForActiveShards, timeout, refresh);
+        return new Handler(client, indexingPressure, waitForActiveShards, timeout, refresh);
     }
 
     private void ensureEnabled() {
@@ -94,7 +91,6 @@ public static class Handler implements Releasable {
         public static final BulkRequest.IncrementalState EMPTY_STATE = new BulkRequest.IncrementalState(Collections.emptyMap(), true);
 
         private final Client client;
-        private final ThreadContext threadContext;
         private final IndexingPressure indexingPressure;
         private final ActiveShardCount waitForActiveShards;
         private final TimeValue timeout;
@@ -106,22 +102,18 @@ public static class Handler implements Releasable {
         private boolean globalFailure = false;
         private boolean incrementalRequestSubmitted = false;
         private boolean bulkInProgress = false;
-        private ThreadContext.StoredContext requestContext;
         private Exception bulkActionLevelFailure = null;
         private long currentBulkSize = 0L;
         private BulkRequest bulkRequest = null;
 
         protected Handler(
             Client client,
-            ThreadContext threadContext,
             IndexingPressure indexingPressure,
             @Nullable String waitForActiveShards,
             @Nullable TimeValue timeout,
             @Nullable String refresh
         ) {
             this.client = client;
-            this.threadContext = threadContext;
-            this.requestContext = threadContext.newStoredContext();
             this.indexingPressure = indexingPressure;
             this.waitForActiveShards = waitForActiveShards != null ? ActiveShardCount.parseString(waitForActiveShards) : null;
             this.timeout = timeout;
@@ -141,31 +133,28 @@ public void addItems(List<DocWriteRequest<?>> items, Releasable releasable, Runn
                     if (shouldBackOff()) {
                         final boolean isFirstRequest = incrementalRequestSubmitted == false;
                         incrementalRequestSubmitted = true;
-                        try (var ignored = threadContext.restoreExistingContext(requestContext)) {
-                            final ArrayList<Releasable> toRelease = new ArrayList<>(releasables);
-                            releasables.clear();
-                            bulkInProgress = true;
-                            client.bulk(bulkRequest, ActionListener.runAfter(new ActionListener<>() {
-
-                                @Override
-                                public void onResponse(BulkResponse bulkResponse) {
-                                    handleBulkSuccess(bulkResponse);
-                                    createNewBulkRequest(
-                                        new BulkRequest.IncrementalState(bulkResponse.getIncrementalState().shardLevelFailures(), true)
-                                    );
-                                }
-
-                                @Override
-                                public void onFailure(Exception e) {
-                                    handleBulkFailure(isFirstRequest, e);
-                                }
-                            }, () -> {
-                                bulkInProgress = false;
-                                requestContext = threadContext.newStoredContext();
-                                toRelease.forEach(Releasable::close);
-                                nextItems.run();
-                            }));
-                        }
+                        final ArrayList<Releasable> toRelease = new ArrayList<>(releasables);
+                        releasables.clear();
+                        bulkInProgress = true;
+                        client.bulk(bulkRequest, ActionListener.runAfter(new ActionListener<>() {
+
+                            @Override
+                            public void onResponse(BulkResponse bulkResponse) {
+                                handleBulkSuccess(bulkResponse);
+                                createNewBulkRequest(
+                                    new BulkRequest.IncrementalState(bulkResponse.getIncrementalState().shardLevelFailures(), true)
+                                );
+                            }
+
+                            @Override
+                            public void onFailure(Exception e) {
+                                handleBulkFailure(isFirstRequest, e);
+                            }
+                        }, () -> {
+                            bulkInProgress = false;
+                            toRelease.forEach(Releasable::close);
+                            nextItems.run();
+                        }));
                     } else {
                         nextItems.run();
                     }
@@ -187,28 +176,26 @@ public void lastItems(List<DocWriteRequest<?>> items, Releasable releasable, Act
             } else {
                 assert bulkRequest != null;
                 if (internalAddItems(items, releasable)) {
-                    try (var ignored = threadContext.restoreExistingContext(requestContext)) {
-                        final ArrayList<Releasable> toRelease = new ArrayList<>(releasables);
-                        releasables.clear();
-                        // We do not need to set this back to false as this will be the last request.
-                        bulkInProgress = true;
-                        client.bulk(bulkRequest, ActionListener.runBefore(new ActionListener<>() {
-
-                            private final boolean isFirstRequest = incrementalRequestSubmitted == false;
-
-                            @Override
-                            public void onResponse(BulkResponse bulkResponse) {
-                                handleBulkSuccess(bulkResponse);
-                                listener.onResponse(combineResponses());
-                            }
+                    final ArrayList<Releasable> toRelease = new ArrayList<>(releasables);
+                    releasables.clear();
+                    // We do not need to set this back to false as this will be the last request.
+                    bulkInProgress = true;
+                    client.bulk(bulkRequest, ActionListener.runBefore(new ActionListener<>() {
+
+                        private final boolean isFirstRequest = incrementalRequestSubmitted == false;
+
+                        @Override
+                        public void onResponse(BulkResponse bulkResponse) {
+                            handleBulkSuccess(bulkResponse);
+                            listener.onResponse(combineResponses());
+                        }
 
-                            @Override
-                            public void onFailure(Exception e) {
-                                handleBulkFailure(isFirstRequest, e);
-                                errorResponse(listener);
-                            }
-                        }, () -> toRelease.forEach(Releasable::close)));
-                    }
+                        @Override
+                        public void onFailure(Exception e) {
+                            handleBulkFailure(isFirstRequest, e);
+                            errorResponse(listener);
+                        }
+                    }, () -> toRelease.forEach(Releasable::close)));
                 } else {
                     errorResponse(listener);
                 }
diff --git a/server/src/main/java/org/elasticsearch/action/search/DfsQueryPhase.java b/server/src/main/java/org/elasticsearch/action/search/DfsQueryPhase.java
index e0e240be0377a..93c8d66447e34 100644
--- a/server/src/main/java/org/elasticsearch/action/search/DfsQueryPhase.java
+++ b/server/src/main/java/org/elasticsearch/action/search/DfsQueryPhase.java
@@ -84,15 +84,20 @@ public void run() {
 
         for (final DfsSearchResult dfsResult : searchResults) {
             final SearchShardTarget shardTarget = dfsResult.getSearchShardTarget();
-            Transport.Connection connection = context.getConnection(shardTarget.getClusterAlias(), shardTarget.getNodeId());
-            ShardSearchRequest shardRequest = rewriteShardSearchRequest(dfsResult.getShardSearchRequest());
+            final int shardIndex = dfsResult.getShardIndex();
             QuerySearchRequest querySearchRequest = new QuerySearchRequest(
-                context.getOriginalIndices(dfsResult.getShardIndex()),
+                context.getOriginalIndices(shardIndex),
                 dfsResult.getContextId(),
-                shardRequest,
+                rewriteShardSearchRequest(dfsResult.getShardSearchRequest()),
                 dfs
             );
-            final int shardIndex = dfsResult.getShardIndex();
+            final Transport.Connection connection;
+            try {
+                connection = context.getConnection(shardTarget.getClusterAlias(), shardTarget.getNodeId());
+            } catch (Exception e) {
+                shardFailure(e, querySearchRequest, shardIndex, shardTarget, counter);
+                return;
+            }
             searchTransportService.sendExecuteQuery(
                 connection,
                 querySearchRequest,
@@ -112,10 +117,7 @@ protected void innerOnResponse(QuerySearchResult response) {
                     @Override
                     public void onFailure(Exception exception) {
                         try {
-                            context.getLogger()
-                                .debug(() -> "[" + querySearchRequest.contextId() + "] Failed to execute query phase", exception);
-                            progressListener.notifyQueryFailure(shardIndex, shardTarget, exception);
-                            counter.onFailure(shardIndex, shardTarget, exception);
+                            shardFailure(exception, querySearchRequest, shardIndex, shardTarget, counter);
                         } finally {
                             if (context.isPartOfPointInTime(querySearchRequest.contextId()) == false) {
                                 // the query might not have been executed at all (for example because thread pool rejected
@@ -134,6 +136,18 @@ public void onFailure(Exception exception) {
         }
     }
 
+    private void shardFailure(
+        Exception exception,
+        QuerySearchRequest querySearchRequest,
+        int shardIndex,
+        SearchShardTarget shardTarget,
+        CountedCollector<SearchPhaseResult> counter
+    ) {
+        context.getLogger().debug(() -> "[" + querySearchRequest.contextId() + "] Failed to execute query phase", exception);
+        progressListener.notifyQueryFailure(shardIndex, shardTarget, exception);
+        counter.onFailure(shardIndex, shardTarget, exception);
+    }
+
     // package private for testing
     ShardSearchRequest rewriteShardSearchRequest(ShardSearchRequest request) {
         SearchSourceBuilder source = request.source();
diff --git a/server/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java b/server/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java
index 99b24bd483fb4..29aba0eee1f55 100644
--- a/server/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java
+++ b/server/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java
@@ -21,6 +21,7 @@
 import org.elasticsearch.search.internal.ShardSearchContextId;
 import org.elasticsearch.search.rank.RankDoc;
 import org.elasticsearch.search.rank.RankDocShardInfo;
+import org.elasticsearch.transport.Transport;
 
 import java.util.ArrayList;
 import java.util.HashMap;
@@ -214,9 +215,41 @@ private void executeFetch(
         final ShardSearchContextId contextId = shardPhaseResult.queryResult() != null
             ? shardPhaseResult.queryResult().getContextId()
             : shardPhaseResult.rankFeatureResult().getContextId();
+        var listener = new SearchActionListener<FetchSearchResult>(shardTarget, shardIndex) {
+            @Override
+            public void innerOnResponse(FetchSearchResult result) {
+                try {
+                    progressListener.notifyFetchResult(shardIndex);
+                    counter.onResult(result);
+                } catch (Exception e) {
+                    context.onPhaseFailure(FetchSearchPhase.this, "", e);
+                }
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                try {
+                    logger.debug(() -> "[" + contextId + "] Failed to execute fetch phase", e);
+                    progressListener.notifyFetchFailure(shardIndex, shardTarget, e);
+                    counter.onFailure(shardIndex, shardTarget, e);
+                } finally {
+                    // the search context might not be cleared on the node where the fetch was executed for example
+                    // because the action was rejected by the thread pool. in this case we need to send a dedicated
+                    // request to clear the search context.
+                    releaseIrrelevantSearchContext(shardPhaseResult, context);
+                }
+            }
+        };
+        final Transport.Connection connection;
+        try {
+            connection = context.getConnection(shardTarget.getClusterAlias(), shardTarget.getNodeId());
+        } catch (Exception e) {
+            listener.onFailure(e);
+            return;
+        }
         context.getSearchTransport()
             .sendExecuteFetch(
-                context.getConnection(shardTarget.getClusterAlias(), shardTarget.getNodeId()),
+                connection,
                 new ShardFetchSearchRequest(
                     context.getOriginalIndices(shardPhaseResult.getShardIndex()),
                     contextId,
@@ -228,31 +261,7 @@ private void executeFetch(
                     aggregatedDfs
                 ),
                 context.getTask(),
-                new SearchActionListener<>(shardTarget, shardIndex) {
-                    @Override
-                    public void innerOnResponse(FetchSearchResult result) {
-                        try {
-                            progressListener.notifyFetchResult(shardIndex);
-                            counter.onResult(result);
-                        } catch (Exception e) {
-                            context.onPhaseFailure(FetchSearchPhase.this, "", e);
-                        }
-                    }
-
-                    @Override
-                    public void onFailure(Exception e) {
-                        try {
-                            logger.debug(() -> "[" + contextId + "] Failed to execute fetch phase", e);
-                            progressListener.notifyFetchFailure(shardIndex, shardTarget, e);
-                            counter.onFailure(shardIndex, shardTarget, e);
-                        } finally {
-                            // the search context might not be cleared on the node where the fetch was executed for example
-                            // because the action was rejected by the thread pool. in this case we need to send a dedicated
-                            // request to clear the search context.
-                            releaseIrrelevantSearchContext(shardPhaseResult, context);
-                        }
-                    }
-                }
+                listener
             );
     }
 
diff --git a/server/src/main/java/org/elasticsearch/action/search/RankFeaturePhase.java b/server/src/main/java/org/elasticsearch/action/search/RankFeaturePhase.java
index dd3c28bba0fce..e37d6d1729f9f 100644
--- a/server/src/main/java/org/elasticsearch/action/search/RankFeaturePhase.java
+++ b/server/src/main/java/org/elasticsearch/action/search/RankFeaturePhase.java
@@ -24,6 +24,7 @@
 import org.elasticsearch.search.rank.feature.RankFeatureDoc;
 import org.elasticsearch.search.rank.feature.RankFeatureResult;
 import org.elasticsearch.search.rank.feature.RankFeatureShardRequest;
+import org.elasticsearch.transport.Transport;
 
 import java.util.List;
 
@@ -131,9 +132,38 @@ private void executeRankFeatureShardPhase(
         final SearchShardTarget shardTarget = queryResult.queryResult().getSearchShardTarget();
         final ShardSearchContextId contextId = queryResult.queryResult().getContextId();
         final int shardIndex = queryResult.getShardIndex();
+        var listener = new SearchActionListener<RankFeatureResult>(shardTarget, shardIndex) {
+            @Override
+            protected void innerOnResponse(RankFeatureResult response) {
+                try {
+                    progressListener.notifyRankFeatureResult(shardIndex);
+                    rankRequestCounter.onResult(response);
+                } catch (Exception e) {
+                    context.onPhaseFailure(RankFeaturePhase.this, "", e);
+                }
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                try {
+                    logger.debug(() -> "[" + contextId + "] Failed to execute rank phase", e);
+                    progressListener.notifyRankFeatureFailure(shardIndex, shardTarget, e);
+                    rankRequestCounter.onFailure(shardIndex, shardTarget, e);
+                } finally {
+                    releaseIrrelevantSearchContext(queryResult, context);
+                }
+            }
+        };
+        final Transport.Connection connection;
+        try {
+            connection = context.getConnection(shardTarget.getClusterAlias(), shardTarget.getNodeId());
+        } catch (Exception e) {
+            listener.onFailure(e);
+            return;
+        }
         context.getSearchTransport()
             .sendExecuteRankFeature(
-                context.getConnection(shardTarget.getClusterAlias(), shardTarget.getNodeId()),
+                connection,
                 new RankFeatureShardRequest(
                     context.getOriginalIndices(queryResult.getShardIndex()),
                     queryResult.getContextId(),
@@ -141,28 +171,7 @@ private void executeRankFeatureShardPhase(
                     entry
                 ),
                 context.getTask(),
-                new SearchActionListener<>(shardTarget, shardIndex) {
-                    @Override
-                    protected void innerOnResponse(RankFeatureResult response) {
-                        try {
-                            progressListener.notifyRankFeatureResult(shardIndex);
-                            rankRequestCounter.onResult(response);
-                        } catch (Exception e) {
-                            context.onPhaseFailure(RankFeaturePhase.this, "", e);
-                        }
-                    }
-
-                    @Override
-                    public void onFailure(Exception e) {
-                        try {
-                            logger.debug(() -> "[" + contextId + "] Failed to execute rank phase", e);
-                            progressListener.notifyRankFeatureFailure(shardIndex, shardTarget, e);
-                            rankRequestCounter.onFailure(shardIndex, shardTarget, e);
-                        } finally {
-                            releaseIrrelevantSearchContext(queryResult, context);
-                        }
-                    }
-                }
+                listener
             );
     }
 
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchDfsQueryThenFetchAsyncAction.java b/server/src/main/java/org/elasticsearch/action/search/SearchDfsQueryThenFetchAsyncAction.java
index 5b7ee04d020fc..26eb266cd457e 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchDfsQueryThenFetchAsyncAction.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchDfsQueryThenFetchAsyncAction.java
@@ -87,12 +87,14 @@ protected void executePhaseOnShard(
         final SearchShardTarget shard,
         final SearchActionListener<DfsSearchResult> listener
     ) {
-        getSearchTransport().sendExecuteDfs(
-            getConnection(shard.getClusterAlias(), shard.getNodeId()),
-            buildShardSearchRequest(shardIt, listener.requestIndex),
-            getTask(),
-            listener
-        );
+        final Transport.Connection connection;
+        try {
+            connection = getConnection(shard.getClusterAlias(), shard.getNodeId());
+        } catch (Exception e) {
+            listener.onFailure(e);
+            return;
+        }
+        getSearchTransport().sendExecuteDfs(connection, buildShardSearchRequest(shardIt, listener.requestIndex), getTask(), listener);
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncAction.java b/server/src/main/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncAction.java
index e0ad4691fa991..33b2cdf74cd79 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncAction.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncAction.java
@@ -94,8 +94,15 @@ protected void executePhaseOnShard(
         final SearchShardTarget shard,
         final SearchActionListener<SearchPhaseResult> listener
     ) {
+        final Transport.Connection connection;
+        try {
+            connection = getConnection(shard.getClusterAlias(), shard.getNodeId());
+        } catch (Exception e) {
+            listener.onFailure(e);
+            return;
+        }
         ShardSearchRequest request = rewriteShardSearchRequest(super.buildShardSearchRequest(shardIt, listener.requestIndex));
-        getSearchTransport().sendExecuteQuery(getConnection(shard.getClusterAlias(), shard.getNodeId()), request, getTask(), listener);
+        getSearchTransport().sendExecuteQuery(connection, request, getTask(), listener);
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ReservedStateMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ReservedStateMetadata.java
index 2390c96664057..a0b35f7cfc3eb 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/ReservedStateMetadata.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ReservedStateMetadata.java
@@ -91,6 +91,21 @@ public Set<String> conflicts(String handlerName, Set<String> modified) {
         return Collections.unmodifiableSet(intersect);
     }
 
+    /**
+     * Get the reserved keys for the handler name
+     *
+     * @param handlerName handler name to get keys for
+     * @return set of keys for that handler
+     */
+    public Set<String> keys(String handlerName) {
+        ReservedStateHandlerMetadata handlerMetadata = handlers.get(handlerName);
+        if (handlerMetadata == null || handlerMetadata.keys().isEmpty()) {
+            return Collections.emptySet();
+        }
+
+        return Collections.unmodifiableSet(handlerMetadata.keys());
+    }
+
     /**
      * Reads an {@link ReservedStateMetadata} from a {@link StreamInput}
      *
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java
index 840aa3a3c1d3f..108bb83d90871 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java
@@ -32,8 +32,6 @@
 import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
 import org.elasticsearch.cluster.routing.allocation.decider.Decision;
 import org.elasticsearch.cluster.routing.allocation.decider.Decision.Type;
-import org.elasticsearch.common.logging.DeprecationCategory;
-import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
@@ -41,7 +39,6 @@
 import org.elasticsearch.common.util.Maps;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.core.Tuple;
-import org.elasticsearch.core.UpdateForV9;
 import org.elasticsearch.gateway.PriorityComparator;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.injection.guice.Inject;
@@ -109,7 +106,7 @@ public class BalancedShardsAllocator implements ShardsAllocator {
     public static final Setting<Float> THRESHOLD_SETTING = Setting.floatSetting(
         "cluster.routing.allocation.balance.threshold",
         1.0f,
-        0.0f,
+        1.0f,
         Property.Dynamic,
         Property.NodeScope
     );
@@ -140,34 +137,10 @@ public BalancedShardsAllocator(ClusterSettings clusterSettings, WriteLoadForecas
         clusterSettings.initializeAndWatch(INDEX_BALANCE_FACTOR_SETTING, value -> this.indexBalanceFactor = value);
         clusterSettings.initializeAndWatch(WRITE_LOAD_BALANCE_FACTOR_SETTING, value -> this.writeLoadBalanceFactor = value);
         clusterSettings.initializeAndWatch(DISK_USAGE_BALANCE_FACTOR_SETTING, value -> this.diskUsageBalanceFactor = value);
-        clusterSettings.initializeAndWatch(THRESHOLD_SETTING, value -> this.threshold = ensureValidThreshold(value));
+        clusterSettings.initializeAndWatch(THRESHOLD_SETTING, value -> this.threshold = value);
         this.writeLoadForecaster = writeLoadForecaster;
     }
 
-    /**
-     * Clamp threshold to be at least 1, and log a critical deprecation warning if smaller values are given.
-     *
-     * Once {@link org.elasticsearch.Version#V_7_17_0} goes out of scope, start to properly reject such bad values.
-     */
-    @UpdateForV9(owner = UpdateForV9.Owner.DISTRIBUTED_COORDINATION)
-    private static float ensureValidThreshold(float threshold) {
-        if (1.0f <= threshold) {
-            return threshold;
-        } else {
-            DeprecationLogger.getLogger(BalancedShardsAllocator.class)
-                .critical(
-                    DeprecationCategory.SETTINGS,
-                    "balance_threshold_too_small",
-                    "ignoring value [{}] for [{}] since it is smaller than 1.0; "
-                        + "setting [{}] to a value smaller than 1.0 will be forbidden in a future release",
-                    threshold,
-                    THRESHOLD_SETTING.getKey(),
-                    THRESHOLD_SETTING.getKey()
-                );
-            return 1.0f;
-        }
-    }
-
     @Override
     public void allocate(RoutingAllocation allocation) {
         assert allocation.ignoreDisable() == false;
diff --git a/server/src/main/java/org/elasticsearch/index/IndexVersions.java b/server/src/main/java/org/elasticsearch/index/IndexVersions.java
index efb1facc79b3a..2919f98ee200e 100644
--- a/server/src/main/java/org/elasticsearch/index/IndexVersions.java
+++ b/server/src/main/java/org/elasticsearch/index/IndexVersions.java
@@ -128,7 +128,7 @@ private static Version parseUnchecked(String version) {
     public static final IndexVersion MERGE_ON_RECOVERY_VERSION = def(8_515_00_0, Version.LUCENE_9_11_1);
     public static final IndexVersion UPGRADE_TO_LUCENE_9_12 = def(8_516_00_0, Version.LUCENE_9_12_0);
     public static final IndexVersion ENABLE_IGNORE_ABOVE_LOGSDB = def(8_517_00_0, Version.LUCENE_9_12_0);
-
+    public static final IndexVersion ADD_ROLE_MAPPING_CLEANUP_MIGRATION = def(8_518_00_0, Version.LUCENE_9_12_0);
     public static final IndexVersion UPGRADE_TO_LUCENE_10_0_0 = def(9_000_00_0, Version.LUCENE_10_0_0);
 
     /*
diff --git a/server/src/main/java/org/elasticsearch/index/get/GetResult.java b/server/src/main/java/org/elasticsearch/index/get/GetResult.java
index 109f645f24caf..3c504d400c7c6 100644
--- a/server/src/main/java/org/elasticsearch/index/get/GetResult.java
+++ b/server/src/main/java/org/elasticsearch/index/get/GetResult.java
@@ -21,6 +21,7 @@
 import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.index.mapper.IgnoredFieldMapper;
+import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.SourceFieldMapper;
 import org.elasticsearch.search.lookup.Source;
@@ -247,7 +248,7 @@ public XContentBuilder toXContentEmbedded(XContentBuilder builder, Params params
 
         for (DocumentField field : metaFields.values()) {
             // TODO: can we avoid having an exception here?
-            if (field.getName().equals(IgnoredFieldMapper.NAME)) {
+            if (field.getName().equals(IgnoredFieldMapper.NAME) || field.getName().equals(IgnoredSourceFieldMapper.NAME)) {
                 builder.field(field.getName(), field.getValues());
             } else {
                 builder.field(field.getName(), field.<Object>getValue());
@@ -341,7 +342,7 @@ public static GetResult fromXContentEmbedded(XContentParser parser, String index
                     parser.skipChildren(); // skip potential inner objects for forward compatibility
                 }
             } else if (token == XContentParser.Token.START_ARRAY) {
-                if (IgnoredFieldMapper.NAME.equals(currentFieldName)) {
+                if (IgnoredFieldMapper.NAME.equals(currentFieldName) || IgnoredSourceFieldMapper.NAME.equals(currentFieldName)) {
                     metaFields.put(currentFieldName, new DocumentField(currentFieldName, parser.list()));
                 } else {
                     parser.skipChildren(); // skip potential inner arrays for forward compatibility
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java
index 3b1f1a6d2809a..c884d68c8f0ee 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java
@@ -528,11 +528,7 @@ public final boolean addDynamicMapper(Mapper mapper) {
                     if (canAddIgnoredField()) {
                         try {
                             addIgnoredField(
-                                IgnoredSourceFieldMapper.NameValue.fromContext(
-                                    this,
-                                    mapper.fullPath(),
-                                    XContentDataHelper.encodeToken(parser())
-                                )
+                                IgnoredSourceFieldMapper.NameValue.fromContext(this, mapper.fullPath(), encodeFlattenedToken())
                             );
                         } catch (IOException e) {
                             throw new IllegalArgumentException("failed to parse field [" + mapper.fullPath() + " ]", e);
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DotExpandingXContentParser.java b/server/src/main/java/org/elasticsearch/index/mapper/DotExpandingXContentParser.java
index fc003e709cbca..42784e0974417 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/DotExpandingXContentParser.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/DotExpandingXContentParser.java
@@ -34,6 +34,10 @@
  */
 class DotExpandingXContentParser extends FilterXContentParserWrapper {
 
+    static boolean isInstance(XContentParser parser) {
+        return parser instanceof WrappingParser;
+    }
+
     private static final class WrappingParser extends FilterXContentParser {
 
         private final ContentPath contentPath;
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/XContentDataHelper.java b/server/src/main/java/org/elasticsearch/index/mapper/XContentDataHelper.java
index 8bacaf8505f91..dee5ff92040a9 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/XContentDataHelper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/XContentDataHelper.java
@@ -221,8 +221,11 @@ static Tuple<DocumentParserContext, XContentParser> cloneSubContextWithParser(Do
     private static Tuple<XContentParserConfiguration, XContentBuilder> cloneSubContextParserConfiguration(DocumentParserContext context)
         throws IOException {
         XContentParser parser = context.parser();
+        var oldValue = context.path().isWithinLeafObject();
+        context.path().setWithinLeafObject(true);
         XContentBuilder builder = XContentBuilder.builder(parser.contentType().xContent());
         builder.copyCurrentStructure(parser);
+        context.path().setWithinLeafObject(oldValue);
 
         XContentParserConfiguration configuration = XContentParserConfiguration.EMPTY.withRegistry(parser.getXContentRegistry())
             .withDeprecationHandler(parser.getDeprecationHandler())
@@ -235,9 +238,17 @@ private static DocumentParserContext cloneDocumentParserContext(
         XContentParserConfiguration configuration,
         XContentBuilder builder
     ) throws IOException {
-        DocumentParserContext subcontext = context.switchParser(
-            XContentHelper.createParserNotCompressed(configuration, BytesReference.bytes(builder), context.parser().contentType())
+        XContentParser newParser = XContentHelper.createParserNotCompressed(
+            configuration,
+            BytesReference.bytes(builder),
+            context.parser().contentType()
         );
+        if (DotExpandingXContentParser.isInstance(context.parser())) {
+            // If we performed dot expanding originally we need to continue to do so when we replace the parser.
+            newParser = DotExpandingXContentParser.expandDots(newParser, context.path());
+        }
+
+        DocumentParserContext subcontext = context.switchParser(newParser);
         subcontext.setRecordedSource();  // Avoids double-storing parts of the source for the same parser subtree.
         subcontext.parser().nextToken();
         return subcontext;
diff --git a/server/src/main/java/org/elasticsearch/node/NodeConstruction.java b/server/src/main/java/org/elasticsearch/node/NodeConstruction.java
index 0a88a202ac8d3..5354b1097326b 100644
--- a/server/src/main/java/org/elasticsearch/node/NodeConstruction.java
+++ b/server/src/main/java/org/elasticsearch/node/NodeConstruction.java
@@ -915,11 +915,7 @@ private void construct(
         terminationHandler = getSinglePlugin(terminationHandlers, TerminationHandler.class).orElse(null);
 
         final IndexingPressure indexingLimits = new IndexingPressure(settings);
-        final IncrementalBulkService incrementalBulkService = new IncrementalBulkService(
-            client,
-            indexingLimits,
-            threadPool.getThreadContext()
-        );
+        final IncrementalBulkService incrementalBulkService = new IncrementalBulkService(client, indexingLimits);
 
         ActionModule actionModule = new ActionModule(
             settings,
diff --git a/server/src/main/java/org/elasticsearch/rest/BaseRestHandler.java b/server/src/main/java/org/elasticsearch/rest/BaseRestHandler.java
index c8cf0bf93879b..f1b59ed14cefb 100644
--- a/server/src/main/java/org/elasticsearch/rest/BaseRestHandler.java
+++ b/server/src/main/java/org/elasticsearch/rest/BaseRestHandler.java
@@ -125,6 +125,7 @@ public final void handleRequest(RestRequest request, RestChannel channel, NodeCl
             if (request.isStreamedContent()) {
                 assert action instanceof RequestBodyChunkConsumer;
                 var chunkConsumer = (RequestBodyChunkConsumer) action;
+
                 request.contentStream().setHandler(new HttpBody.ChunkHandler() {
                     @Override
                     public void onNext(ReleasableBytesReference chunk, boolean isLast) {
diff --git a/server/src/main/java/org/elasticsearch/rest/action/document/RestBulkAction.java b/server/src/main/java/org/elasticsearch/rest/action/document/RestBulkAction.java
index 1e80e6de60d65..7b82481d3d283 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/document/RestBulkAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/document/RestBulkAction.java
@@ -173,8 +173,7 @@ static class ChunkHandler implements BaseRestHandler.RequestBodyChunkConsumer {
             this.defaultListExecutedPipelines = request.paramAsBoolean("list_executed_pipelines", false);
             this.defaultRequireAlias = request.paramAsBoolean(DocWriteRequest.REQUIRE_ALIAS, false);
             this.defaultRequireDataStream = request.paramAsBoolean(DocWriteRequest.REQUIRE_DATA_STREAM, false);
-            // TODO: Fix type deprecation logging
-            this.parser = new BulkRequestParser(false, request.getRestApiVersion());
+            this.parser = new BulkRequestParser(true, request.getRestApiVersion());
             this.handlerSupplier = handlerSupplier;
         }
 
diff --git a/server/src/test/java/org/elasticsearch/action/ActionModuleTests.java b/server/src/test/java/org/elasticsearch/action/ActionModuleTests.java
index 871062a687429..8d3561f2179cd 100644
--- a/server/src/test/java/org/elasticsearch/action/ActionModuleTests.java
+++ b/server/src/test/java/org/elasticsearch/action/ActionModuleTests.java
@@ -131,7 +131,7 @@ public void testSetupRestHandlerContainsKnownBuiltin() {
             null,
             List.of(),
             RestExtension.allowAll(),
-            new IncrementalBulkService(null, null, new ThreadContext(Settings.EMPTY))
+            new IncrementalBulkService(null, null)
         );
         actionModule.initRestHandlers(null, null);
         // At this point the easiest way to confirm that a handler is loaded is to try to register another one on top of it and to fail
@@ -196,7 +196,7 @@ public String getName() {
                 null,
                 List.of(),
                 RestExtension.allowAll(),
-                new IncrementalBulkService(null, null, new ThreadContext(Settings.EMPTY))
+                new IncrementalBulkService(null, null)
             );
             Exception e = expectThrows(IllegalArgumentException.class, () -> actionModule.initRestHandlers(null, null));
             assertThat(e.getMessage(), startsWith("Cannot replace existing handler for [/_nodes] for method: GET"));
@@ -254,7 +254,7 @@ public List<RestHandler> getRestHandlers(
                 null,
                 List.of(),
                 RestExtension.allowAll(),
-                new IncrementalBulkService(null, null, new ThreadContext(Settings.EMPTY))
+                new IncrementalBulkService(null, null)
             );
             actionModule.initRestHandlers(null, null);
             // At this point the easiest way to confirm that a handler is loaded is to try to register another one on top of it and to fail
@@ -305,7 +305,7 @@ public void test3rdPartyHandlerIsNotInstalled() {
                     null,
                     List.of(),
                     RestExtension.allowAll(),
-                    new IncrementalBulkService(null, null, new ThreadContext(Settings.EMPTY))
+                    new IncrementalBulkService(null, null)
                 )
             );
             assertThat(
@@ -347,7 +347,7 @@ public void test3rdPartyRestControllerIsNotInstalled() {
                     null,
                     List.of(),
                     RestExtension.allowAll(),
-                    new IncrementalBulkService(null, null, new ThreadContext(Settings.EMPTY))
+                    new IncrementalBulkService(null, null)
                 )
             );
             assertThat(
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/BalancedSingleShardTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/BalancedSingleShardTests.java
index 41207a2d968b8..9a769567bee1c 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/BalancedSingleShardTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/BalancedSingleShardTests.java
@@ -246,7 +246,7 @@ public void testNodeDecisionsRanking() {
         // return the same ranking as the current node
         ClusterState clusterState = ClusterStateCreationUtils.state(randomIntBetween(1, 10), new String[] { "idx" }, 1);
         ShardRouting shardToRebalance = clusterState.routingTable().index("idx").shardsWithState(ShardRoutingState.STARTED).get(0);
-        MoveDecision decision = executeRebalanceFor(shardToRebalance, clusterState, emptySet(), -1);
+        MoveDecision decision = executeRebalanceFor(shardToRebalance, clusterState, emptySet());
         int currentRanking = decision.getCurrentNodeRanking();
         assertEquals(1, currentRanking);
         for (NodeAllocationResult result : decision.getNodeDecisions()) {
@@ -258,7 +258,7 @@ public void testNodeDecisionsRanking() {
         clusterState = ClusterStateCreationUtils.state(1, new String[] { "idx" }, randomIntBetween(2, 10));
         shardToRebalance = clusterState.routingTable().index("idx").shardsWithState(ShardRoutingState.STARTED).get(0);
         clusterState = addNodesToClusterState(clusterState, randomIntBetween(1, 10));
-        decision = executeRebalanceFor(shardToRebalance, clusterState, emptySet(), 0.01f);
+        decision = executeRebalanceFor(shardToRebalance, clusterState, emptySet());
         for (NodeAllocationResult result : decision.getNodeDecisions()) {
             assertThat(result.getWeightRanking(), lessThan(decision.getCurrentNodeRanking()));
         }
@@ -285,7 +285,7 @@ public void testNodeDecisionsRanking() {
             }
         }
         clusterState = addNodesToClusterState(clusterState, 1);
-        decision = executeRebalanceFor(shardToRebalance, clusterState, emptySet(), 0.01f);
+        decision = executeRebalanceFor(shardToRebalance, clusterState, emptySet());
         for (NodeAllocationResult result : decision.getNodeDecisions()) {
             if (result.getWeightRanking() < decision.getCurrentNodeRanking()) {
                 // highest ranked node should not be any of the initial nodes
@@ -298,22 +298,13 @@ public void testNodeDecisionsRanking() {
                 assertTrue(nodesWithTwoShards.contains(result.getNode().getId()));
             }
         }
-
-        assertCriticalWarnings("""
-            ignoring value [0.01] for [cluster.routing.allocation.balance.threshold] since it is smaller than 1.0; setting \
-            [cluster.routing.allocation.balance.threshold] to a value smaller than 1.0 will be forbidden in a future release""");
     }
 
     private MoveDecision executeRebalanceFor(
         final ShardRouting shardRouting,
         final ClusterState clusterState,
-        final Set<String> noDecisionNodes,
-        final float threshold
+        final Set<String> noDecisionNodes
     ) {
-        Settings settings = Settings.EMPTY;
-        if (Float.compare(-1.0f, threshold) != 0) {
-            settings = Settings.builder().put(BalancedShardsAllocator.THRESHOLD_SETTING.getKey(), threshold).build();
-        }
         AllocationDecider allocationDecider = new AllocationDecider() {
             @Override
             public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) {
@@ -329,7 +320,7 @@ public Decision canRebalance(ShardRouting shardRouting, RoutingAllocation alloca
                 return Decision.YES;
             }
         };
-        BalancedShardsAllocator allocator = new BalancedShardsAllocator(settings);
+        BalancedShardsAllocator allocator = new BalancedShardsAllocator(Settings.EMPTY);
         RoutingAllocation routingAllocation = newRoutingAllocation(
             new AllocationDeciders(Arrays.asList(allocationDecider, rebalanceDecider)),
             clusterState
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocatorTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocatorTests.java
index 8392b6fe3e148..98c3451329f52 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocatorTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocatorTests.java
@@ -441,15 +441,10 @@ public void testGetIndexDiskUsageInBytes() {
 
     public void testThresholdLimit() {
         final var badValue = (float) randomDoubleBetween(0.0, Math.nextDown(1.0f), true);
-        assertEquals(
-            1.0f,
-            new BalancedShardsAllocator(Settings.builder().put(BalancedShardsAllocator.THRESHOLD_SETTING.getKey(), badValue).build())
-                .getThreshold(),
-            0.0f
+        expectThrows(
+            IllegalArgumentException.class,
+            () -> new BalancedShardsAllocator(Settings.builder().put(BalancedShardsAllocator.THRESHOLD_SETTING.getKey(), badValue).build())
         );
-        assertCriticalWarnings("ignoring value [" + badValue + """
-            ] for [cluster.routing.allocation.balance.threshold] since it is smaller than 1.0; setting \
-            [cluster.routing.allocation.balance.threshold] to a value smaller than 1.0 will be forbidden in a future release""");
 
         final var goodValue = (float) randomDoubleBetween(1.0, 10.0, true);
         assertEquals(
diff --git a/server/src/test/java/org/elasticsearch/http/AbstractHttpServerTransportTests.java b/server/src/test/java/org/elasticsearch/http/AbstractHttpServerTransportTests.java
index 77133516f37d5..cf623e77f740a 100644
--- a/server/src/test/java/org/elasticsearch/http/AbstractHttpServerTransportTests.java
+++ b/server/src/test/java/org/elasticsearch/http/AbstractHttpServerTransportTests.java
@@ -1179,7 +1179,7 @@ public Collection<RestHeaderDefinition> getRestHeaders() {
             null,
             List.of(),
             RestExtension.allowAll(),
-            new IncrementalBulkService(null, null, new ThreadContext(Settings.EMPTY))
+            new IncrementalBulkService(null, null)
         );
     }
 
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java
index 7a4ce8bcb03fa..884372d249287 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java
@@ -2075,6 +2075,136 @@ public void testDisabledObjectWithFlatFields() throws IOException {
             {"top":[{"file.name":"A","file.line":10},{"file.name":"B","file.line":20}]}""", syntheticSourceWithArray);
     }
 
+    public void testRegularObjectWithFlatFields() throws IOException {
+        DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> {
+            b.startObject("top").field("type", "object").field("synthetic_source_keep", "all").endObject();
+        })).documentMapper();
+
+        CheckedConsumer<XContentBuilder, IOException> document = b -> {
+            b.startObject("top");
+            b.field("file.name", "A");
+            b.field("file.line", 10);
+            b.endObject();
+        };
+
+        var syntheticSource = syntheticSource(documentMapper, document);
+        assertEquals("{\"top\":{\"file.name\":\"A\",\"file.line\":10}}", syntheticSource);
+
+        CheckedConsumer<XContentBuilder, IOException> documentWithArray = b -> {
+            b.startArray("top");
+            b.startObject();
+            b.field("file.name", "A");
+            b.field("file.line", 10);
+            b.endObject();
+            b.startObject();
+            b.field("file.name", "B");
+            b.field("file.line", 20);
+            b.endObject();
+            b.endArray();
+        };
+
+        var syntheticSourceWithArray = syntheticSource(documentMapper, documentWithArray);
+        assertEquals("""
+            {"top":[{"file.name":"A","file.line":10},{"file.name":"B","file.line":20}]}""", syntheticSourceWithArray);
+    }
+
+    public void testRegularObjectWithFlatFieldsInsideAnArray() throws IOException {
+        DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> {
+            b.startObject("top");
+            b.startObject("properties");
+            {
+                b.startObject("inner").field("type", "object").field("synthetic_source_keep", "all").endObject();
+            }
+            b.endObject();
+            b.endObject();
+        })).documentMapper();
+
+        CheckedConsumer<XContentBuilder, IOException> document = b -> {
+            b.startArray("top");
+            b.startObject();
+            {
+                b.startObject("inner");
+                b.field("file.name", "A");
+                b.field("file.line", 10);
+                b.endObject();
+            }
+            b.endObject();
+            b.endArray();
+        };
+
+        var syntheticSource = syntheticSource(documentMapper, document);
+        assertEquals("{\"top\":{\"inner\":{\"file.name\":\"A\",\"file.line\":10}}}", syntheticSource);
+
+        CheckedConsumer<XContentBuilder, IOException> documentWithArray = b -> {
+            b.startArray("top");
+            b.startObject();
+            {
+                b.startObject("inner");
+                b.field("file.name", "A");
+                b.field("file.line", 10);
+                b.endObject();
+            }
+            b.endObject();
+            b.startObject();
+            {
+                b.startObject("inner");
+                b.field("file.name", "B");
+                b.field("file.line", 20);
+                b.endObject();
+            }
+            b.endObject();
+            b.endArray();
+        };
+
+        var syntheticSourceWithArray = syntheticSource(documentMapper, documentWithArray);
+        assertEquals("""
+            {"top":{"inner":[{"file.name":"A","file.line":10},{"file.name":"B","file.line":20}]}}""", syntheticSourceWithArray);
+    }
+
+    public void testIgnoredDynamicObjectWithFlatFields() throws IOException {
+        var syntheticSource = getSyntheticSourceWithFieldLimit(b -> {
+            b.startObject("top");
+            b.field("file.name", "A");
+            b.field("file.line", 10);
+            b.endObject();
+        });
+        assertEquals("{\"top\":{\"file.name\":\"A\",\"file.line\":10}}", syntheticSource);
+
+        var syntheticSourceWithArray = getSyntheticSourceWithFieldLimit(b -> {
+            b.startArray("top");
+            b.startObject();
+            b.field("file.name", "A");
+            b.field("file.line", 10);
+            b.endObject();
+            b.startObject();
+            b.field("file.name", "B");
+            b.field("file.line", 20);
+            b.endObject();
+            b.endArray();
+        });
+        assertEquals("""
+            {"top":[{"file.name":"A","file.line":10},{"file.name":"B","file.line":20}]}""", syntheticSourceWithArray);
+    }
+
+    public void testStoredArrayWithFlatFields() throws IOException {
+        DocumentMapper documentMapper = createMapperServiceWithStoredArraySource(syntheticSourceMapping(b -> {
+            b.startObject("outer").startObject("properties");
+            {
+                b.startObject("inner").field("type", "object").endObject();
+            }
+            b.endObject().endObject();
+        })).documentMapper();
+        var syntheticSource = syntheticSource(documentMapper, b -> {
+            b.startObject("outer").startArray("inner");
+            {
+                b.startObject().field("a.b", "a.b").field("a.c", "a.c").endObject();
+            }
+            b.endArray().endObject();
+        });
+        assertEquals("""
+            {"outer":{"inner":[{"a.b":"a.b","a.c":"a.c"}]}}""", syntheticSource);
+    }
+
     protected void validateRoundTripReader(String syntheticSource, DirectoryReader reader, DirectoryReader roundTripReader)
         throws IOException {
         // We exclude ignored source field since in some cases it contains an exact copy of a part of document source.
diff --git a/server/src/test/java/org/elasticsearch/reservedstate/service/FileSettingsServiceTests.java b/server/src/test/java/org/elasticsearch/reservedstate/service/FileSettingsServiceTests.java
index 8af36e2f9677e..f67d7ddcc7550 100644
--- a/server/src/test/java/org/elasticsearch/reservedstate/service/FileSettingsServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/reservedstate/service/FileSettingsServiceTests.java
@@ -24,6 +24,8 @@
 import org.elasticsearch.common.component.Lifecycle;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.IOUtils;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.env.BuildVersion;
 import org.elasticsearch.env.Environment;
@@ -39,9 +41,10 @@
 import org.mockito.stubbing.Answer;
 
 import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.file.AtomicMoveNotSupportedException;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.nio.file.StandardCopyOption;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -50,6 +53,8 @@
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.function.Consumer;
 
+import static java.nio.file.StandardCopyOption.ATOMIC_MOVE;
+import static java.nio.file.StandardCopyOption.REPLACE_EXISTING;
 import static org.elasticsearch.node.Node.NODE_NAME_SETTING;
 import static org.hamcrest.Matchers.anEmptyMap;
 import static org.hamcrest.Matchers.hasEntry;
@@ -190,9 +195,7 @@ public void testInitialFileWorks() throws Exception {
             return null;
         }).when(controller).process(any(), any(XContentParser.class), any(), any());
 
-        CountDownLatch latch = new CountDownLatch(1);
-
-        fileSettingsService.addFileChangedListener(latch::countDown);
+        CountDownLatch fileProcessingLatch = new CountDownLatch(1);
 
         Files.createDirectories(fileSettingsService.watchedFileDir());
         // contents of the JSON don't matter, we just need a file to exist
@@ -202,15 +205,14 @@ public void testInitialFileWorks() throws Exception {
             try {
                 return invocation.callRealMethod();
             } finally {
-                latch.countDown();
+                fileProcessingLatch.countDown();
             }
         }).when(fileSettingsService).processFileOnServiceStart();
 
         fileSettingsService.start();
         fileSettingsService.clusterChanged(new ClusterChangedEvent("test", clusterService.state(), ClusterState.EMPTY_STATE));
 
-        // wait for listener to be called
-        assertTrue(latch.await(20, TimeUnit.SECONDS));
+        longAwait(fileProcessingLatch);
 
         verify(fileSettingsService, times(1)).processFileOnServiceStart();
         verify(controller, times(1)).process(any(), any(XContentParser.class), eq(ReservedStateVersionCheck.HIGHER_OR_SAME_VERSION), any());
@@ -223,40 +225,40 @@ public void testProcessFileChanges() throws Exception {
             return null;
         }).when(controller).process(any(), any(XContentParser.class), any(), any());
 
-        // we get three events: initial clusterChanged event, first write, second write
-        CountDownLatch latch = new CountDownLatch(3);
-
-        fileSettingsService.addFileChangedListener(latch::countDown);
-
-        Files.createDirectories(fileSettingsService.watchedFileDir());
-        // contents of the JSON don't matter, we just need a file to exist
-        writeTestFile(fileSettingsService.watchedFile(), "{}");
-
+        CountDownLatch changesOnStartLatch = new CountDownLatch(1);
         doAnswer((Answer<?>) invocation -> {
             try {
                 return invocation.callRealMethod();
             } finally {
-                latch.countDown();
+                changesOnStartLatch.countDown();
             }
         }).when(fileSettingsService).processFileOnServiceStart();
+
+        CountDownLatch changesLatch = new CountDownLatch(1);
         doAnswer((Answer<?>) invocation -> {
             try {
                 return invocation.callRealMethod();
             } finally {
-                latch.countDown();
+                changesLatch.countDown();
             }
         }).when(fileSettingsService).processFileChanges();
 
+        Files.createDirectories(fileSettingsService.watchedFileDir());
+        // contents of the JSON don't matter, we just need a file to exist
+        writeTestFile(fileSettingsService.watchedFile(), "{}");
+
         fileSettingsService.start();
         fileSettingsService.clusterChanged(new ClusterChangedEvent("test", clusterService.state(), ClusterState.EMPTY_STATE));
-        // second file change; contents still don't matter
-        overwriteTestFile(fileSettingsService.watchedFile(), "{}");
 
-        // wait for listener to be called (once for initial processing, once for subsequent update)
-        assertTrue(latch.await(20, TimeUnit.SECONDS));
+        longAwait(changesOnStartLatch);
 
         verify(fileSettingsService, times(1)).processFileOnServiceStart();
         verify(controller, times(1)).process(any(), any(XContentParser.class), eq(ReservedStateVersionCheck.HIGHER_OR_SAME_VERSION), any());
+
+        // second file change; contents still don't matter
+        writeTestFile(fileSettingsService.watchedFile(), "[]");
+        longAwait(changesLatch);
+
         verify(fileSettingsService, times(1)).processFileChanges();
         verify(controller, times(1)).process(any(), any(XContentParser.class), eq(ReservedStateVersionCheck.HIGHER_VERSION_ONLY), any());
     }
@@ -295,9 +297,7 @@ public void testStopWorksInMiddleOfProcessing() throws Exception {
         // Make some fake settings file to cause the file settings service to process it
         writeTestFile(fileSettingsService.watchedFile(), "{}");
 
-        // we need to wait a bit, on MacOS it may take up to 10 seconds for the Java watcher service to notice the file,
-        // on Linux is instantaneous. Windows is instantaneous too.
-        assertTrue(processFileLatch.await(30, TimeUnit.SECONDS));
+        longAwait(processFileLatch);
 
         // Stopping the service should interrupt the watcher thread, we should be able to stop
         fileSettingsService.stop();
@@ -352,15 +352,34 @@ public void testHandleSnapshotRestoreResetsMetadata() throws Exception {
     }
 
     // helpers
-    private void writeTestFile(Path path, String contents) throws IOException {
-        Path tempFilePath = createTempFile();
-        Files.writeString(tempFilePath, contents);
-        Files.move(tempFilePath, path, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING);
+    private static void writeTestFile(Path path, String contents) {
+        Path tempFile = null;
+        try {
+            tempFile = Files.createTempFile(path.getParent(), path.getFileName().toString(), "tmp");
+            Files.writeString(tempFile, contents);
+
+            try {
+                Files.move(tempFile, path, REPLACE_EXISTING, ATOMIC_MOVE);
+            } catch (AtomicMoveNotSupportedException e) {
+                Files.move(tempFile, path, REPLACE_EXISTING);
+            }
+        } catch (final IOException e) {
+            throw new UncheckedIOException(Strings.format("could not write file [%s]", path.toAbsolutePath()), e);
+        } finally {
+            // we are ignoring exceptions here, so we do not need handle whether or not tempFile was initialized nor if the file exists
+            IOUtils.deleteFilesIgnoringExceptions(tempFile);
+        }
     }
 
-    private void overwriteTestFile(Path path, String contents) throws IOException {
-        Path tempFilePath = createTempFile();
-        Files.writeString(tempFilePath, contents);
-        Files.move(tempFilePath, path, StandardCopyOption.REPLACE_EXISTING);
+    // this waits for up to 20 seconds to account for watcher service differences between OSes
+    // on MacOS it may take up to 10 seconds for the Java watcher service to notice the file,
+    // on Linux is instantaneous. Windows is instantaneous too.
+    private static void longAwait(CountDownLatch latch) {
+        try {
+            assertTrue("longAwait: CountDownLatch did not reach zero within the timeout", latch.await(20, TimeUnit.SECONDS));
+        } catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
+            fail(e, "longAwait: interrupted waiting for CountDownLatch to reach zero");
+        }
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/rest/action/document/RestBulkActionTests.java b/server/src/test/java/org/elasticsearch/rest/action/document/RestBulkActionTests.java
index d3cd6dd9ca420..25cfd1e56514c 100644
--- a/server/src/test/java/org/elasticsearch/rest/action/document/RestBulkActionTests.java
+++ b/server/src/test/java/org/elasticsearch/rest/action/document/RestBulkActionTests.java
@@ -20,8 +20,6 @@
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.ReleasableBytesReference;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.core.Releasable;
 import org.elasticsearch.http.HttpBody;
 import org.elasticsearch.index.IndexVersion;
@@ -67,7 +65,7 @@ public void bulk(BulkRequest request, ActionListener<BulkResponse> listener) {
             params.put("pipeline", "timestamps");
             new RestBulkAction(
                 settings(IndexVersion.current()).build(),
-                new IncrementalBulkService(mock(Client.class), mock(IndexingPressure.class), new ThreadContext(Settings.EMPTY))
+                new IncrementalBulkService(mock(Client.class), mock(IndexingPressure.class))
             ).handleRequest(
                 new FakeRestRequest.Builder(xContentRegistry()).withPath("my_index/_bulk").withParams(params).withContent(new BytesArray("""
                     {"index":{"_id":"1"}}
@@ -102,7 +100,7 @@ public void bulk(BulkRequest request, ActionListener<BulkResponse> listener) {
             {
                 new RestBulkAction(
                     settings(IndexVersion.current()).build(),
-                    new IncrementalBulkService(mock(Client.class), mock(IndexingPressure.class), new ThreadContext(Settings.EMPTY))
+                    new IncrementalBulkService(mock(Client.class), mock(IndexingPressure.class))
                 ).handleRequest(
                     new FakeRestRequest.Builder(xContentRegistry()).withPath("my_index/_bulk")
                         .withParams(params)
@@ -126,7 +124,7 @@ public void bulk(BulkRequest request, ActionListener<BulkResponse> listener) {
                 bulkCalled.set(false);
                 new RestBulkAction(
                     settings(IndexVersion.current()).build(),
-                    new IncrementalBulkService(mock(Client.class), mock(IndexingPressure.class), new ThreadContext(Settings.EMPTY))
+                    new IncrementalBulkService(mock(Client.class), mock(IndexingPressure.class))
                 ).handleRequest(
                     new FakeRestRequest.Builder(xContentRegistry()).withPath("my_index/_bulk")
                         .withParams(params)
@@ -149,7 +147,7 @@ public void bulk(BulkRequest request, ActionListener<BulkResponse> listener) {
                 bulkCalled.set(false);
                 new RestBulkAction(
                     settings(IndexVersion.current()).build(),
-                    new IncrementalBulkService(mock(Client.class), mock(IndexingPressure.class), new ThreadContext(Settings.EMPTY))
+                    new IncrementalBulkService(mock(Client.class), mock(IndexingPressure.class))
                 ).handleRequest(
                     new FakeRestRequest.Builder(xContentRegistry()).withPath("my_index/_bulk")
                         .withParams(params)
@@ -173,7 +171,7 @@ public void bulk(BulkRequest request, ActionListener<BulkResponse> listener) {
                 bulkCalled.set(false);
                 new RestBulkAction(
                     settings(IndexVersion.current()).build(),
-                    new IncrementalBulkService(mock(Client.class), mock(IndexingPressure.class), new ThreadContext(Settings.EMPTY))
+                    new IncrementalBulkService(mock(Client.class), mock(IndexingPressure.class))
                 ).handleRequest(
                     new FakeRestRequest.Builder(xContentRegistry()).withPath("my_index/_bulk")
                         .withParams(params)
@@ -229,7 +227,7 @@ public void next() {
         RestBulkAction.ChunkHandler chunkHandler = new RestBulkAction.ChunkHandler(
             true,
             request,
-            () -> new IncrementalBulkService.Handler(null, new ThreadContext(Settings.EMPTY), null, null, null, null) {
+            () -> new IncrementalBulkService.Handler(null, null, null, null, null) {
 
                 @Override
                 public void addItems(List<DocWriteRequest<?>> items, Releasable releasable, Runnable nextItems) {
diff --git a/settings.gradle b/settings.gradle
index 39453e8d0935a..25ed048d57253 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -155,17 +155,7 @@ addSubProjects('', new File(rootProject.projectDir, 'x-pack/libs'))
 
 include projects.toArray(new String[0])
 
-project(":libs").children.each { libsProject ->
-  libsProject.name = "elasticsearch-${libsProject.name}"
-  libsProject.children.each { lp ->
-    lp.name = lp.name  // for :libs:elasticsearch-x-content:impl
-  }
-}
-project(":libs:elasticsearch-native:libraries").name = "elasticsearch-native-libraries"
-
-project(":qa:stable-api").children.each { libsProject ->
-  libsProject.name = "elasticsearch-${libsProject.name}"
-}
+project(":libs:native:libraries").name = "native-libraries"
 
 project(":test:external-modules").children.each { testProject ->
   testProject.name = "test-${testProject.name}"
diff --git a/test/external-modules/apm-integration/build.gradle b/test/external-modules/apm-integration/build.gradle
index 98090f33ee2c7..d0f5f889e9b30 100644
--- a/test/external-modules/apm-integration/build.gradle
+++ b/test/external-modules/apm-integration/build.gradle
@@ -22,5 +22,5 @@ tasks.named('javaRestTest').configure {
 
 dependencies {
   clusterModules project(':modules:apm')
-  implementation project(':libs:elasticsearch-logging')
+  implementation project(':libs:logging')
 }
diff --git a/test/fixtures/geoip-fixture/build.gradle b/test/fixtures/geoip-fixture/build.gradle
index f20db481814ea..13d2b6ae88e9c 100644
--- a/test/fixtures/geoip-fixture/build.gradle
+++ b/test/fixtures/geoip-fixture/build.gradle
@@ -13,8 +13,8 @@ description = 'Fixture for GeoIPv2 service'
 dependencies {
   api project(':server')
   api project(':distribution:tools:geoip-cli')
-  api project(":libs:elasticsearch-cli")
-  api project(":libs:elasticsearch-x-content")
+  api project(":libs:cli")
+  api project(":libs:x-content")
   api("junit:junit:${versions.junit}") {
     exclude module: 'hamcrest-core'
   }
diff --git a/test/framework/build.gradle b/test/framework/build.gradle
index 72a8eade3bce0..f130ecf131848 100644
--- a/test/framework/build.gradle
+++ b/test/framework/build.gradle
@@ -14,9 +14,9 @@ apply plugin: 'elasticsearch.publish'
 dependencies {
   api project(":client:rest")
   api project(':modules:transport-netty4')
-  api project(':libs:elasticsearch-ssl-config')
+  api project(':libs:ssl-config')
   api project(":server")
-  api project(":libs:elasticsearch-cli")
+  api project(":libs:cli")
   api "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}"
   api "junit:junit:${versions.junit}"
   api "org.hamcrest:hamcrest:${versions.hamcrest}"
diff --git a/test/x-content/build.gradle b/test/x-content/build.gradle
index 48667eeb58735..281148a0fe819 100644
--- a/test/x-content/build.gradle
+++ b/test/x-content/build.gradle
@@ -12,7 +12,7 @@ apply plugin: 'elasticsearch.publish'
 
 dependencies {
   api project(":test:framework")
-  api project(":libs:elasticsearch-x-content")
+  api project(":libs:x-content")
 
   // json schema validation dependencies
   implementation "com.fasterxml.jackson.core:jackson-core:${versions.jackson}"
diff --git a/x-pack/plugin/blob-cache/build.gradle b/x-pack/plugin/blob-cache/build.gradle
index ff21b64def768..c5c91a5ef87e3 100644
--- a/x-pack/plugin/blob-cache/build.gradle
+++ b/x-pack/plugin/blob-cache/build.gradle
@@ -15,5 +15,5 @@ esplugin {
 }
 
 dependencies {
-    compileOnly project(path: ':libs:elasticsearch-native')
+    compileOnly project(path: ':libs:native')
 }
diff --git a/x-pack/plugin/core/build.gradle b/x-pack/plugin/core/build.gradle
index 1ed59d6fe3581..fb4acb0055a8c 100644
--- a/x-pack/plugin/core/build.gradle
+++ b/x-pack/plugin/core/build.gradle
@@ -36,8 +36,8 @@ configurations {
 
 dependencies {
   compileOnly project(":server")
-  api project(':libs:elasticsearch-grok')
-  api project(":libs:elasticsearch-ssl-config")
+  api project(':libs:grok')
+  api project(":libs:ssl-config")
   api "org.apache.httpcomponents:httpclient:${versions.httpclient}"
   api "org.apache.httpcomponents:httpcore:${versions.httpcore}"
   api "org.apache.httpcomponents:httpcore-nio:${versions.httpcore}"
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/ResultUtils.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/ResultUtils.java
index 4fe2c9ae486f1..eb68af7589717 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/ResultUtils.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/ResultUtils.java
@@ -14,8 +14,9 @@ public class ResultUtils {
 
     public static ElasticsearchStatusException createInvalidChunkedResultException(String expectedResultName, String receivedResultName) {
         return new ElasticsearchStatusException(
-            "Expected a chunked inference [{}] received [{}]",
-            RestStatus.INTERNAL_SERVER_ERROR,
+            "Received incompatible results. Check that your model_id matches the task_type of this endpoint. "
+                + "Expected chunked output of type [{}] but received [{}].",
+            RestStatus.CONFLICT,
             expectedResultName,
             receivedResultName
         );
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/assignment/TrainedModelAssignment.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/assignment/TrainedModelAssignment.java
index 06c3f75587d62..efd07cceae09b 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/assignment/TrainedModelAssignment.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/assignment/TrainedModelAssignment.java
@@ -533,6 +533,9 @@ public AssignmentState calculateAssignmentState() {
             if (assignmentState.equals(AssignmentState.STOPPING)) {
                 return assignmentState;
             }
+            if (taskParams.getNumberOfAllocations() == 0) {
+                return AssignmentState.STARTED;
+            }
             if (nodeRoutingTable.values().stream().anyMatch(r -> r.getState().equals(RoutingState.STARTED))) {
                 return AssignmentState.STARTED;
             }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/mapper/ExpressionRoleMapping.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/mapper/ExpressionRoleMapping.java
index c504ebe56ed45..41fd3c6938dfc 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/mapper/ExpressionRoleMapping.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/mapper/ExpressionRoleMapping.java
@@ -206,7 +206,7 @@ public RoleMapperExpression getExpression() {
      * that match the {@link #getExpression() expression} in this mapping.
      */
     public List<String> getRoles() {
-        return Collections.unmodifiableList(roles);
+        return roles != null ? Collections.unmodifiableList(roles) : Collections.emptyList();
     }
 
     /**
@@ -214,7 +214,7 @@ public List<String> getRoles() {
      * that should be assigned to users that match the {@link #getExpression() expression} in this mapping.
      */
     public List<TemplateRoleName> getRoleTemplates() {
-        return Collections.unmodifiableList(roleTemplates);
+        return roleTemplates != null ? Collections.unmodifiableList(roleTemplates) : Collections.emptyList();
     }
 
     /**
@@ -223,7 +223,7 @@ public List<TemplateRoleName> getRoleTemplates() {
      * This is not used within the mapping process, and does not affect whether the expression matches, nor which roles are assigned.
      */
     public Map<String, Object> getMetadata() {
-        return Collections.unmodifiableMap(metadata);
+        return metadata != null ? Collections.unmodifiableMap(metadata) : Collections.emptyMap();
     }
 
     /**
@@ -233,6 +233,15 @@ public boolean isEnabled() {
         return enabled;
     }
 
+    /**
+     * Whether this mapping is an operator defined/read only role mapping
+     */
+    public boolean isReadOnly() {
+        return metadata != null && metadata.get(ExpressionRoleMapping.READ_ONLY_ROLE_MAPPING_METADATA_FLAG) instanceof Boolean readOnly
+            ? readOnly
+            : false;
+    }
+
     @Override
     public String toString() {
         return getClass().getSimpleName() + "<" + name + " ; " + roles + "/" + roleTemplates + " = " + Strings.toString(expression) + ">";
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/RoleMappingMetadata.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/RoleMappingMetadata.java
index 74c6223b1ebdd..31fe86ca77edd 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/RoleMappingMetadata.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/RoleMappingMetadata.java
@@ -191,6 +191,14 @@ public static boolean hasFallbackName(ExpressionRoleMapping expressionRoleMappin
         return expressionRoleMapping.getName().equals(FALLBACK_NAME);
     }
 
+    /**
+     * Check if any of the role mappings have a fallback name
+     * @return true if any role mappings have the fallback name
+     */
+    public boolean hasAnyMappingWithFallbackName() {
+        return roleMappings.stream().anyMatch(RoleMappingMetadata::hasFallbackName);
+    }
+
     /**
      * Parse a role mapping from XContent, restoring the name from a reserved metadata field.
      * Used to parse a role mapping annotated with its name in metadata via @see {@link #copyWithNameInMetadata(ExpressionRoleMapping)}.
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/assignment/TrainedModelAssignmentTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/assignment/TrainedModelAssignmentTests.java
index c3b6e0089b4ae..dc0a8b52e585a 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/assignment/TrainedModelAssignmentTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/assignment/TrainedModelAssignmentTests.java
@@ -172,6 +172,11 @@ public void testCalculateAssignmentState_GivenNoStartedAssignments() {
         assertThat(builder.calculateAssignmentState(), equalTo(AssignmentState.STARTING));
     }
 
+    public void testCalculateAssignmentState_GivenNumAllocationsIsZero() {
+        TrainedModelAssignment.Builder builder = TrainedModelAssignment.Builder.empty(randomTaskParams(0), null);
+        assertThat(builder.calculateAssignmentState(), equalTo(AssignmentState.STARTED));
+    }
+
     public void testCalculateAssignmentState_GivenOneStartedAssignment() {
         TrainedModelAssignment.Builder builder = TrainedModelAssignment.Builder.empty(randomTaskParams(5), null);
         builder.addRoutingEntry("node-1", new RoutingInfo(4, 4, RoutingState.STARTING, ""));
diff --git a/x-pack/plugin/esql/build.gradle b/x-pack/plugin/esql/build.gradle
index 766d0c0f13892..150017ce9e955 100644
--- a/x-pack/plugin/esql/build.gradle
+++ b/x-pack/plugin/esql/build.gradle
@@ -26,8 +26,8 @@ dependencies {
   compileOnly project(xpackModule('ml'))
   implementation project('compute')
   implementation project('compute:ann')
-  implementation project(':libs:elasticsearch-dissect')
-  implementation project(':libs:elasticsearch-grok')
+  implementation project(':libs:dissect')
+  implementation project(':libs:grok')
   implementation project('arrow')
 
   // Also contains a dummy processor to allow compilation with unused annotations.
diff --git a/x-pack/plugin/esql/qa/testFixtures/build.gradle b/x-pack/plugin/esql/qa/testFixtures/build.gradle
index b6ed610406631..903986466b77f 100644
--- a/x-pack/plugin/esql/qa/testFixtures/build.gradle
+++ b/x-pack/plugin/esql/qa/testFixtures/build.gradle
@@ -5,9 +5,9 @@ dependencies {
   implementation project(':x-pack:plugin:esql:compute')
   implementation project(':x-pack:plugin:esql')
   compileOnly project(path: xpackModule('core'))
-  implementation project(":libs:elasticsearch-x-content")
+  implementation project(":libs:x-content")
   implementation project(':client:rest')
-  implementation project(':libs:elasticsearch-logging')
+  implementation project(':libs:logging')
   implementation project(':test:framework')
   api(testArtifact(project(xpackModule('esql-core'))))
   implementation project(':server')
diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterEnrichUnavailableClustersIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterEnrichUnavailableClustersIT.java
new file mode 100644
index 0000000000000..d142752d0c408
--- /dev/null
+++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterEnrichUnavailableClustersIT.java
@@ -0,0 +1,690 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.action;
+
+import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.client.internal.Client;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.CollectionUtils;
+import org.elasticsearch.core.Tuple;
+import org.elasticsearch.ingest.common.IngestCommonPlugin;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.reindex.ReindexPlugin;
+import org.elasticsearch.test.AbstractMultiClustersTestCase;
+import org.elasticsearch.transport.RemoteClusterAware;
+import org.elasticsearch.xpack.core.XPackSettings;
+import org.elasticsearch.xpack.core.enrich.action.ExecuteEnrichPolicyAction;
+import org.elasticsearch.xpack.core.enrich.action.PutEnrichPolicyAction;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.plan.logical.Enrich;
+import org.elasticsearch.xpack.esql.plugin.EsqlPlugin;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.elasticsearch.xpack.esql.EsqlTestUtils.getValuesList;
+import static org.elasticsearch.xpack.esql.action.CrossClustersEnrichIT.enrichHosts;
+import static org.elasticsearch.xpack.esql.action.CrossClustersEnrichIT.enrichVendors;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.lessThanOrEqualTo;
+
+/**
+ * This IT test is the dual of CrossClustersEnrichIT, which tests "happy path"
+ * and this one tests unavailable cluster scenarios using (most of) the same tests.
+ */
+public class CrossClusterEnrichUnavailableClustersIT extends AbstractMultiClustersTestCase {
+
+    public static String REMOTE_CLUSTER_1 = "c1";
+    public static String REMOTE_CLUSTER_2 = "c2";
+
+    @Override
+    protected Collection<String> remoteClusterAlias() {
+        return List.of(REMOTE_CLUSTER_1, REMOTE_CLUSTER_2);
+    }
+
+    @Override
+    protected boolean reuseClusters() {
+        return false;
+    }
+
+    private Collection<String> allClusters() {
+        return CollectionUtils.appendToCopy(remoteClusterAlias(), LOCAL_CLUSTER);
+    }
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins(String clusterAlias) {
+        List<Class<? extends Plugin>> plugins = new ArrayList<>(super.nodePlugins(clusterAlias));
+        plugins.add(EsqlPlugin.class);
+        plugins.add(CrossClustersEnrichIT.LocalStateEnrich.class);
+        plugins.add(IngestCommonPlugin.class);
+        plugins.add(ReindexPlugin.class);
+        return plugins;
+    }
+
+    @Override
+    protected Settings nodeSettings() {
+        return Settings.builder().put(super.nodeSettings()).put(XPackSettings.SECURITY_ENABLED.getKey(), false).build();
+    }
+
+    @Before
+    public void setupHostsEnrich() {
+        // the hosts policy are identical on every node
+        Map<String, String> allHosts = Map.of(
+            "192.168.1.2",
+            "Windows",
+            "192.168.1.3",
+            "MacOS",
+            "192.168.1.4",
+            "Linux",
+            "192.168.1.5",
+            "Android",
+            "192.168.1.6",
+            "iOS",
+            "192.168.1.7",
+            "Windows",
+            "192.168.1.8",
+            "MacOS",
+            "192.168.1.9",
+            "Linux",
+            "192.168.1.10",
+            "Linux",
+            "192.168.1.11",
+            "Windows"
+        );
+        for (String cluster : allClusters()) {
+            Client client = client(cluster);
+            client.admin().indices().prepareCreate("hosts").setMapping("ip", "type=ip", "os", "type=keyword").get();
+            for (Map.Entry<String, String> h : allHosts.entrySet()) {
+                client.prepareIndex("hosts").setSource("ip", h.getKey(), "os", h.getValue()).get();
+            }
+            client.admin().indices().prepareRefresh("hosts").get();
+            client.execute(
+                PutEnrichPolicyAction.INSTANCE,
+                new PutEnrichPolicyAction.Request(TEST_REQUEST_TIMEOUT, "hosts", CrossClustersEnrichIT.hostPolicy)
+            ).actionGet();
+            client.execute(ExecuteEnrichPolicyAction.INSTANCE, new ExecuteEnrichPolicyAction.Request(TEST_REQUEST_TIMEOUT, "hosts"))
+                .actionGet();
+            assertAcked(client.admin().indices().prepareDelete("hosts"));
+        }
+    }
+
+    @Before
+    public void setupVendorPolicy() {
+        var localVendors = Map.of("Windows", "Microsoft", "MacOS", "Apple", "iOS", "Apple", "Android", "Samsung", "Linux", "Redhat");
+        var c1Vendors = Map.of("Windows", "Microsoft", "MacOS", "Apple", "iOS", "Apple", "Android", "Google", "Linux", "Suse");
+        var c2Vendors = Map.of("Windows", "Microsoft", "MacOS", "Apple", "iOS", "Apple", "Android", "Sony", "Linux", "Ubuntu");
+        var vendors = Map.of(LOCAL_CLUSTER, localVendors, "c1", c1Vendors, "c2", c2Vendors);
+        for (Map.Entry<String, Map<String, String>> e : vendors.entrySet()) {
+            Client client = client(e.getKey());
+            client.admin().indices().prepareCreate("vendors").setMapping("os", "type=keyword", "vendor", "type=keyword").get();
+            for (Map.Entry<String, String> v : e.getValue().entrySet()) {
+                client.prepareIndex("vendors").setSource("os", v.getKey(), "vendor", v.getValue()).get();
+            }
+            client.admin().indices().prepareRefresh("vendors").get();
+            client.execute(
+                PutEnrichPolicyAction.INSTANCE,
+                new PutEnrichPolicyAction.Request(TEST_REQUEST_TIMEOUT, "vendors", CrossClustersEnrichIT.vendorPolicy)
+            ).actionGet();
+            client.execute(ExecuteEnrichPolicyAction.INSTANCE, new ExecuteEnrichPolicyAction.Request(TEST_REQUEST_TIMEOUT, "vendors"))
+                .actionGet();
+            assertAcked(client.admin().indices().prepareDelete("vendors"));
+        }
+    }
+
+    @Before
+    public void setupEventsIndices() {
+        record Event(long timestamp, String user, String host) {}
+
+        List<Event> e0 = List.of(
+            new Event(1, "matthew", "192.168.1.3"),
+            new Event(2, "simon", "192.168.1.5"),
+            new Event(3, "park", "192.168.1.2"),
+            new Event(4, "andrew", "192.168.1.7"),
+            new Event(5, "simon", "192.168.1.20"),
+            new Event(6, "kevin", "192.168.1.2"),
+            new Event(7, "akio", "192.168.1.5"),
+            new Event(8, "luke", "192.168.1.2"),
+            new Event(9, "jack", "192.168.1.4")
+        );
+        List<Event> e1 = List.of(
+            new Event(1, "andres", "192.168.1.2"),
+            new Event(2, "sergio", "192.168.1.6"),
+            new Event(3, "kylian", "192.168.1.8"),
+            new Event(4, "andrew", "192.168.1.9"),
+            new Event(5, "jack", "192.168.1.3"),
+            new Event(6, "kevin", "192.168.1.4"),
+            new Event(7, "akio", "192.168.1.7"),
+            new Event(8, "kevin", "192.168.1.21"),
+            new Event(9, "andres", "192.168.1.8")
+        );
+        List<Event> e2 = List.of(
+            new Event(1, "park", "192.168.1.25"),
+            new Event(2, "akio", "192.168.1.5"),
+            new Event(3, "park", "192.168.1.2"),
+            new Event(4, "kevin", "192.168.1.3")
+        );
+        for (var c : Map.of(LOCAL_CLUSTER, e0, "c1", e1, "c2", e2).entrySet()) {
+            Client client = client(c.getKey());
+            client.admin()
+                .indices()
+                .prepareCreate("events")
+                .setMapping("timestamp", "type=long", "user", "type=keyword", "host", "type=ip")
+                .get();
+            for (var e : c.getValue()) {
+                client.prepareIndex("events").setSource("timestamp", e.timestamp, "user", e.user, "host", e.host).get();
+            }
+            client.admin().indices().prepareRefresh("events").get();
+        }
+    }
+
+    public void testEnrichWithHostsPolicyAndDisconnectedRemotesWithSkipUnavailableTrue() throws IOException {
+        setSkipUnavailable(REMOTE_CLUSTER_1, true);
+        setSkipUnavailable(REMOTE_CLUSTER_2, true);
+
+        try {
+            // close remote-cluster-1 so that it is unavailable
+            cluster(REMOTE_CLUSTER_1).close();
+
+            Tuple<Boolean, Boolean> includeCCSMetadata = CrossClustersEnrichIT.randomIncludeCCSMetadata();
+            Boolean requestIncludeMeta = includeCCSMetadata.v1();
+            boolean responseExpectMeta = includeCCSMetadata.v2();
+
+            {
+                Enrich.Mode mode = randomFrom(Enrich.Mode.values());
+                String query = "FROM *:events | eval ip= TO_STR(host) | " + enrichHosts(mode) + " | stats c = COUNT(*) by os | SORT os";
+                try (EsqlQueryResponse resp = runQuery(query, requestIncludeMeta)) {
+                    List<List<Object>> rows = getValuesList(resp);
+                    assertThat(rows.size(), greaterThanOrEqualTo(1));
+                    EsqlExecutionInfo executionInfo = resp.getExecutionInfo();
+                    assertCCSExecutionInfoDetails(executionInfo);
+
+                    assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta));
+                    assertThat(executionInfo.clusterAliases(), equalTo(Set.of(REMOTE_CLUSTER_1, REMOTE_CLUSTER_2)));
+
+                    EsqlExecutionInfo.Cluster cluster1 = executionInfo.getCluster(REMOTE_CLUSTER_1);
+                    assertThat(cluster1.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SKIPPED));
+                    assertThat(cluster1.getTotalShards(), equalTo(0));
+                    assertThat(cluster1.getSuccessfulShards(), equalTo(0));
+                    assertThat(cluster1.getSkippedShards(), equalTo(0));
+                    assertThat(cluster1.getFailedShards(), equalTo(0));
+
+                    EsqlExecutionInfo.Cluster cluster2 = executionInfo.getCluster(REMOTE_CLUSTER_2);
+                    assertThat(cluster2.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL));
+                    assertThat(cluster2.getTotalShards(), greaterThanOrEqualTo(0));
+                    assertThat(cluster2.getSuccessfulShards(), equalTo(cluster2.getSuccessfulShards()));
+                    assertThat(cluster2.getSkippedShards(), equalTo(0));
+                    assertThat(cluster2.getFailedShards(), equalTo(0));
+                }
+            }
+
+            // close remote-cluster-2 so that it is also unavailable
+            cluster(REMOTE_CLUSTER_2).close();
+
+            {
+                Enrich.Mode mode = randomFrom(Enrich.Mode.values());
+                String query = "FROM *:events | eval ip= TO_STR(host) | " + enrichHosts(mode) + " | stats c = COUNT(*) by os | SORT os";
+                try (EsqlQueryResponse resp = runQuery(query, requestIncludeMeta)) {
+                    List<ColumnInfoImpl> columns = resp.columns();
+                    assertThat(columns.size(), equalTo(1));
+                    // column from an empty result should be {"name":"<no-fields>","type":"null"}
+                    assertThat(columns.get(0).name(), equalTo("<no-fields>"));
+                    assertThat(columns.get(0).type(), equalTo(DataType.NULL));
+
+                    List<List<Object>> rows = getValuesList(resp);
+                    assertThat(rows.size(), equalTo(0));
+
+                    EsqlExecutionInfo executionInfo = resp.getExecutionInfo();
+                    assertCCSExecutionInfoDetails(executionInfo);
+
+                    assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta));
+                    assertThat(executionInfo.clusterAliases(), equalTo(Set.of(REMOTE_CLUSTER_1, REMOTE_CLUSTER_2)));
+
+                    EsqlExecutionInfo.Cluster cluster1 = executionInfo.getCluster(REMOTE_CLUSTER_1);
+                    assertThat(cluster1.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SKIPPED));
+                    assertThat(cluster1.getTotalShards(), equalTo(0));
+                    assertThat(cluster1.getSuccessfulShards(), equalTo(0));
+                    assertThat(cluster1.getSkippedShards(), equalTo(0));
+                    assertThat(cluster1.getFailedShards(), equalTo(0));
+
+                    EsqlExecutionInfo.Cluster cluster2 = executionInfo.getCluster(REMOTE_CLUSTER_2);
+                    assertThat(cluster2.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SKIPPED));
+                    assertThat(cluster2.getTotalShards(), equalTo(0));
+                    assertThat(cluster2.getSuccessfulShards(), equalTo(0));
+                    assertThat(cluster2.getSkippedShards(), equalTo(0));
+                    assertThat(cluster2.getFailedShards(), equalTo(0));
+                }
+            }
+        } finally {
+            clearSkipUnavailable();
+        }
+    }
+
+    public void testEnrichWithHostsPolicyAndDisconnectedRemotesWithSkipUnavailableFalse() throws IOException {
+        setSkipUnavailable(REMOTE_CLUSTER_1, true);
+        setSkipUnavailable(REMOTE_CLUSTER_2, false);
+
+        try {
+            // close remote-cluster-1 so that it is unavailable
+            cluster(REMOTE_CLUSTER_1).close();
+
+            Tuple<Boolean, Boolean> includeCCSMetadata = CrossClustersEnrichIT.randomIncludeCCSMetadata();
+            Boolean requestIncludeMeta = includeCCSMetadata.v1();
+            boolean responseExpectMeta = includeCCSMetadata.v2();
+
+            {
+                Enrich.Mode mode = randomFrom(Enrich.Mode.values());
+                String query = "FROM *:events | EVAL ip= TO_STR(host) | " + enrichHosts(mode) + " | STATS c = COUNT(*) by os | SORT os";
+                try (EsqlQueryResponse resp = runQuery(query, requestIncludeMeta)) {
+                    List<List<Object>> rows = getValuesList(resp);
+                    assertThat(rows.size(), greaterThanOrEqualTo(1));
+                    EsqlExecutionInfo executionInfo = resp.getExecutionInfo();
+                    assertCCSExecutionInfoDetails(executionInfo);
+
+                    assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta));
+                    assertThat(executionInfo.clusterAliases(), equalTo(Set.of(REMOTE_CLUSTER_1, REMOTE_CLUSTER_2)));
+
+                    EsqlExecutionInfo.Cluster cluster1 = executionInfo.getCluster(REMOTE_CLUSTER_1);
+                    assertThat(cluster1.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SKIPPED));
+                    assertThat(cluster1.getTotalShards(), equalTo(0));
+                    assertThat(cluster1.getSuccessfulShards(), equalTo(0));
+                    assertThat(cluster1.getSkippedShards(), equalTo(0));
+                    assertThat(cluster1.getFailedShards(), equalTo(0));
+
+                    EsqlExecutionInfo.Cluster cluster2 = executionInfo.getCluster(REMOTE_CLUSTER_2);
+                    assertThat(cluster2.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL));
+                    assertThat(cluster2.getTotalShards(), greaterThanOrEqualTo(0));
+                    assertThat(cluster2.getSuccessfulShards(), equalTo(cluster2.getSuccessfulShards()));
+                    assertThat(cluster2.getSkippedShards(), equalTo(0));
+                    assertThat(cluster2.getFailedShards(), equalTo(0));
+                }
+            }
+
+            // close remote-cluster-2 so that it is also unavailable
+            cluster(REMOTE_CLUSTER_2).close();
+            {
+                Enrich.Mode mode = randomFrom(Enrich.Mode.values());
+                String query = "FROM *:events | eval ip= TO_STR(host) | " + enrichHosts(mode) + " | stats c = COUNT(*) by os | SORT os";
+                Exception exception = expectThrows(Exception.class, () -> runQuery(query, requestIncludeMeta));
+                assertTrue(ExceptionsHelper.isRemoteUnavailableException(exception));
+            }
+        } finally {
+            clearSkipUnavailable();
+        }
+    }
+
+    public void testEnrichTwiceThenAggsWithUnavailableRemotes() throws IOException {
+        Tuple<Boolean, Boolean> includeCCSMetadata = CrossClustersEnrichIT.randomIncludeCCSMetadata();
+        Boolean requestIncludeMeta = includeCCSMetadata.v1();
+        boolean responseExpectMeta = includeCCSMetadata.v2();
+
+        boolean skipUnavailableRemote1 = randomBoolean();
+        setSkipUnavailable(REMOTE_CLUSTER_1, skipUnavailableRemote1);
+        setSkipUnavailable(REMOTE_CLUSTER_2, true);
+
+        try {
+            // close remote-cluster-2 so that it is unavailable
+            cluster(REMOTE_CLUSTER_2).close();
+
+            for (var hostMode : Enrich.Mode.values()) {
+                String query = String.format(Locale.ROOT, """
+                    FROM *:events,events
+                    | eval ip= TO_STR(host)
+                    | %s
+                    | %s
+                    | stats c = COUNT(*) by vendor
+                    | sort vendor
+                    """, enrichHosts(hostMode), enrichVendors(Enrich.Mode.COORDINATOR));
+                try (EsqlQueryResponse resp = runQuery(query, requestIncludeMeta)) {
+                    assertThat(getValuesList(resp).size(), greaterThanOrEqualTo(1));
+                    EsqlExecutionInfo executionInfo = resp.getExecutionInfo();
+                    assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta));
+                    assertThat(executionInfo.clusterAliases(), equalTo(Set.of("", REMOTE_CLUSTER_1, REMOTE_CLUSTER_2)));
+                    assertCCSExecutionInfoDetails(executionInfo);
+
+                    EsqlExecutionInfo.Cluster cluster1 = executionInfo.getCluster(REMOTE_CLUSTER_1);
+                    assertThat(cluster1.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL));
+                    assertThat(cluster1.getTotalShards(), greaterThanOrEqualTo(0));
+                    assertThat(cluster1.getSuccessfulShards(), equalTo(cluster1.getSuccessfulShards()));
+                    assertThat(cluster1.getSkippedShards(), equalTo(0));
+                    assertThat(cluster1.getFailedShards(), equalTo(0));
+
+                    EsqlExecutionInfo.Cluster cluster2 = executionInfo.getCluster(REMOTE_CLUSTER_2);
+                    assertThat(cluster2.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SKIPPED));
+                    assertThat(cluster2.getTotalShards(), equalTo(0));
+                    assertThat(cluster2.getSuccessfulShards(), equalTo(0));
+                    assertThat(cluster2.getSkippedShards(), equalTo(0));
+                    assertThat(cluster2.getFailedShards(), equalTo(0));
+
+                    EsqlExecutionInfo.Cluster localCluster = executionInfo.getCluster(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY);
+                    assertThat(localCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL));
+                    assertThat(localCluster.getTotalShards(), greaterThan(0));
+                    assertThat(localCluster.getSuccessfulShards(), equalTo(localCluster.getTotalShards()));
+                    assertThat(localCluster.getSkippedShards(), equalTo(0));
+                    assertThat(localCluster.getFailedShards(), equalTo(0));
+                }
+            }
+
+            // close remote-cluster-1 so that it is also unavailable
+            cluster(REMOTE_CLUSTER_1).close();
+
+            for (var hostMode : Enrich.Mode.values()) {
+                String query = String.format(Locale.ROOT, """
+                    FROM *:events,events
+                    | eval ip= TO_STR(host)
+                    | %s
+                    | %s
+                    | stats c = COUNT(*) by vendor
+                    | sort vendor
+                    """, enrichHosts(hostMode), enrichVendors(Enrich.Mode.COORDINATOR));
+                if (skipUnavailableRemote1 == false) {
+                    Exception exception = expectThrows(Exception.class, () -> runQuery(query, requestIncludeMeta));
+                    assertTrue(ExceptionsHelper.isRemoteUnavailableException(exception));
+                } else {
+                    try (EsqlQueryResponse resp = runQuery(query, requestIncludeMeta)) {
+                        assertThat(getValuesList(resp).size(), greaterThanOrEqualTo(1));
+                        EsqlExecutionInfo executionInfo = resp.getExecutionInfo();
+                        assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta));
+                        assertThat(executionInfo.clusterAliases(), equalTo(Set.of("", REMOTE_CLUSTER_1, REMOTE_CLUSTER_2)));
+                        assertCCSExecutionInfoDetails(executionInfo);
+
+                        EsqlExecutionInfo.Cluster cluster1 = executionInfo.getCluster(REMOTE_CLUSTER_1);
+                        assertThat(cluster1.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SKIPPED));
+                        assertThat(cluster1.getTotalShards(), equalTo(0));
+                        assertThat(cluster1.getSuccessfulShards(), equalTo(0));
+                        assertThat(cluster1.getSkippedShards(), equalTo(0));
+                        assertThat(cluster1.getFailedShards(), equalTo(0));
+                        assertThat(cluster1.getTook().millis(), greaterThanOrEqualTo(0L));
+
+                        EsqlExecutionInfo.Cluster cluster2 = executionInfo.getCluster(REMOTE_CLUSTER_2);
+                        assertThat(cluster2.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SKIPPED));
+                        assertThat(cluster2.getTotalShards(), equalTo(0));
+                        assertThat(cluster2.getSuccessfulShards(), equalTo(0));
+                        assertThat(cluster2.getSkippedShards(), equalTo(0));
+                        assertThat(cluster2.getFailedShards(), equalTo(0));
+
+                        EsqlExecutionInfo.Cluster localCluster = executionInfo.getCluster(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY);
+                        assertThat(localCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL));
+                        assertThat(localCluster.getTotalShards(), greaterThan(0));
+                        assertThat(localCluster.getSuccessfulShards(), equalTo(localCluster.getTotalShards()));
+                        assertThat(localCluster.getSkippedShards(), equalTo(0));
+                        assertThat(localCluster.getFailedShards(), equalTo(0));
+                    }
+                }
+            }
+        } finally {
+            clearSkipUnavailable();
+        }
+    }
+
+    public void testEnrichCoordinatorThenAnyWithSingleUnavailableRemoteAndLocal() throws IOException {
+        Tuple<Boolean, Boolean> includeCCSMetadata = CrossClustersEnrichIT.randomIncludeCCSMetadata();
+        Boolean requestIncludeMeta = includeCCSMetadata.v1();
+        boolean responseExpectMeta = includeCCSMetadata.v2();
+
+        boolean skipUnavailableRemote1 = randomBoolean();
+        setSkipUnavailable(REMOTE_CLUSTER_1, skipUnavailableRemote1);
+
+        try {
+            // close remote-cluster-1 so that it is unavailable
+            cluster(REMOTE_CLUSTER_1).close();
+            String query = String.format(Locale.ROOT, """
+                FROM %s:events,events
+                | eval ip= TO_STR(host)
+                | %s
+                | %s
+                | stats c = COUNT(*) by vendor
+                | sort vendor
+                """, REMOTE_CLUSTER_1, enrichHosts(Enrich.Mode.COORDINATOR), enrichVendors(Enrich.Mode.ANY));
+            if (skipUnavailableRemote1 == false) {
+                Exception exception = expectThrows(Exception.class, () -> runQuery(query, requestIncludeMeta));
+                assertTrue(ExceptionsHelper.isRemoteUnavailableException(exception));
+            } else {
+                try (EsqlQueryResponse resp = runQuery(query, requestIncludeMeta)) {
+                    assertThat(getValuesList(resp).size(), greaterThanOrEqualTo(1));
+                    EsqlExecutionInfo executionInfo = resp.getExecutionInfo();
+                    assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta));
+                    assertThat(
+                        executionInfo.clusterAliases(),
+                        equalTo(Set.of(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY, REMOTE_CLUSTER_1))
+                    );
+                    assertCCSExecutionInfoDetails(executionInfo);
+
+                    EsqlExecutionInfo.Cluster cluster1 = executionInfo.getCluster(REMOTE_CLUSTER_1);
+                    assertThat(cluster1.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SKIPPED));
+                    assertThat(cluster1.getTotalShards(), equalTo(0));
+                    assertThat(cluster1.getSuccessfulShards(), equalTo(0));
+                    assertThat(cluster1.getSkippedShards(), equalTo(0));
+                    assertThat(cluster1.getFailedShards(), equalTo(0));
+
+                    EsqlExecutionInfo.Cluster localCluster = executionInfo.getCluster(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY);
+                    assertThat(localCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL));
+                    assertThat(localCluster.getTotalShards(), greaterThan(0));
+                    assertThat(localCluster.getSuccessfulShards(), equalTo(localCluster.getTotalShards()));
+                    assertThat(localCluster.getSkippedShards(), equalTo(0));
+                    assertThat(localCluster.getFailedShards(), equalTo(0));
+                }
+            }
+        } finally {
+            clearSkipUnavailable();
+        }
+    }
+
+    public void testEnrichCoordinatorThenAnyWithSingleUnavailableRemoteAndNotLocal() throws IOException {
+        Tuple<Boolean, Boolean> includeCCSMetadata = CrossClustersEnrichIT.randomIncludeCCSMetadata();
+        Boolean requestIncludeMeta = includeCCSMetadata.v1();
+        boolean responseExpectMeta = includeCCSMetadata.v2();
+
+        boolean skipUnavailableRemote1 = randomBoolean();
+        setSkipUnavailable(REMOTE_CLUSTER_1, skipUnavailableRemote1);
+
+        try {
+            // close remote-cluster-1 so that it is unavailable
+            cluster(REMOTE_CLUSTER_1).close();
+            String query = String.format(Locale.ROOT, """
+                FROM %s:events
+                | eval ip= TO_STR(host)
+                | %s
+                | %s
+                | stats c = COUNT(*) by vendor
+                | sort vendor
+                """, REMOTE_CLUSTER_1, enrichHosts(Enrich.Mode.COORDINATOR), enrichVendors(Enrich.Mode.ANY));
+            if (skipUnavailableRemote1 == false) {
+                Exception exception = expectThrows(Exception.class, () -> runQuery(query, requestIncludeMeta));
+                assertTrue(ExceptionsHelper.isRemoteUnavailableException(exception));
+            } else {
+                try (EsqlQueryResponse resp = runQuery(query, requestIncludeMeta)) {
+                    List<ColumnInfoImpl> columns = resp.columns();
+                    assertThat(columns.size(), equalTo(1));
+                    // column from an empty result should be {"name":"<no-fields>","type":"null"}
+                    assertThat(columns.get(0).name(), equalTo("<no-fields>"));
+                    assertThat(columns.get(0).type(), equalTo(DataType.NULL));
+
+                    assertThat(getValuesList(resp).size(), equalTo(0));
+                    EsqlExecutionInfo executionInfo = resp.getExecutionInfo();
+                    assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta));
+                    assertThat(executionInfo.clusterAliases(), equalTo(Set.of(REMOTE_CLUSTER_1)));
+                    assertCCSExecutionInfoDetails(executionInfo);
+
+                    EsqlExecutionInfo.Cluster cluster1 = executionInfo.getCluster(REMOTE_CLUSTER_1);
+                    assertThat(cluster1.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SKIPPED));
+                    assertThat(cluster1.getTotalShards(), equalTo(0));
+                    assertThat(cluster1.getSuccessfulShards(), equalTo(0));
+                    assertThat(cluster1.getSkippedShards(), equalTo(0));
+                    assertThat(cluster1.getFailedShards(), equalTo(0));
+                }
+            }
+        } finally {
+            clearSkipUnavailable();
+        }
+    }
+
+    public void testEnrichRemoteWithVendor() throws IOException {
+        Tuple<Boolean, Boolean> includeCCSMetadata = CrossClustersEnrichIT.randomIncludeCCSMetadata();
+        Boolean requestIncludeMeta = includeCCSMetadata.v1();
+        boolean responseExpectMeta = includeCCSMetadata.v2();
+
+        boolean skipUnavailableRemote2 = randomBoolean();
+        setSkipUnavailable(REMOTE_CLUSTER_1, true);
+        setSkipUnavailable(REMOTE_CLUSTER_2, skipUnavailableRemote2);
+
+        try {
+            // close remote-cluster-1 so that it is unavailable
+            cluster(REMOTE_CLUSTER_1).close();
+
+            for (Enrich.Mode hostMode : List.of(Enrich.Mode.ANY, Enrich.Mode.REMOTE)) {
+                var query = String.format(Locale.ROOT, """
+                    FROM *:events,events
+                    | eval ip= TO_STR(host)
+                    | %s
+                    | %s
+                    | stats c = COUNT(*) by vendor
+                    | sort vendor
+                    """, enrichHosts(hostMode), enrichVendors(Enrich.Mode.REMOTE));
+                try (EsqlQueryResponse resp = runQuery(query, requestIncludeMeta)) {
+                    assertThat(getValuesList(resp).size(), greaterThan(0));
+                    EsqlExecutionInfo executionInfo = resp.getExecutionInfo();
+                    assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta));
+                    assertThat(
+                        executionInfo.clusterAliases(),
+                        equalTo(Set.of(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY, REMOTE_CLUSTER_1, REMOTE_CLUSTER_2))
+                    );
+                    assertCCSExecutionInfoDetails(executionInfo);
+
+                    EsqlExecutionInfo.Cluster cluster1 = executionInfo.getCluster(REMOTE_CLUSTER_1);
+                    assertThat(cluster1.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SKIPPED));
+                    assertThat(cluster1.getTotalShards(), equalTo(0));
+                    assertThat(cluster1.getSuccessfulShards(), equalTo(0));
+                    assertThat(cluster1.getSkippedShards(), equalTo(0));
+                    assertThat(cluster1.getFailedShards(), equalTo(0));
+                    assertThat(cluster1.getTook().millis(), greaterThanOrEqualTo(0L));
+
+                    EsqlExecutionInfo.Cluster cluster2 = executionInfo.getCluster(REMOTE_CLUSTER_2);
+                    assertThat(cluster2.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL));
+                    assertThat(cluster2.getTotalShards(), greaterThan(0));
+                    assertThat(cluster2.getSuccessfulShards(), equalTo(cluster2.getSuccessfulShards()));
+                    assertThat(cluster2.getSkippedShards(), equalTo(0));
+                    assertThat(cluster2.getFailedShards(), equalTo(0));
+
+                    EsqlExecutionInfo.Cluster localCluster = executionInfo.getCluster(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY);
+                    assertThat(localCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL));
+                    assertThat(localCluster.getTotalShards(), greaterThan(0));
+                    assertThat(localCluster.getSuccessfulShards(), equalTo(localCluster.getTotalShards()));
+                    assertThat(localCluster.getSkippedShards(), equalTo(0));
+                    assertThat(localCluster.getFailedShards(), equalTo(0));
+                }
+            }
+
+            // close remote-cluster-2 so that it is also unavailable
+            cluster(REMOTE_CLUSTER_2).close();
+
+            for (Enrich.Mode hostMode : List.of(Enrich.Mode.ANY, Enrich.Mode.REMOTE)) {
+                var query = String.format(Locale.ROOT, """
+                    FROM *:events,events
+                    | eval ip= TO_STR(host)
+                    | %s
+                    | %s
+                    | stats c = COUNT(*) by vendor
+                    | sort vendor
+                    """, enrichHosts(hostMode), enrichVendors(Enrich.Mode.REMOTE));
+                if (skipUnavailableRemote2 == false) {
+                    Exception exception = expectThrows(Exception.class, () -> runQuery(query, requestIncludeMeta));
+                    assertTrue(ExceptionsHelper.isRemoteUnavailableException(exception));
+                } else {
+
+                    try (EsqlQueryResponse resp = runQuery(query, requestIncludeMeta)) {
+                        assertThat(getValuesList(resp).size(), greaterThan(0));
+                        EsqlExecutionInfo executionInfo = resp.getExecutionInfo();
+                        assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta));
+                        assertThat(
+                            executionInfo.clusterAliases(),
+                            equalTo(Set.of(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY, REMOTE_CLUSTER_1, REMOTE_CLUSTER_2))
+                        );
+                        assertCCSExecutionInfoDetails(executionInfo);
+
+                        EsqlExecutionInfo.Cluster cluster1 = executionInfo.getCluster(REMOTE_CLUSTER_1);
+                        assertThat(cluster1.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SKIPPED));
+                        assertThat(cluster1.getTotalShards(), equalTo(0));
+                        assertThat(cluster1.getSuccessfulShards(), equalTo(0));
+                        assertThat(cluster1.getSkippedShards(), equalTo(0));
+                        assertThat(cluster1.getFailedShards(), equalTo(0));
+                        assertThat(cluster1.getTook().millis(), greaterThanOrEqualTo(0L));
+
+                        EsqlExecutionInfo.Cluster cluster2 = executionInfo.getCluster(REMOTE_CLUSTER_2);
+                        assertThat(cluster2.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SKIPPED));
+                        assertThat(cluster2.getTotalShards(), equalTo(0));
+                        assertThat(cluster2.getSuccessfulShards(), equalTo(0));
+                        assertThat(cluster2.getSkippedShards(), equalTo(0));
+                        assertThat(cluster2.getFailedShards(), equalTo(0));
+
+                        EsqlExecutionInfo.Cluster localCluster = executionInfo.getCluster(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY);
+                        assertThat(localCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL));
+                        assertThat(localCluster.getTotalShards(), greaterThan(0));
+                        assertThat(localCluster.getSuccessfulShards(), equalTo(localCluster.getTotalShards()));
+                        assertThat(localCluster.getSkippedShards(), equalTo(0));
+                        assertThat(localCluster.getFailedShards(), equalTo(0));
+                    }
+                }
+            }
+        } finally {
+            clearSkipUnavailable();
+        }
+    }
+
+    protected EsqlQueryResponse runQuery(String query, Boolean ccsMetadataInResponse) {
+        EsqlQueryRequest request = EsqlQueryRequest.syncEsqlQueryRequest();
+        request.query(query);
+        request.pragmas(AbstractEsqlIntegTestCase.randomPragmas());
+        if (randomBoolean()) {
+            request.profile(true);
+        }
+        if (ccsMetadataInResponse != null) {
+            request.includeCCSMetadata(ccsMetadataInResponse);
+        }
+        return client(LOCAL_CLUSTER).execute(EsqlQueryAction.INSTANCE, request).actionGet(30, TimeUnit.SECONDS);
+    }
+
+    private static void assertCCSExecutionInfoDetails(EsqlExecutionInfo executionInfo) {
+        assertThat(executionInfo.overallTook().millis(), greaterThanOrEqualTo(0L));
+        assertTrue(executionInfo.isCrossClusterSearch());
+
+        for (String clusterAlias : executionInfo.clusterAliases()) {
+            EsqlExecutionInfo.Cluster cluster = executionInfo.getCluster(clusterAlias);
+            assertThat(cluster.getTook().millis(), greaterThanOrEqualTo(0L));
+            assertThat(cluster.getTook().millis(), lessThanOrEqualTo(executionInfo.overallTook().millis()));
+        }
+    }
+
+    private void setSkipUnavailable(String clusterAlias, boolean skip) {
+        client(LOCAL_CLUSTER).admin()
+            .cluster()
+            .prepareUpdateSettings(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT)
+            .setPersistentSettings(Settings.builder().put("cluster.remote." + clusterAlias + ".skip_unavailable", skip).build())
+            .get();
+    }
+
+    private void clearSkipUnavailable() {
+        Settings.Builder settingsBuilder = Settings.builder()
+            .putNull("cluster.remote." + REMOTE_CLUSTER_1 + ".skip_unavailable")
+            .putNull("cluster.remote." + REMOTE_CLUSTER_2 + ".skip_unavailable");
+        client(LOCAL_CLUSTER).admin()
+            .cluster()
+            .prepareUpdateSettings(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT)
+            .setPersistentSettings(settingsBuilder.build())
+            .get();
+    }
+}
diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterQueryUnavailableRemotesIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterQueryUnavailableRemotesIT.java
new file mode 100644
index 0000000000000..0f1aa8541fdd9
--- /dev/null
+++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterQueryUnavailableRemotesIT.java
@@ -0,0 +1,525 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.action;
+
+import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.client.internal.Client;
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.compute.operator.exchange.ExchangeService;
+import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.core.Tuple;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.test.AbstractMultiClustersTestCase;
+import org.elasticsearch.test.XContentTestUtils;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.plugin.EsqlPlugin;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.elasticsearch.xpack.esql.EsqlTestUtils.getValuesList;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.lessThanOrEqualTo;
+
+public class CrossClusterQueryUnavailableRemotesIT extends AbstractMultiClustersTestCase {
+    private static final String REMOTE_CLUSTER_1 = "cluster-a";
+    private static final String REMOTE_CLUSTER_2 = "cluster-b";
+
+    @Override
+    protected Collection<String> remoteClusterAlias() {
+        return List.of(REMOTE_CLUSTER_1, REMOTE_CLUSTER_2);
+    }
+
+    @Override
+    protected boolean reuseClusters() {
+        return false;
+    }
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins(String clusterAlias) {
+        List<Class<? extends Plugin>> plugins = new ArrayList<>(super.nodePlugins(clusterAlias));
+        plugins.add(EsqlPlugin.class);
+        plugins.add(org.elasticsearch.xpack.esql.action.CrossClustersQueryIT.InternalExchangePlugin.class);
+        return plugins;
+    }
+
+    public static class InternalExchangePlugin extends Plugin {
+        @Override
+        public List<Setting<?>> getSettings() {
+            return List.of(
+                Setting.timeSetting(
+                    ExchangeService.INACTIVE_SINKS_INTERVAL_SETTING,
+                    TimeValue.timeValueSeconds(30),
+                    Setting.Property.NodeScope
+                )
+            );
+        }
+    }
+
+    public void testCCSAgainstDisconnectedRemoteWithSkipUnavailableTrue() throws Exception {
+        int numClusters = 3;
+        Map<String, Object> testClusterInfo = setupClusters(numClusters);
+        int localNumShards = (Integer) testClusterInfo.get("local.num_shards");
+        int remote2NumShards = (Integer) testClusterInfo.get("remote2.num_shards");
+        setSkipUnavailable(REMOTE_CLUSTER_1, true);
+        setSkipUnavailable(REMOTE_CLUSTER_2, true);
+
+        Tuple<Boolean, Boolean> includeCCSMetadata = randomIncludeCCSMetadata();
+        Boolean requestIncludeMeta = includeCCSMetadata.v1();
+        boolean responseExpectMeta = includeCCSMetadata.v2();
+
+        try {
+            // close remote-cluster-1 so that it is unavailable
+            cluster(REMOTE_CLUSTER_1).close();
+
+            try (EsqlQueryResponse resp = runQuery("FROM logs-*,*:logs-* | STATS sum (v)", requestIncludeMeta)) {
+                List<List<Object>> values = getValuesList(resp);
+                assertThat(values, hasSize(1));
+                assertThat(values.get(0), equalTo(List.of(330L)));
+
+                EsqlExecutionInfo executionInfo = resp.getExecutionInfo();
+                assertNotNull(executionInfo);
+                assertThat(executionInfo.isCrossClusterSearch(), is(true));
+                long overallTookMillis = executionInfo.overallTook().millis();
+                assertThat(overallTookMillis, greaterThanOrEqualTo(0L));
+                assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta));
+
+                assertThat(executionInfo.clusterAliases(), equalTo(Set.of(REMOTE_CLUSTER_1, REMOTE_CLUSTER_2, LOCAL_CLUSTER)));
+
+                EsqlExecutionInfo.Cluster remote1Cluster = executionInfo.getCluster(REMOTE_CLUSTER_1);
+                assertThat(remote1Cluster.getIndexExpression(), equalTo("logs-*"));
+                assertThat(remote1Cluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SKIPPED));
+                assertThat(remote1Cluster.getTook().millis(), greaterThanOrEqualTo(0L));
+                assertThat(remote1Cluster.getTook().millis(), lessThanOrEqualTo(overallTookMillis));
+                assertThat(remote1Cluster.getTotalShards(), equalTo(0));
+                assertThat(remote1Cluster.getSuccessfulShards(), equalTo(0));
+                assertThat(remote1Cluster.getSkippedShards(), equalTo(0));
+                assertThat(remote1Cluster.getFailedShards(), equalTo(0));
+
+                EsqlExecutionInfo.Cluster remote2Cluster = executionInfo.getCluster(REMOTE_CLUSTER_2);
+                assertThat(remote2Cluster.getIndexExpression(), equalTo("logs-*"));
+                assertThat(remote2Cluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL));
+                assertThat(remote2Cluster.getTook().millis(), greaterThanOrEqualTo(0L));
+                assertThat(remote2Cluster.getTook().millis(), lessThanOrEqualTo(overallTookMillis));
+                assertThat(remote2Cluster.getTotalShards(), equalTo(remote2NumShards));
+                assertThat(remote2Cluster.getSuccessfulShards(), equalTo(remote2NumShards));
+                assertThat(remote2Cluster.getSkippedShards(), equalTo(0));
+                assertThat(remote2Cluster.getFailedShards(), equalTo(0));
+
+                EsqlExecutionInfo.Cluster localCluster = executionInfo.getCluster(LOCAL_CLUSTER);
+                assertThat(localCluster.getIndexExpression(), equalTo("logs-*"));
+                assertThat(localCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL));
+                assertThat(localCluster.getTook().millis(), greaterThanOrEqualTo(0L));
+                assertThat(localCluster.getTook().millis(), lessThanOrEqualTo(overallTookMillis));
+                assertThat(localCluster.getTotalShards(), equalTo(localNumShards));
+                assertThat(localCluster.getSuccessfulShards(), equalTo(localNumShards));
+                assertThat(localCluster.getSkippedShards(), equalTo(0));
+                assertThat(localCluster.getFailedShards(), equalTo(0));
+
+                // ensure that the _clusters metadata is present only if requested
+                assertClusterMetadataInResponse(resp, responseExpectMeta);
+            }
+
+            // scenario where there are no indices to match because
+            // 1) the local cluster indexExpression and REMOTE_CLUSTER_2 indexExpression match no indices
+            // 2) the REMOTE_CLUSTER_1 is unavailable
+            // 3) both remotes are marked as skip_un=true
+            String query = "FROM nomatch*," + REMOTE_CLUSTER_1 + ":logs-*," + REMOTE_CLUSTER_2 + ":nomatch* | STATS sum (v)";
+            try (EsqlQueryResponse resp = runQuery(query, requestIncludeMeta)) {
+                List<List<Object>> values = getValuesList(resp);
+                assertThat(values, hasSize(0));
+
+                EsqlExecutionInfo executionInfo = resp.getExecutionInfo();
+                assertNotNull(executionInfo);
+                assertThat(executionInfo.isCrossClusterSearch(), is(true));
+                long overallTookMillis = executionInfo.overallTook().millis();
+                assertThat(overallTookMillis, greaterThanOrEqualTo(0L));
+                assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta));
+
+                assertThat(executionInfo.clusterAliases(), equalTo(Set.of(REMOTE_CLUSTER_1, REMOTE_CLUSTER_2, LOCAL_CLUSTER)));
+
+                EsqlExecutionInfo.Cluster remote1Cluster = executionInfo.getCluster(REMOTE_CLUSTER_1);
+                assertThat(remote1Cluster.getIndexExpression(), equalTo("logs-*"));
+                assertThat(remote1Cluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SKIPPED));
+                assertThat(remote1Cluster.getTook().millis(), greaterThanOrEqualTo(0L));
+                assertThat(remote1Cluster.getTook().millis(), lessThanOrEqualTo(overallTookMillis));
+                assertThat(remote1Cluster.getTotalShards(), equalTo(0));
+                assertThat(remote1Cluster.getSuccessfulShards(), equalTo(0));
+                assertThat(remote1Cluster.getSkippedShards(), equalTo(0));
+                assertThat(remote1Cluster.getFailedShards(), equalTo(0));
+
+                EsqlExecutionInfo.Cluster remote2Cluster = executionInfo.getCluster(REMOTE_CLUSTER_2);
+                assertThat(remote2Cluster.getIndexExpression(), equalTo("nomatch*"));
+                assertThat(remote2Cluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SKIPPED));
+                assertThat(remote2Cluster.getTook().millis(), greaterThanOrEqualTo(0L));
+                assertThat(remote2Cluster.getTook().millis(), lessThanOrEqualTo(overallTookMillis));
+                assertThat(remote2Cluster.getTotalShards(), equalTo(0));
+                assertThat(remote2Cluster.getSuccessfulShards(), equalTo(0));
+                assertThat(remote2Cluster.getSkippedShards(), equalTo(0));
+                assertThat(remote2Cluster.getFailedShards(), equalTo(0));
+
+                EsqlExecutionInfo.Cluster localCluster = executionInfo.getCluster(LOCAL_CLUSTER);
+                assertThat(localCluster.getIndexExpression(), equalTo("nomatch*"));
+                // local cluster should never be marked as SKIPPED
+                assertThat(localCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL));
+                assertThat(localCluster.getTook().millis(), greaterThanOrEqualTo(0L));
+                assertThat(localCluster.getTook().millis(), lessThanOrEqualTo(overallTookMillis));
+                assertThat(localCluster.getTotalShards(), equalTo(0));
+                assertThat(localCluster.getSuccessfulShards(), equalTo(0));
+                assertThat(localCluster.getSkippedShards(), equalTo(0));
+                assertThat(localCluster.getFailedShards(), equalTo(0));
+
+                // ensure that the _clusters metadata is present only if requested
+                assertClusterMetadataInResponse(resp, responseExpectMeta);
+            }
+
+            // close remote-cluster-2 so that it is also unavailable
+            cluster(REMOTE_CLUSTER_2).close();
+
+            try (EsqlQueryResponse resp = runQuery("FROM logs-*,*:logs-* | STATS sum (v)", requestIncludeMeta)) {
+                List<List<Object>> values = getValuesList(resp);
+                assertThat(values, hasSize(1));
+                assertThat(values.get(0), equalTo(List.of(45L)));
+
+                EsqlExecutionInfo executionInfo = resp.getExecutionInfo();
+                assertNotNull(executionInfo);
+                assertThat(executionInfo.isCrossClusterSearch(), is(true));
+                long overallTookMillis = executionInfo.overallTook().millis();
+                assertThat(overallTookMillis, greaterThanOrEqualTo(0L));
+                assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta));
+
+                assertThat(executionInfo.clusterAliases(), equalTo(Set.of(REMOTE_CLUSTER_1, REMOTE_CLUSTER_2, LOCAL_CLUSTER)));
+
+                EsqlExecutionInfo.Cluster remote1Cluster = executionInfo.getCluster(REMOTE_CLUSTER_1);
+                assertThat(remote1Cluster.getIndexExpression(), equalTo("logs-*"));
+                assertThat(remote1Cluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SKIPPED));
+                assertThat(remote1Cluster.getTook().millis(), greaterThanOrEqualTo(0L));
+                assertThat(remote1Cluster.getTook().millis(), lessThanOrEqualTo(overallTookMillis));
+                assertThat(remote1Cluster.getTotalShards(), equalTo(0));
+                assertThat(remote1Cluster.getSuccessfulShards(), equalTo(0));
+                assertThat(remote1Cluster.getSkippedShards(), equalTo(0));
+                assertThat(remote1Cluster.getFailedShards(), equalTo(0));
+
+                EsqlExecutionInfo.Cluster remote2Cluster = executionInfo.getCluster(REMOTE_CLUSTER_2);
+                assertThat(remote2Cluster.getIndexExpression(), equalTo("logs-*"));
+                assertThat(remote2Cluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SKIPPED));
+                assertThat(remote2Cluster.getTook().millis(), greaterThanOrEqualTo(0L));
+                assertThat(remote2Cluster.getTook().millis(), lessThanOrEqualTo(overallTookMillis));
+                assertThat(remote2Cluster.getTotalShards(), equalTo(0));
+                assertThat(remote2Cluster.getSuccessfulShards(), equalTo(0));
+                assertThat(remote2Cluster.getSkippedShards(), equalTo(0));
+                assertThat(remote2Cluster.getFailedShards(), equalTo(0));
+
+                EsqlExecutionInfo.Cluster localCluster = executionInfo.getCluster(LOCAL_CLUSTER);
+                assertThat(localCluster.getIndexExpression(), equalTo("logs-*"));
+                assertThat(localCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL));
+                assertThat(localCluster.getTook().millis(), greaterThanOrEqualTo(0L));
+                assertThat(localCluster.getTook().millis(), lessThanOrEqualTo(overallTookMillis));
+                assertThat(localCluster.getTotalShards(), equalTo(localNumShards));
+                assertThat(localCluster.getSuccessfulShards(), equalTo(localNumShards));
+                assertThat(localCluster.getSkippedShards(), equalTo(0));
+                assertThat(localCluster.getFailedShards(), equalTo(0));
+
+                // ensure that the _clusters metadata is present only if requested
+                assertClusterMetadataInResponse(resp, responseExpectMeta);
+            }
+        } finally {
+            clearSkipUnavailable(numClusters);
+        }
+    }
+
+    public void testRemoteOnlyCCSAgainstDisconnectedRemoteWithSkipUnavailableTrue() throws Exception {
+        int numClusters = 3;
+        setupClusters(numClusters);
+        setSkipUnavailable(REMOTE_CLUSTER_1, true);
+        setSkipUnavailable(REMOTE_CLUSTER_2, true);
+
+        try {
+            // close remote cluster 1 so that it is unavailable
+            cluster(REMOTE_CLUSTER_1).close();
+
+            Tuple<Boolean, Boolean> includeCCSMetadata = randomIncludeCCSMetadata();
+            Boolean requestIncludeMeta = includeCCSMetadata.v1();
+            boolean responseExpectMeta = includeCCSMetadata.v2();
+
+            // query only the REMOTE_CLUSTER_1
+            try (EsqlQueryResponse resp = runQuery("FROM " + REMOTE_CLUSTER_1 + ":logs-* | STATS sum (v)", requestIncludeMeta)) {
+                List<ColumnInfoImpl> columns = resp.columns();
+                assertThat(columns.size(), equalTo(1));
+                // column from an empty result should be {"name":"<no-fields>","type":"null"}
+                assertThat(columns.get(0).name(), equalTo("<no-fields>"));
+                assertThat(columns.get(0).type(), equalTo(DataType.NULL));
+
+                List<List<Object>> values = getValuesList(resp);
+                assertThat(values, hasSize(0));
+
+                EsqlExecutionInfo executionInfo = resp.getExecutionInfo();
+                assertNotNull(executionInfo);
+                assertThat(executionInfo.isCrossClusterSearch(), is(true));
+                long overallTookMillis = executionInfo.overallTook().millis();
+                assertThat(overallTookMillis, greaterThanOrEqualTo(0L));
+                assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta));
+
+                assertThat(executionInfo.clusterAliases(), equalTo(Set.of(REMOTE_CLUSTER_1)));
+
+                EsqlExecutionInfo.Cluster remoteCluster = executionInfo.getCluster(REMOTE_CLUSTER_1);
+                assertThat(remoteCluster.getIndexExpression(), equalTo("logs-*"));
+                assertThat(remoteCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SKIPPED));
+                assertThat(remoteCluster.getTook().millis(), greaterThanOrEqualTo(0L));
+                assertThat(remoteCluster.getTook().millis(), lessThanOrEqualTo(overallTookMillis));
+                assertThat(remoteCluster.getTotalShards(), equalTo(0));
+                assertThat(remoteCluster.getSuccessfulShards(), equalTo(0));
+                assertThat(remoteCluster.getSkippedShards(), equalTo(0));
+                assertThat(remoteCluster.getFailedShards(), equalTo(0));
+
+                // ensure that the _clusters metadata is present only if requested
+                assertClusterMetadataInResponse(resp, responseExpectMeta);
+            }
+
+            // close remote cluster 2 so that it is also unavailable
+            cluster(REMOTE_CLUSTER_2).close();
+
+            // query only the both remote clusters
+            try (
+                EsqlQueryResponse resp = runQuery(
+                    "FROM " + REMOTE_CLUSTER_1 + ":logs-*," + REMOTE_CLUSTER_2 + ":logs-* | STATS sum (v)",
+                    requestIncludeMeta
+                )
+            ) {
+                List<ColumnInfoImpl> columns = resp.columns();
+                assertThat(columns.size(), equalTo(1));
+                // column from an empty result should be {"name":"<no-fields>","type":"null"}
+                assertThat(columns.get(0).name(), equalTo("<no-fields>"));
+                assertThat(columns.get(0).type(), equalTo(DataType.NULL));
+
+                List<List<Object>> values = getValuesList(resp);
+                assertThat(values, hasSize(0));
+
+                EsqlExecutionInfo executionInfo = resp.getExecutionInfo();
+                assertNotNull(executionInfo);
+                assertThat(executionInfo.isCrossClusterSearch(), is(true));
+                long overallTookMillis = executionInfo.overallTook().millis();
+                assertThat(overallTookMillis, greaterThanOrEqualTo(0L));
+                assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta));
+
+                assertThat(executionInfo.clusterAliases(), equalTo(Set.of(REMOTE_CLUSTER_1, REMOTE_CLUSTER_2)));
+
+                EsqlExecutionInfo.Cluster remote1Cluster = executionInfo.getCluster(REMOTE_CLUSTER_1);
+                assertThat(remote1Cluster.getIndexExpression(), equalTo("logs-*"));
+                assertThat(remote1Cluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SKIPPED));
+                assertThat(remote1Cluster.getTook().millis(), greaterThanOrEqualTo(0L));
+                assertThat(remote1Cluster.getTook().millis(), lessThanOrEqualTo(overallTookMillis));
+                assertThat(remote1Cluster.getTotalShards(), equalTo(0));
+                assertThat(remote1Cluster.getSuccessfulShards(), equalTo(0));
+                assertThat(remote1Cluster.getSkippedShards(), equalTo(0));
+                assertThat(remote1Cluster.getFailedShards(), equalTo(0));
+
+                EsqlExecutionInfo.Cluster remote2Cluster = executionInfo.getCluster(REMOTE_CLUSTER_2);
+                assertThat(remote2Cluster.getIndexExpression(), equalTo("logs-*"));
+                assertThat(remote2Cluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SKIPPED));
+                assertThat(remote2Cluster.getTook().millis(), greaterThanOrEqualTo(0L));
+                assertThat(remote2Cluster.getTook().millis(), lessThanOrEqualTo(overallTookMillis));
+                assertThat(remote2Cluster.getTotalShards(), equalTo(0));
+                assertThat(remote2Cluster.getSuccessfulShards(), equalTo(0));
+                assertThat(remote2Cluster.getSkippedShards(), equalTo(0));
+                assertThat(remote2Cluster.getFailedShards(), equalTo(0));
+
+                // ensure that the _clusters metadata is present only if requested
+                assertClusterMetadataInResponse(resp, responseExpectMeta);
+            }
+
+        } finally {
+            clearSkipUnavailable(numClusters);
+        }
+    }
+
+    public void testCCSAgainstDisconnectedRemoteWithSkipUnavailableFalse() throws Exception {
+        int numClusters = 2;
+        setupClusters(numClusters);
+        setSkipUnavailable(REMOTE_CLUSTER_1, false);
+
+        try {
+            // close the remote cluster so that it is unavailable
+            cluster(REMOTE_CLUSTER_1).close();
+
+            Tuple<Boolean, Boolean> includeCCSMetadata = randomIncludeCCSMetadata();
+            Boolean requestIncludeMeta = includeCCSMetadata.v1();
+
+            final Exception exception = expectThrows(
+                Exception.class,
+                () -> runQuery("FROM logs-*,*:logs-* | STATS sum (v)", requestIncludeMeta)
+            );
+            assertThat(ExceptionsHelper.isRemoteUnavailableException(exception), is(true));
+        } finally {
+            clearSkipUnavailable(numClusters);
+        }
+    }
+
+    public void testRemoteOnlyCCSAgainstDisconnectedRemoteWithSkipUnavailableFalse() throws Exception {
+        int numClusters = 3;
+        setupClusters(numClusters);
+        setSkipUnavailable(REMOTE_CLUSTER_1, false);
+        setSkipUnavailable(REMOTE_CLUSTER_2, randomBoolean());
+
+        try {
+            Tuple<Boolean, Boolean> includeCCSMetadata = randomIncludeCCSMetadata();
+            Boolean requestIncludeMeta = includeCCSMetadata.v1();
+            {
+                // close the remote cluster so that it is unavailable
+                cluster(REMOTE_CLUSTER_1).close();
+                Exception exception = expectThrows(Exception.class, () -> runQuery("FROM *:logs-* | STATS sum (v)", requestIncludeMeta));
+                assertThat(ExceptionsHelper.isRemoteUnavailableException(exception), is(true));
+            }
+            {
+                // close remote cluster 2 so that it is unavailable
+                cluster(REMOTE_CLUSTER_2).close();
+                Exception exception = expectThrows(Exception.class, () -> runQuery("FROM *:logs-* | STATS sum (v)", requestIncludeMeta));
+                assertThat(ExceptionsHelper.isRemoteUnavailableException(exception), is(true));
+            }
+        } finally {
+            clearSkipUnavailable(numClusters);
+        }
+    }
+
+    private void setSkipUnavailable(String clusterAlias, boolean skip) {
+        client(LOCAL_CLUSTER).admin()
+            .cluster()
+            .prepareUpdateSettings(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT)
+            .setPersistentSettings(Settings.builder().put("cluster.remote." + clusterAlias + ".skip_unavailable", skip).build())
+            .get();
+    }
+
+    private void clearSkipUnavailable(int numClusters) {
+        assert numClusters == 2 || numClusters == 3 : "Only 2 or 3 clusters supported";
+        Settings.Builder settingsBuilder = Settings.builder().putNull("cluster.remote." + REMOTE_CLUSTER_1 + ".skip_unavailable");
+        if (numClusters == 3) {
+            settingsBuilder.putNull("cluster.remote." + REMOTE_CLUSTER_2 + ".skip_unavailable");
+        }
+        client(LOCAL_CLUSTER).admin()
+            .cluster()
+            .prepareUpdateSettings(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT)
+            .setPersistentSettings(settingsBuilder.build())
+            .get();
+    }
+
+    private static void assertClusterMetadataInResponse(EsqlQueryResponse resp, boolean responseExpectMeta) {
+        try {
+            final Map<String, Object> esqlResponseAsMap = XContentTestUtils.convertToMap(resp);
+            final Object clusters = esqlResponseAsMap.get("_clusters");
+            if (responseExpectMeta) {
+                assertNotNull(clusters);
+                // test a few entries to ensure it looks correct (other tests do a full analysis of the metadata in the response)
+                @SuppressWarnings("unchecked")
+                Map<String, Object> inner = (Map<String, Object>) clusters;
+                assertTrue(inner.containsKey("total"));
+                assertTrue(inner.containsKey("details"));
+            } else {
+                assertNull(clusters);
+            }
+        } catch (IOException e) {
+            fail("Could not convert ESQL response to Map: " + e);
+        }
+    }
+
+    protected EsqlQueryResponse runQuery(String query, Boolean ccsMetadataInResponse) {
+        EsqlQueryRequest request = EsqlQueryRequest.syncEsqlQueryRequest();
+        request.query(query);
+        request.pragmas(AbstractEsqlIntegTestCase.randomPragmas());
+        request.profile(randomInt(5) == 2);
+        request.columnar(randomBoolean());
+        if (ccsMetadataInResponse != null) {
+            request.includeCCSMetadata(ccsMetadataInResponse);
+        }
+        return runQuery(request);
+    }
+
+    protected EsqlQueryResponse runQuery(EsqlQueryRequest request) {
+        return client(LOCAL_CLUSTER).execute(EsqlQueryAction.INSTANCE, request).actionGet(30, TimeUnit.SECONDS);
+    }
+
+    /**
+     * v1: value to send to runQuery (can be null; null means use default value)
+     * v2: whether to expect CCS Metadata in the response (cannot be null)
+     * @return
+     */
+    public static Tuple<Boolean, Boolean> randomIncludeCCSMetadata() {
+        return switch (randomIntBetween(1, 3)) {
+            case 1 -> new Tuple<>(Boolean.TRUE, Boolean.TRUE);
+            case 2 -> new Tuple<>(Boolean.FALSE, Boolean.FALSE);
+            case 3 -> new Tuple<>(null, Boolean.FALSE);
+            default -> throw new AssertionError("should not get here");
+        };
+    }
+
+    Map<String, Object> setupClusters(int numClusters) {
+        assert numClusters == 2 || numClusters == 3 : "2 or 3 clusters supported not: " + numClusters;
+        String localIndex = "logs-1";
+        int numShardsLocal = randomIntBetween(1, 5);
+        populateLocalIndices(localIndex, numShardsLocal);
+
+        String remoteIndex = "logs-2";
+        int numShardsRemote = randomIntBetween(1, 5);
+        populateRemoteIndices(REMOTE_CLUSTER_1, remoteIndex, numShardsRemote);
+
+        Map<String, Object> clusterInfo = new HashMap<>();
+        clusterInfo.put("local.num_shards", numShardsLocal);
+        clusterInfo.put("local.index", localIndex);
+        clusterInfo.put("remote.num_shards", numShardsRemote);
+        clusterInfo.put("remote.index", remoteIndex);
+
+        if (numClusters == 3) {
+            int numShardsRemote2 = randomIntBetween(1, 5);
+            populateRemoteIndices(REMOTE_CLUSTER_2, remoteIndex, numShardsRemote2);
+            clusterInfo.put("remote2.index", remoteIndex);
+            clusterInfo.put("remote2.num_shards", numShardsRemote2);
+        }
+
+        return clusterInfo;
+    }
+
+    void populateLocalIndices(String indexName, int numShards) {
+        Client localClient = client(LOCAL_CLUSTER);
+        assertAcked(
+            localClient.admin()
+                .indices()
+                .prepareCreate(indexName)
+                .setSettings(Settings.builder().put("index.number_of_shards", numShards))
+                .setMapping("id", "type=keyword", "tag", "type=keyword", "v", "type=long")
+        );
+        for (int i = 0; i < 10; i++) {
+            localClient.prepareIndex(indexName).setSource("id", "local-" + i, "tag", "local", "v", i).get();
+        }
+        localClient.admin().indices().prepareRefresh(indexName).get();
+    }
+
+    void populateRemoteIndices(String clusterAlias, String indexName, int numShards) {
+        Client remoteClient = client(clusterAlias);
+        assertAcked(
+            remoteClient.admin()
+                .indices()
+                .prepareCreate(indexName)
+                .setSettings(Settings.builder().put("index.number_of_shards", numShards))
+                .setMapping("id", "type=keyword", "tag", "type=keyword", "v", "type=long")
+        );
+        for (int i = 0; i < 10; i++) {
+            remoteClient.prepareIndex(indexName).setSource("id", "remote-" + i, "tag", "remote", "v", i * i).get();
+        }
+        remoteClient.admin().indices().prepareRefresh(indexName).get();
+    }
+}
diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersQueryIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersQueryIT.java
index ddd5cff014ed2..ba44adb5a85e0 100644
--- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersQueryIT.java
+++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersQueryIT.java
@@ -25,6 +25,7 @@
 import org.elasticsearch.test.AbstractMultiClustersTestCase;
 import org.elasticsearch.test.InternalTestCluster;
 import org.elasticsearch.test.XContentTestUtils;
+import org.elasticsearch.transport.RemoteClusterAware;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.esql.plugin.EsqlPlugin;
 import org.elasticsearch.xpack.esql.plugin.QueryPragmas;
@@ -246,7 +247,8 @@ public void testSearchesWhereMissingIndicesAreSpecified() {
 
             EsqlExecutionInfo.Cluster localCluster = executionInfo.getCluster(LOCAL_CLUSTER);
             assertThat(localCluster.getIndexExpression(), equalTo("no_such_index"));
-            assertThat(localCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SKIPPED));
+            // TODO: a follow on PR will change this to throw an Exception when the local cluster requests a concrete index that is missing
+            assertThat(localCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL));
             assertThat(localCluster.getTook().millis(), greaterThanOrEqualTo(0L));
             assertThat(localCluster.getTook().millis(), lessThanOrEqualTo(overallTookMillis));
             assertThat(localCluster.getTotalShards(), equalTo(0));
@@ -499,7 +501,7 @@ public void testCCSExecutionOnSearchesWithLimit0() {
 
             EsqlExecutionInfo.Cluster localCluster = executionInfo.getCluster(LOCAL_CLUSTER);
             assertThat(localCluster.getIndexExpression(), equalTo("nomatch*"));
-            assertThat(localCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SKIPPED));
+            assertThat(localCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL));
             assertThat(localCluster.getTook().millis(), greaterThanOrEqualTo(0L));
             assertThat(localCluster.getTook().millis(), lessThanOrEqualTo(overallTookMillis));
             assertThat(remoteCluster.getTotalShards(), equalTo(0));
@@ -803,6 +805,14 @@ Map<String, Object> setupTwoClusters() {
         clusterInfo.put("local.index", localIndex);
         clusterInfo.put("remote.num_shards", numShardsRemote);
         clusterInfo.put("remote.index", remoteIndex);
+
+        String skipUnavailableKey = Strings.format("cluster.remote.%s.skip_unavailable", REMOTE_CLUSTER);
+        Setting<?> skipUnavailableSetting = cluster(REMOTE_CLUSTER).clusterService().getClusterSettings().get(skipUnavailableKey);
+        boolean skipUnavailable = (boolean) cluster(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY).clusterService()
+            .getClusterSettings()
+            .get(skipUnavailableSetting);
+        clusterInfo.put("remote.skip_unavailable", skipUnavailable);
+
         return clusterInfo;
     }
 
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlExecutionInfo.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlExecutionInfo.java
index aeac14091f378..f2ab0355304b3 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlExecutionInfo.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlExecutionInfo.java
@@ -8,6 +8,7 @@
 package org.elasticsearch.xpack.esql.action;
 
 import org.elasticsearch.TransportVersions;
+import org.elasticsearch.action.search.ShardSearchFailure;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -281,6 +282,7 @@ public static class Cluster implements ToXContentFragment, Writeable {
         private final Integer successfulShards;
         private final Integer skippedShards;
         private final Integer failedShards;
+        private final List<ShardSearchFailure> failures;
         private final TimeValue took;  // search latency for this cluster sub-search
 
         /**
@@ -300,7 +302,7 @@ public String toString() {
         }
 
         public Cluster(String clusterAlias, String indexExpression) {
-            this(clusterAlias, indexExpression, true, Cluster.Status.RUNNING, null, null, null, null, null);
+            this(clusterAlias, indexExpression, true, Cluster.Status.RUNNING, null, null, null, null, null, null);
         }
 
         /**
@@ -312,7 +314,7 @@ public Cluster(String clusterAlias, String indexExpression) {
          * @param skipUnavailable whether this Cluster is marked as skip_unavailable in remote cluster settings
          */
         public Cluster(String clusterAlias, String indexExpression, boolean skipUnavailable) {
-            this(clusterAlias, indexExpression, skipUnavailable, Cluster.Status.RUNNING, null, null, null, null, null);
+            this(clusterAlias, indexExpression, skipUnavailable, Cluster.Status.RUNNING, null, null, null, null, null, null);
         }
 
         /**
@@ -324,7 +326,7 @@ public Cluster(String clusterAlias, String indexExpression, boolean skipUnavaila
          * @param status current status of the search on this Cluster
          */
         public Cluster(String clusterAlias, String indexExpression, boolean skipUnavailable, Cluster.Status status) {
-            this(clusterAlias, indexExpression, skipUnavailable, status, null, null, null, null, null);
+            this(clusterAlias, indexExpression, skipUnavailable, status, null, null, null, null, null, null);
         }
 
         public Cluster(
@@ -336,6 +338,7 @@ public Cluster(
             Integer successfulShards,
             Integer skippedShards,
             Integer failedShards,
+            List<ShardSearchFailure> failures,
             TimeValue took
         ) {
             assert clusterAlias != null : "clusterAlias cannot be null";
@@ -349,6 +352,11 @@ public Cluster(
             this.successfulShards = successfulShards;
             this.skippedShards = skippedShards;
             this.failedShards = failedShards;
+            if (failures == null) {
+                this.failures = List.of();
+            } else {
+                this.failures = failures;
+            }
             this.took = took;
         }
 
@@ -362,6 +370,11 @@ public Cluster(StreamInput in) throws IOException {
             this.failedShards = in.readOptionalInt();
             this.took = in.readOptionalTimeValue();
             this.skipUnavailable = in.readBoolean();
+            if (in.getTransportVersion().onOrAfter(TransportVersions.ESQL_CCS_EXEC_INFO_WITH_FAILURES)) {
+                this.failures = Collections.unmodifiableList(in.readCollectionAsList(ShardSearchFailure::readShardSearchFailure));
+            } else {
+                this.failures = List.of();
+            }
         }
 
         @Override
@@ -375,6 +388,9 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeOptionalInt(failedShards);
             out.writeOptionalTimeValue(took);
             out.writeBoolean(skipUnavailable);
+            if (out.getTransportVersion().onOrAfter(TransportVersions.ESQL_CCS_EXEC_INFO_WITH_FAILURES)) {
+                out.writeCollection(failures);
+            }
         }
 
         /**
@@ -387,12 +403,12 @@ public void writeTo(StreamOutput out) throws IOException {
          * All other fields can be set and override the value in the "copyFrom" Cluster.
          */
         public static class Builder {
-            private String indexExpression;
             private Cluster.Status status;
             private Integer totalShards;
             private Integer successfulShards;
             private Integer skippedShards;
             private Integer failedShards;
+            private List<ShardSearchFailure> failures;
             private TimeValue took;
             private final Cluster original;
 
@@ -408,22 +424,18 @@ public Builder(Cluster copyFrom) {
             public Cluster build() {
                 return new Cluster(
                     original.getClusterAlias(),
-                    indexExpression == null ? original.getIndexExpression() : indexExpression,
+                    original.getIndexExpression(),
                     original.isSkipUnavailable(),
                     status != null ? status : original.getStatus(),
                     totalShards != null ? totalShards : original.getTotalShards(),
                     successfulShards != null ? successfulShards : original.getSuccessfulShards(),
                     skippedShards != null ? skippedShards : original.getSkippedShards(),
                     failedShards != null ? failedShards : original.getFailedShards(),
+                    failures != null ? failures : original.getFailures(),
                     took != null ? took : original.getTook()
                 );
             }
 
-            public Cluster.Builder setIndexExpression(String indexExpression) {
-                this.indexExpression = indexExpression;
-                return this;
-            }
-
             public Cluster.Builder setStatus(Cluster.Status status) {
                 this.status = status;
                 return this;
@@ -449,6 +461,11 @@ public Cluster.Builder setFailedShards(int failedShards) {
                 return this;
             }
 
+            public Cluster.Builder setFailures(List<ShardSearchFailure> failures) {
+                this.failures = failures;
+                return this;
+            }
+
             public Cluster.Builder setTook(TimeValue took) {
                 this.took = took;
                 return this;
@@ -466,7 +483,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
                 builder.field(STATUS_FIELD.getPreferredName(), getStatus().toString());
                 builder.field(INDICES_FIELD.getPreferredName(), indexExpression);
                 if (took != null) {
-                    // TODO: change this to took_nanos and call took.nanos?
                     builder.field(TOOK.getPreferredName(), took.millis());
                 }
                 if (totalShards != null) {
@@ -483,6 +499,13 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
                     }
                     builder.endObject();
                 }
+                if (failures != null && failures.size() > 0) {
+                    builder.startArray(RestActions.FAILURES_FIELD.getPreferredName());
+                    for (ShardSearchFailure failure : failures) {
+                        failure.toXContent(builder, params);
+                    }
+                    builder.endArray();
+                }
             }
             builder.endObject();
             return builder;
@@ -529,6 +552,10 @@ public Integer getFailedShards() {
             return failedShards;
         }
 
+        public List<ShardSearchFailure> getFailures() {
+            return failures;
+        }
+
         @Override
         public boolean equals(Object o) {
             if (this == o) return true;
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/EnrichResolution.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/EnrichResolution.java
index 7fb279f18b1dc..4f6886edc5fbc 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/EnrichResolution.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/EnrichResolution.java
@@ -23,6 +23,7 @@ public final class EnrichResolution {
 
     private final Map<Key, ResolvedEnrichPolicy> resolvedPolicies = ConcurrentCollections.newConcurrentMap();
     private final Map<Key, String> errors = ConcurrentCollections.newConcurrentMap();
+    private final Map<String, Exception> unavailableClusters = ConcurrentCollections.newConcurrentMap();
 
     public ResolvedEnrichPolicy getResolvedPolicy(String policyName, Enrich.Mode mode) {
         return resolvedPolicies.get(new Key(policyName, mode));
@@ -51,6 +52,14 @@ public void addError(String policyName, Enrich.Mode mode, String reason) {
         errors.putIfAbsent(new Key(policyName, mode), reason);
     }
 
+    public void addUnavailableCluster(String clusterAlias, Exception e) {
+        unavailableClusters.put(clusterAlias, e);
+    }
+
+    public Map<String, Exception> getUnavailableClusters() {
+        return unavailableClusters;
+    }
+
     private record Key(String policyName, Enrich.Mode mode) {
 
     }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichPolicyResolver.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichPolicyResolver.java
index e67c406e26929..77ef5ef597bb5 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichPolicyResolver.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichPolicyResolver.java
@@ -7,6 +7,7 @@
 
 package org.elasticsearch.xpack.esql.enrich;
 
+import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionListenerResponseHandler;
 import org.elasticsearch.action.search.SearchRequest;
@@ -50,6 +51,7 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -113,12 +115,27 @@ public void resolvePolicies(
         final boolean includeLocal = remoteClusters.remove(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY);
         lookupPolicies(remoteClusters, includeLocal, unresolvedPolicies, listener.map(lookupResponses -> {
             final EnrichResolution enrichResolution = new EnrichResolution();
+
+            Map<String, LookupResponse> lookupResponsesToProcess = new HashMap<>();
+
+            for (Map.Entry<String, LookupResponse> entry : lookupResponses.entrySet()) {
+                String clusterAlias = entry.getKey();
+                if (entry.getValue().connectionError != null) {
+                    enrichResolution.addUnavailableCluster(clusterAlias, entry.getValue().connectionError);
+                    // remove unavailable cluster from the list of clusters which is used below to create the ResolvedEnrichPolicy
+                    remoteClusters.remove(clusterAlias);
+                } else {
+                    lookupResponsesToProcess.put(clusterAlias, entry.getValue());
+                }
+            }
+
             for (UnresolvedPolicy unresolved : unresolvedPolicies) {
                 Tuple<ResolvedEnrichPolicy, String> resolved = mergeLookupResults(
                     unresolved,
                     calculateTargetClusters(unresolved.mode, includeLocal, remoteClusters),
-                    lookupResponses
+                    lookupResponsesToProcess
                 );
+
                 if (resolved.v1() != null) {
                     enrichResolution.addResolvedPolicy(unresolved.name, unresolved.mode, resolved.v1());
                 } else {
@@ -149,13 +166,16 @@ private Tuple<ResolvedEnrichPolicy, String> mergeLookupResults(
         Collection<String> targetClusters,
         Map<String, LookupResponse> lookupResults
     ) {
-        assert targetClusters.isEmpty() == false;
         String policyName = unresolved.name;
+        if (targetClusters.isEmpty()) {
+            return Tuple.tuple(null, "enrich policy [" + policyName + "] cannot be resolved since remote clusters are unavailable");
+        }
         final Map<String, ResolvedEnrichPolicy> policies = new HashMap<>();
         final List<String> failures = new ArrayList<>();
         for (String cluster : targetClusters) {
             LookupResponse lookupResult = lookupResults.get(cluster);
             if (lookupResult != null) {
+                assert lookupResult.connectionError == null : "Should never have a non-null connectionError here";
                 ResolvedEnrichPolicy policy = lookupResult.policies.get(policyName);
                 if (policy != null) {
                     policies.put(cluster, policy);
@@ -261,22 +281,34 @@ private void lookupPolicies(
             if (remotePolicies.isEmpty() == false) {
                 for (String cluster : remoteClusters) {
                     ActionListener<LookupResponse> lookupListener = refs.acquire(resp -> lookupResponses.put(cluster, resp));
-                    getRemoteConnection(
-                        cluster,
-                        lookupListener.delegateFailureAndWrap(
-                            (delegate, connection) -> transportService.sendRequest(
+                    getRemoteConnection(cluster, new ActionListener<Transport.Connection>() {
+                        @Override
+                        public void onResponse(Transport.Connection connection) {
+                            transportService.sendRequest(
                                 connection,
                                 RESOLVE_ACTION_NAME,
                                 new LookupRequest(cluster, remotePolicies),
                                 TransportRequestOptions.EMPTY,
-                                new ActionListenerResponseHandler<>(
-                                    delegate,
-                                    LookupResponse::new,
-                                    threadPool.executor(ThreadPool.Names.SEARCH)
-                                )
-                            )
-                        )
-                    );
+                                new ActionListenerResponseHandler<>(lookupListener.delegateResponse((l, e) -> {
+                                    if (ExceptionsHelper.isRemoteUnavailableException(e)
+                                        && remoteClusterService.isSkipUnavailable(cluster)) {
+                                        l.onResponse(new LookupResponse(e));
+                                    } else {
+                                        l.onFailure(e);
+                                    }
+                                }), LookupResponse::new, threadPool.executor(ThreadPool.Names.SEARCH))
+                            );
+                        }
+
+                        @Override
+                        public void onFailure(Exception e) {
+                            if (ExceptionsHelper.isRemoteUnavailableException(e) && remoteClusterService.isSkipUnavailable(cluster)) {
+                                lookupListener.onResponse(new LookupResponse(e));
+                            } else {
+                                lookupListener.onFailure(e);
+                            }
+                        }
+                    });
                 }
             }
             // local cluster
@@ -323,16 +355,30 @@ public void writeTo(StreamOutput out) throws IOException {
     private static class LookupResponse extends TransportResponse {
         final Map<String, ResolvedEnrichPolicy> policies;
         final Map<String, String> failures;
+        // does not need to be Writable since this indicates a failure to contact a remote cluster, so only set on querying cluster
+        final transient Exception connectionError;
 
         LookupResponse(Map<String, ResolvedEnrichPolicy> policies, Map<String, String> failures) {
             this.policies = policies;
             this.failures = failures;
+            this.connectionError = null;
+        }
+
+        /**
+         * Use this constructor when the remote cluster is unavailable to indicate inability to do the enrich policy lookup
+         * @param connectionError Exception received when trying to connect to a remote cluster
+         */
+        LookupResponse(Exception connectionError) {
+            this.policies = Collections.emptyMap();
+            this.failures = Collections.emptyMap();
+            this.connectionError = connectionError;
         }
 
         LookupResponse(StreamInput in) throws IOException {
             PlanStreamInput planIn = new PlanStreamInput(in, in.namedWriteableRegistry(), null);
             this.policies = planIn.readMap(StreamInput::readString, ResolvedEnrichPolicy::new);
             this.failures = planIn.readMap(StreamInput::readString, StreamInput::readString);
+            this.connectionError = null;
         }
 
         @Override
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/index/IndexResolution.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/index/IndexResolution.java
index 371aa1b632309..b2eaefcf09d65 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/index/IndexResolution.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/index/IndexResolution.java
@@ -6,27 +6,28 @@
  */
 package org.elasticsearch.xpack.esql.index;
 
+import org.elasticsearch.action.fieldcaps.FieldCapabilitiesFailure;
 import org.elasticsearch.core.Nullable;
 
 import java.util.Collections;
+import java.util.Map;
 import java.util.Objects;
-import java.util.Set;
 
 public final class IndexResolution {
 
-    public static IndexResolution valid(EsIndex index, Set<String> unavailableClusters) {
+    public static IndexResolution valid(EsIndex index, Map<String, FieldCapabilitiesFailure> unavailableClusters) {
         Objects.requireNonNull(index, "index must not be null if it was found");
         Objects.requireNonNull(unavailableClusters, "unavailableClusters must not be null");
         return new IndexResolution(index, null, unavailableClusters);
     }
 
     public static IndexResolution valid(EsIndex index) {
-        return valid(index, Collections.emptySet());
+        return valid(index, Collections.emptyMap());
     }
 
     public static IndexResolution invalid(String invalid) {
         Objects.requireNonNull(invalid, "invalid must not be null to signal that the index is invalid");
-        return new IndexResolution(null, invalid, Collections.emptySet());
+        return new IndexResolution(null, invalid, Collections.emptyMap());
     }
 
     public static IndexResolution notFound(String name) {
@@ -39,9 +40,9 @@ public static IndexResolution notFound(String name) {
     private final String invalid;
 
     // remote clusters included in the user's index expression that could not be connected to
-    private final Set<String> unavailableClusters;
+    private final Map<String, FieldCapabilitiesFailure> unavailableClusters;
 
-    private IndexResolution(EsIndex index, @Nullable String invalid, Set<String> unavailableClusters) {
+    private IndexResolution(EsIndex index, @Nullable String invalid, Map<String, FieldCapabilitiesFailure> unavailableClusters) {
         this.index = index;
         this.invalid = invalid;
         this.unavailableClusters = unavailableClusters;
@@ -70,7 +71,11 @@ public boolean isValid() {
         return invalid == null;
     }
 
-    public Set<String> getUnavailableClusters() {
+    /**
+     * @return Map of unavailable clusters (could not be connected to during field-caps query). Key of map is cluster alias,
+     * value is the {@link FieldCapabilitiesFailure} describing the issue.
+     */
+    public Map<String, FieldCapabilitiesFailure> getUnavailableClusters() {
         return unavailableClusters;
     }
 
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java
index ccd167942340c..1e78f454b7531 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java
@@ -7,8 +7,11 @@
 
 package org.elasticsearch.xpack.esql.session;
 
+import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.OriginalIndices;
+import org.elasticsearch.action.fieldcaps.FieldCapabilitiesFailure;
+import org.elasticsearch.action.search.ShardSearchFailure;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.collect.Iterators;
@@ -22,7 +25,9 @@
 import org.elasticsearch.indices.IndicesExpressionGrouper;
 import org.elasticsearch.logging.LogManager;
 import org.elasticsearch.logging.Logger;
+import org.elasticsearch.transport.ConnectTransportException;
 import org.elasticsearch.transport.RemoteClusterAware;
+import org.elasticsearch.transport.RemoteTransportException;
 import org.elasticsearch.xpack.esql.action.EsqlExecutionInfo;
 import org.elasticsearch.xpack.esql.action.EsqlQueryRequest;
 import org.elasticsearch.xpack.esql.analysis.Analyzer;
@@ -44,6 +49,7 @@
 import org.elasticsearch.xpack.esql.enrich.ResolvedEnrichPolicy;
 import org.elasticsearch.xpack.esql.expression.UnresolvedNamePattern;
 import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry;
+import org.elasticsearch.xpack.esql.index.EsIndex;
 import org.elasticsearch.xpack.esql.index.IndexResolution;
 import org.elasticsearch.xpack.esql.index.MappingException;
 import org.elasticsearch.xpack.esql.optimizer.LogicalPlanOptimizer;
@@ -68,6 +74,7 @@
 
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
@@ -143,12 +150,105 @@ public void execute(
         analyzedPlan(
             parse(request.query(), request.params()),
             executionInfo,
-            listener.delegateFailureAndWrap(
-                (next, analyzedPlan) -> executeOptimizedPlan(request, executionInfo, runPhase, optimizedPlan(analyzedPlan), next)
-            )
+            new LogicalPlanActionListener(request, executionInfo, runPhase, listener)
         );
     }
 
+    /**
+     * ActionListener that receives LogicalPlan or error from logical planning.
+     * Any Exception sent to onFailure stops processing, but not all are fatal (return a 4xx or 5xx), so
+     * the onFailure handler determines whether to return an empty successful result or a 4xx/5xx error.
+     */
+    class LogicalPlanActionListener implements ActionListener<LogicalPlan> {
+        private final EsqlQueryRequest request;
+        private final EsqlExecutionInfo executionInfo;
+        private final BiConsumer<PhysicalPlan, ActionListener<Result>> runPhase;
+        private final ActionListener<Result> listener;
+
+        LogicalPlanActionListener(
+            EsqlQueryRequest request,
+            EsqlExecutionInfo executionInfo,
+            BiConsumer<PhysicalPlan, ActionListener<Result>> runPhase,
+            ActionListener<Result> listener
+        ) {
+            this.request = request;
+            this.executionInfo = executionInfo;
+            this.runPhase = runPhase;
+            this.listener = listener;
+        }
+
+        @Override
+        public void onResponse(LogicalPlan analyzedPlan) {
+            executeOptimizedPlan(request, executionInfo, runPhase, optimizedPlan(analyzedPlan), listener);
+        }
+
+        /**
+         * Whether to return an empty result (HTTP status 200) for a CCS rather than a top level 4xx/5xx error.
+         *
+         * For cases where field-caps had no indices to search and the remotes were unavailable, we
+         * return an empty successful response (200) if all remotes are marked with skip_unavailable=true.
+         *
+         * Note: a follow-on PR will expand this logic to handle cases where no indices could be found to match
+         * on any of the requested clusters.
+         */
+        private boolean returnSuccessWithEmptyResult(Exception e) {
+            if (executionInfo.isCrossClusterSearch() == false) {
+                return false;
+            }
+
+            if (e instanceof NoClustersToSearchException || ExceptionsHelper.isRemoteUnavailableException(e)) {
+                for (String clusterAlias : executionInfo.clusterAliases()) {
+                    if (executionInfo.isSkipUnavailable(clusterAlias) == false
+                        && clusterAlias.equals(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY) == false) {
+                        return false;
+                    }
+                }
+                return true;
+            }
+            return false;
+        }
+
+        @Override
+        public void onFailure(Exception e) {
+            if (returnSuccessWithEmptyResult(e)) {
+                executionInfo.markEndQuery();
+                Exception exceptionForResponse;
+                if (e instanceof ConnectTransportException) {
+                    // when field-caps has no field info (since no clusters could be connected to or had matching indices)
+                    // it just throws the first exception in its list, so this odd special handling is here is to avoid
+                    // having one specific remote alias name in all failure lists in the metadata response
+                    exceptionForResponse = new RemoteTransportException(
+                        "connect_transport_exception - unable to connect to remote cluster",
+                        null
+                    );
+                } else {
+                    exceptionForResponse = e;
+                }
+                for (String clusterAlias : executionInfo.clusterAliases()) {
+                    executionInfo.swapCluster(clusterAlias, (k, v) -> {
+                        EsqlExecutionInfo.Cluster.Builder builder = new EsqlExecutionInfo.Cluster.Builder(v).setTook(
+                            executionInfo.overallTook()
+                        ).setTotalShards(0).setSuccessfulShards(0).setSkippedShards(0).setFailedShards(0);
+                        if (RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY.equals(clusterAlias)) {
+                            // never mark local cluster as skipped
+                            builder.setStatus(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL);
+                        } else {
+                            builder.setStatus(EsqlExecutionInfo.Cluster.Status.SKIPPED);
+                            // add this exception to the failures list only if there is no failure already recorded there
+                            if (v.getFailures() == null || v.getFailures().size() == 0) {
+                                builder.setFailures(List.of(new ShardSearchFailure(exceptionForResponse)));
+                            }
+                        }
+                        return builder.build();
+                    });
+                }
+                listener.onResponse(new Result(Analyzer.NO_FIELDS, Collections.emptyList(), Collections.emptyList(), executionInfo));
+            } else {
+                listener.onFailure(e);
+            }
+        }
+    }
+
     /**
      * Execute an analyzed plan. Most code should prefer calling {@link #execute} but
      * this is public for testing. See {@link Phased} for the sequence of operations.
@@ -161,8 +261,8 @@ public void executeOptimizedPlan(
         ActionListener<Result> listener
     ) {
         LogicalPlan firstPhase = Phased.extractFirstPhase(optimizedPlan);
+        updateExecutionInfoAtEndOfPlanning(executionInfo);
         if (firstPhase == null) {
-            updateExecutionInfoAtEndOfPlanning(executionInfo);
             runPhase.accept(logicalPlanToPhysicalPlan(optimizedPlan, request), listener);
         } else {
             executePhased(new ArrayList<>(), optimizedPlan, request, executionInfo, firstPhase, runPhase, listener);
@@ -242,17 +342,30 @@ private <T> void preAnalyze(
                 .stream()
                 .map(ResolvedEnrichPolicy::matchField)
                 .collect(Collectors.toSet());
-            preAnalyzeIndices(parsed, executionInfo, l.delegateFailureAndWrap((ll, indexResolution) -> {
+            Map<String, Exception> unavailableClusters = enrichResolution.getUnavailableClusters();
+            preAnalyzeIndices(parsed, executionInfo, unavailableClusters, l.delegateFailureAndWrap((ll, indexResolution) -> {
+                // TODO in follow-PR (for skip_unavailble handling of missing concrete indexes) add some tests for invalid index
+                // resolution to updateExecutionInfo
                 if (indexResolution.isValid()) {
                     updateExecutionInfoWithClustersWithNoMatchingIndices(executionInfo, indexResolution);
                     updateExecutionInfoWithUnavailableClusters(executionInfo, indexResolution.getUnavailableClusters());
+                    if (executionInfo.isCrossClusterSearch()
+                        && executionInfo.getClusterStateCount(EsqlExecutionInfo.Cluster.Status.RUNNING) == 0) {
+                        // for a CCS, if all clusters have been marked as SKIPPED, nothing to search so send a sentinel
+                        // Exception to let the LogicalPlanActionListener decide how to proceed
+                        ll.onFailure(new NoClustersToSearchException());
+                        return;
+                    }
+
                     Set<String> newClusters = enrichPolicyResolver.groupIndicesPerCluster(
                         indexResolution.get().concreteIndices().toArray(String[]::new)
                     ).keySet();
                     // If new clusters appear when resolving the main indices, we need to resolve the enrich policies again
                     // or exclude main concrete indices. Since this is rare, it's simpler to resolve the enrich policies again.
                     // TODO: add a test for this
-                    if (targetClusters.containsAll(newClusters) == false) {
+                    if (targetClusters.containsAll(newClusters) == false
+                        // do not bother with a re-resolution if only remotes were requested and all were offline
+                        && executionInfo.getClusterStateCount(EsqlExecutionInfo.Cluster.Status.RUNNING) > 0) {
                         enrichPolicyResolver.resolvePolicies(
                             newClusters,
                             unresolvedPolicies,
@@ -269,6 +382,7 @@ private <T> void preAnalyze(
     private void preAnalyzeIndices(
         LogicalPlan parsed,
         EsqlExecutionInfo executionInfo,
+        Map<String, Exception> unavailableClusters,  // known to be unavailable from the enrich policy API call
         ActionListener<IndexResolution> listener,
         Set<String> enrichPolicyMatchFields
     ) {
@@ -288,10 +402,34 @@ private void preAnalyzeIndices(
                 String indexExpr = Strings.arrayToCommaDelimitedString(entry.getValue().indices());
                 executionInfo.swapCluster(clusterAlias, (k, v) -> {
                     assert v == null : "No cluster for " + clusterAlias + " should have been added to ExecutionInfo yet";
-                    return new EsqlExecutionInfo.Cluster(clusterAlias, indexExpr, executionInfo.isSkipUnavailable(clusterAlias));
+                    if (unavailableClusters.containsKey(k)) {
+                        return new EsqlExecutionInfo.Cluster(
+                            clusterAlias,
+                            indexExpr,
+                            executionInfo.isSkipUnavailable(clusterAlias),
+                            EsqlExecutionInfo.Cluster.Status.SKIPPED,
+                            0,
+                            0,
+                            0,
+                            0,
+                            List.of(new ShardSearchFailure(unavailableClusters.get(k))),
+                            new TimeValue(0)
+                        );
+                    } else {
+                        return new EsqlExecutionInfo.Cluster(clusterAlias, indexExpr, executionInfo.isSkipUnavailable(clusterAlias));
+                    }
                 });
             }
-            indexResolver.resolveAsMergedMapping(table.index(), fieldNames, listener);
+            // if the preceding call to the enrich policy API found unavailable clusters, recreate the index expression to search
+            // based only on available clusters (which could now be an empty list)
+            String indexExpressionToResolve = createIndexExpressionFromAvailableClusters(executionInfo);
+            if (indexExpressionToResolve.isEmpty()) {
+                // if this was a pure remote CCS request (no local indices) and all remotes are offline, return an empty IndexResolution
+                listener.onResponse(IndexResolution.valid(new EsIndex(table.index(), Map.of(), Map.of())));
+            } else {
+                // call the EsqlResolveFieldsAction (field-caps) to resolve indices and get field types
+                indexResolver.resolveAsMergedMapping(indexExpressionToResolve, fieldNames, listener);
+            }
         } else {
             try {
                 // occurs when dealing with local relations (row a = 1)
@@ -302,6 +440,30 @@ private void preAnalyzeIndices(
         }
     }
 
+    // visible for testing
+    static String createIndexExpressionFromAvailableClusters(EsqlExecutionInfo executionInfo) {
+        StringBuilder sb = new StringBuilder();
+        for (String clusterAlias : executionInfo.clusterAliases()) {
+            EsqlExecutionInfo.Cluster cluster = executionInfo.getCluster(clusterAlias);
+            if (cluster.getStatus() != EsqlExecutionInfo.Cluster.Status.SKIPPED) {
+                if (cluster.getClusterAlias().equals(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY)) {
+                    sb.append(executionInfo.getCluster(clusterAlias).getIndexExpression()).append(',');
+                } else {
+                    String indexExpression = executionInfo.getCluster(clusterAlias).getIndexExpression();
+                    for (String index : indexExpression.split(",")) {
+                        sb.append(clusterAlias).append(':').append(index).append(',');
+                    }
+                }
+            }
+        }
+
+        if (sb.length() > 0) {
+            return sb.substring(0, sb.length() - 1);
+        } else {
+            return "";
+        }
+    }
+
     static Set<String> fieldNames(LogicalPlan parsed, Set<String> enrichPolicyMatchFields) {
         if (false == parsed.anyMatch(plan -> plan instanceof Aggregate || plan instanceof Project)) {
             // no explicit columns selection, for example "from employees"
@@ -446,14 +608,28 @@ public PhysicalPlan optimizedPhysicalPlan(LogicalPlan optimizedPlan) {
         return plan;
     }
 
-    // visible for testing
-    static void updateExecutionInfoWithUnavailableClusters(EsqlExecutionInfo executionInfo, Set<String> unavailableClusters) {
-        for (String clusterAlias : unavailableClusters) {
-            executionInfo.swapCluster(
-                clusterAlias,
-                (k, v) -> new EsqlExecutionInfo.Cluster.Builder(v).setStatus(EsqlExecutionInfo.Cluster.Status.SKIPPED).build()
+    static void updateExecutionInfoWithUnavailableClusters(EsqlExecutionInfo execInfo, Map<String, FieldCapabilitiesFailure> unavailable) {
+        for (Map.Entry<String, FieldCapabilitiesFailure> entry : unavailable.entrySet()) {
+            String clusterAlias = entry.getKey();
+            boolean skipUnavailable = execInfo.getCluster(clusterAlias).isSkipUnavailable();
+            RemoteTransportException e = new RemoteTransportException(
+                Strings.format("Remote cluster [%s] (with setting skip_unavailable=%s) is not available", clusterAlias, skipUnavailable),
+                entry.getValue().getException()
             );
-            // TODO: follow-on PR will set SKIPPED status when skip_unavailable=true and throw an exception when skip_un=false
+            if (skipUnavailable) {
+                execInfo.swapCluster(
+                    clusterAlias,
+                    (k, v) -> new EsqlExecutionInfo.Cluster.Builder(v).setStatus(EsqlExecutionInfo.Cluster.Status.SKIPPED)
+                        .setTotalShards(0)
+                        .setSuccessfulShards(0)
+                        .setSkippedShards(0)
+                        .setFailedShards(0)
+                        .setFailures(List.of(new ShardSearchFailure(e)))
+                        .build()
+                );
+            } else {
+                throw e;
+            }
         }
     }
 
@@ -466,16 +642,22 @@ static void updateExecutionInfoWithClustersWithNoMatchingIndices(EsqlExecutionIn
         }
         Set<String> clustersRequested = executionInfo.clusterAliases();
         Set<String> clustersWithNoMatchingIndices = Sets.difference(clustersRequested, clustersWithResolvedIndices);
-        clustersWithNoMatchingIndices.removeAll(indexResolution.getUnavailableClusters());
+        clustersWithNoMatchingIndices.removeAll(indexResolution.getUnavailableClusters().keySet());
         /*
          * These are clusters in the original request that are not present in the field-caps response. They were
          * specified with an index or indices that do not exist, so the search on that cluster is done.
          * Mark it as SKIPPED with 0 shards searched and took=0.
          */
         for (String c : clustersWithNoMatchingIndices) {
+            // TODO: in a follow-on PR, throw a Verification(400 status code) for local and remotes with skip_unavailable=false if
+            // they were requested with one or more concrete indices
+            // for now we never mark the local cluster as SKIPPED
+            final var status = RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY.equals(c)
+                ? EsqlExecutionInfo.Cluster.Status.SUCCESSFUL
+                : EsqlExecutionInfo.Cluster.Status.SKIPPED;
             executionInfo.swapCluster(
                 c,
-                (k, v) -> new EsqlExecutionInfo.Cluster.Builder(v).setStatus(EsqlExecutionInfo.Cluster.Status.SKIPPED)
+                (k, v) -> new EsqlExecutionInfo.Cluster.Builder(v).setStatus(status)
                     .setTook(new TimeValue(0))
                     .setTotalShards(0)
                     .setSuccessfulShards(0)
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java
index c0f94bccc50a4..f76f7798dece8 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java
@@ -90,6 +90,7 @@ public void resolveAsMergedMapping(String indexWildcard, Set<String> fieldNames,
     public IndexResolution mergedMappings(String indexPattern, FieldCapabilitiesResponse fieldCapsResponse) {
         assert ThreadPool.assertCurrentThreadPool(ThreadPool.Names.SEARCH_COORDINATION); // too expensive to run this on a transport worker
         if (fieldCapsResponse.getIndexResponses().isEmpty()) {
+            // TODO in follow-on PR, handle the case where remotes were specified with non-existent indices, according to skip_unavailable
             return IndexResolution.notFound(indexPattern);
         }
 
@@ -158,18 +159,18 @@ public IndexResolution mergedMappings(String indexPattern, FieldCapabilitiesResp
         for (FieldCapabilitiesIndexResponse ir : fieldCapsResponse.getIndexResponses()) {
             concreteIndices.put(ir.getIndexName(), ir.getIndexMode());
         }
-        Set<String> unavailableRemoteClusters = determineUnavailableRemoteClusters(fieldCapsResponse.getFailures());
-        return IndexResolution.valid(new EsIndex(indexPattern, rootFields, concreteIndices), unavailableRemoteClusters);
+        Map<String, FieldCapabilitiesFailure> unavailableRemotes = determineUnavailableRemoteClusters(fieldCapsResponse.getFailures());
+        return IndexResolution.valid(new EsIndex(indexPattern, rootFields, concreteIndices), unavailableRemotes);
     }
 
     // visible for testing
-    static Set<String> determineUnavailableRemoteClusters(List<FieldCapabilitiesFailure> failures) {
-        Set<String> unavailableRemotes = new HashSet<>();
+    static Map<String, FieldCapabilitiesFailure> determineUnavailableRemoteClusters(List<FieldCapabilitiesFailure> failures) {
+        Map<String, FieldCapabilitiesFailure> unavailableRemotes = new HashMap<>();
         for (FieldCapabilitiesFailure failure : failures) {
             if (ExceptionsHelper.isRemoteUnavailableException(failure.getException())) {
                 for (String indexExpression : failure.getIndices()) {
                     if (indexExpression.indexOf(RemoteClusterAware.REMOTE_CLUSTER_INDEX_SEPARATOR) > 0) {
-                        unavailableRemotes.add(RemoteClusterAware.parseClusterAlias(indexExpression));
+                        unavailableRemotes.put(RemoteClusterAware.parseClusterAlias(indexExpression), failure);
                     }
                 }
             }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/NoClustersToSearchException.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/NoClustersToSearchException.java
new file mode 100644
index 0000000000000..f7ae78a521933
--- /dev/null
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/NoClustersToSearchException.java
@@ -0,0 +1,15 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.session;
+
+/**
+ * Sentinel exception indicating that logical planning could not find any clusters to search
+ * when, for a remote-only cross-cluster search, all clusters have been marked as SKIPPED.
+ * Intended for use only on the querying coordinating during ES|QL logical planning.
+ */
+public class NoClustersToSearchException extends RuntimeException {}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java
index 27343bf7ce205..4aaf4f6cccf0f 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java
@@ -147,6 +147,7 @@ EsqlExecutionInfo createExecutionInfo() {
                 10,
                 3,
                 0,
+                null,
                 new TimeValue(4444L)
             )
         );
@@ -161,6 +162,7 @@ EsqlExecutionInfo createExecutionInfo() {
                 12,
                 5,
                 0,
+                null,
                 new TimeValue(4999L)
             )
         );
@@ -498,6 +500,7 @@ private static EsqlExecutionInfo.Cluster parseCluster(String clusterAlias, XCont
                 successfulShardsFinal,
                 skippedShardsFinal,
                 failedShardsFinal,
+                null,
                 tookTimeValue
             );
         }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/ComputeListenerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/ComputeListenerTests.java
index 5fbd5dd28050f..625cb5628d039 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/ComputeListenerTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/ComputeListenerTests.java
@@ -313,6 +313,7 @@ public void testAcquireComputeRunningOnRemoteClusterFillsInTookTime() {
                 10,
                 3,
                 0,
+                null,
                 null  // to be filled in the acquireCompute listener
             )
         );
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/EsqlSessionTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/EsqlSessionTests.java
index 32b31cf78650b..dddfa67338419 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/EsqlSessionTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/EsqlSessionTests.java
@@ -7,117 +7,200 @@
 
 package org.elasticsearch.xpack.esql.session;
 
+import org.elasticsearch.action.fieldcaps.FieldCapabilitiesFailure;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.transport.NoSeedNodeLeftException;
 import org.elasticsearch.transport.RemoteClusterAware;
+import org.elasticsearch.transport.RemoteTransportException;
 import org.elasticsearch.xpack.esql.action.EsqlExecutionInfo;
 import org.elasticsearch.xpack.esql.core.type.EsField;
 import org.elasticsearch.xpack.esql.index.EsIndex;
 import org.elasticsearch.xpack.esql.index.IndexResolution;
 import org.elasticsearch.xpack.esql.type.EsFieldTests;
 
+import java.util.Arrays;
 import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 
 public class EsqlSessionTests extends ESTestCase {
 
-    public void testUpdateExecutionInfoWithUnavailableClusters() {
-        // skip_unavailable=true clusters are unavailable, both marked as SKIPPED
+    public void testCreateIndexExpressionFromAvailableClusters() {
+        final String localClusterAlias = RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY;
+        final String remote1Alias = "remote1";
+        final String remote2Alias = "remote2";
+
+        // no clusters marked as skipped
         {
-            final String localClusterAlias = RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY;
-            final String remote1Alias = "remote1";
-            final String remote2Alias = "remote2";
             EsqlExecutionInfo executionInfo = new EsqlExecutionInfo(true);
             executionInfo.swapCluster(localClusterAlias, (k, v) -> new EsqlExecutionInfo.Cluster(localClusterAlias, "logs*", false));
             executionInfo.swapCluster(remote1Alias, (k, v) -> new EsqlExecutionInfo.Cluster(remote1Alias, "*", true));
             executionInfo.swapCluster(remote2Alias, (k, v) -> new EsqlExecutionInfo.Cluster(remote2Alias, "mylogs1,mylogs2,logs*", true));
 
-            EsqlSession.updateExecutionInfoWithUnavailableClusters(executionInfo, Set.of(remote1Alias, remote2Alias));
+            String indexExpr = EsqlSession.createIndexExpressionFromAvailableClusters(executionInfo);
+            List<String> list = Arrays.stream(Strings.splitStringByCommaToArray(indexExpr)).toList();
+            assertThat(list.size(), equalTo(5));
+            assertThat(
+                new HashSet<>(list),
+                equalTo(Strings.commaDelimitedListToSet("logs*,remote1:*,remote2:mylogs1,remote2:mylogs2,remote2:logs*"))
+            );
+        }
 
-            assertThat(executionInfo.clusterAliases(), equalTo(Set.of(localClusterAlias, remote1Alias, remote2Alias)));
-            assertNull(executionInfo.overallTook());
+        // one cluster marked as skipped, so not present in revised index expression
+        {
+            EsqlExecutionInfo executionInfo = new EsqlExecutionInfo(true);
+            executionInfo.swapCluster(localClusterAlias, (k, v) -> new EsqlExecutionInfo.Cluster(localClusterAlias, "logs*", false));
+            executionInfo.swapCluster(remote1Alias, (k, v) -> new EsqlExecutionInfo.Cluster(remote1Alias, "*,foo", true));
+            executionInfo.swapCluster(
+                remote2Alias,
+                (k, v) -> new EsqlExecutionInfo.Cluster(
+                    remote2Alias,
+                    "mylogs1,mylogs2,logs*",
+                    true,
+                    EsqlExecutionInfo.Cluster.Status.SKIPPED
+                )
+            );
 
-            EsqlExecutionInfo.Cluster localCluster = executionInfo.getCluster(localClusterAlias);
-            assertThat(localCluster.getIndexExpression(), equalTo("logs*"));
-            assertClusterStatusAndHasNullCounts(localCluster, EsqlExecutionInfo.Cluster.Status.RUNNING);
+            String indexExpr = EsqlSession.createIndexExpressionFromAvailableClusters(executionInfo);
+            List<String> list = Arrays.stream(Strings.splitStringByCommaToArray(indexExpr)).toList();
+            assertThat(list.size(), equalTo(3));
+            assertThat(new HashSet<>(list), equalTo(Strings.commaDelimitedListToSet("logs*,remote1:*,remote1:foo")));
+        }
 
-            EsqlExecutionInfo.Cluster remote1Cluster = executionInfo.getCluster(remote1Alias);
-            assertThat(remote1Cluster.getIndexExpression(), equalTo("*"));
-            assertClusterStatusAndHasNullCounts(remote1Cluster, EsqlExecutionInfo.Cluster.Status.SKIPPED);
+        // two clusters marked as skipped, so only local cluster present in revised index expression
+        {
+            EsqlExecutionInfo executionInfo = new EsqlExecutionInfo(true);
+            executionInfo.swapCluster(localClusterAlias, (k, v) -> new EsqlExecutionInfo.Cluster(localClusterAlias, "logs*", false));
+            executionInfo.swapCluster(
+                remote1Alias,
+                (k, v) -> new EsqlExecutionInfo.Cluster(remote1Alias, "*,foo", true, EsqlExecutionInfo.Cluster.Status.SKIPPED)
+            );
+            executionInfo.swapCluster(
+                remote2Alias,
+                (k, v) -> new EsqlExecutionInfo.Cluster(
+                    remote2Alias,
+                    "mylogs1,mylogs2,logs*",
+                    true,
+                    EsqlExecutionInfo.Cluster.Status.SKIPPED
+                )
+            );
 
-            EsqlExecutionInfo.Cluster remote2Cluster = executionInfo.getCluster(remote2Alias);
-            assertThat(remote2Cluster.getIndexExpression(), equalTo("mylogs1,mylogs2,logs*"));
-            assertClusterStatusAndHasNullCounts(remote2Cluster, EsqlExecutionInfo.Cluster.Status.SKIPPED);
+            assertThat(EsqlSession.createIndexExpressionFromAvailableClusters(executionInfo), equalTo("logs*"));
+        }
+
+        // only remotes present and all marked as skipped, so in revised index expression should be empty string
+        {
+            EsqlExecutionInfo executionInfo = new EsqlExecutionInfo(true);
+            executionInfo.swapCluster(
+                remote1Alias,
+                (k, v) -> new EsqlExecutionInfo.Cluster(remote1Alias, "*,foo", true, EsqlExecutionInfo.Cluster.Status.SKIPPED)
+            );
+            executionInfo.swapCluster(
+                remote2Alias,
+                (k, v) -> new EsqlExecutionInfo.Cluster(
+                    remote2Alias,
+                    "mylogs1,mylogs2,logs*",
+                    true,
+                    EsqlExecutionInfo.Cluster.Status.SKIPPED
+                )
+            );
+
+            assertThat(EsqlSession.createIndexExpressionFromAvailableClusters(executionInfo), equalTo(""));
         }
+    }
+
+    public void testUpdateExecutionInfoWithUnavailableClusters() {
+        final String localClusterAlias = RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY;
+        final String remote1Alias = "remote1";
+        final String remote2Alias = "remote2";
 
-        // skip_unavailable=false cluster is unavailable, marked as SKIPPED // TODO: in follow on PR this will change to throwing an
-        // Exception
+        // skip_unavailable=true clusters are unavailable, both marked as SKIPPED
         {
-            final String localClusterAlias = RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY;
-            final String remote1Alias = "remote1";
-            final String remote2Alias = "remote2";
             EsqlExecutionInfo executionInfo = new EsqlExecutionInfo(true);
             executionInfo.swapCluster(localClusterAlias, (k, v) -> new EsqlExecutionInfo.Cluster(localClusterAlias, "logs*", false));
             executionInfo.swapCluster(remote1Alias, (k, v) -> new EsqlExecutionInfo.Cluster(remote1Alias, "*", true));
-            executionInfo.swapCluster(remote2Alias, (k, v) -> new EsqlExecutionInfo.Cluster(remote2Alias, "mylogs1,mylogs2,logs*", false));
+            executionInfo.swapCluster(remote2Alias, (k, v) -> new EsqlExecutionInfo.Cluster(remote2Alias, "mylogs1,mylogs2,logs*", true));
 
-            EsqlSession.updateExecutionInfoWithUnavailableClusters(executionInfo, Set.of(remote2Alias));
+            var failure = new FieldCapabilitiesFailure(new String[] { "logs-a" }, new NoSeedNodeLeftException("unable to connect"));
+            var unvailableClusters = Map.of(remote1Alias, failure, remote2Alias, failure);
+            EsqlSession.updateExecutionInfoWithUnavailableClusters(executionInfo, unvailableClusters);
 
             assertThat(executionInfo.clusterAliases(), equalTo(Set.of(localClusterAlias, remote1Alias, remote2Alias)));
             assertNull(executionInfo.overallTook());
 
             EsqlExecutionInfo.Cluster localCluster = executionInfo.getCluster(localClusterAlias);
             assertThat(localCluster.getIndexExpression(), equalTo("logs*"));
-            assertClusterStatusAndHasNullCounts(localCluster, EsqlExecutionInfo.Cluster.Status.RUNNING);
+            assertClusterStatusAndShardCounts(localCluster, EsqlExecutionInfo.Cluster.Status.RUNNING);
 
             EsqlExecutionInfo.Cluster remote1Cluster = executionInfo.getCluster(remote1Alias);
             assertThat(remote1Cluster.getIndexExpression(), equalTo("*"));
-            assertClusterStatusAndHasNullCounts(remote1Cluster, EsqlExecutionInfo.Cluster.Status.RUNNING);
+            assertClusterStatusAndShardCounts(remote1Cluster, EsqlExecutionInfo.Cluster.Status.SKIPPED);
 
             EsqlExecutionInfo.Cluster remote2Cluster = executionInfo.getCluster(remote2Alias);
             assertThat(remote2Cluster.getIndexExpression(), equalTo("mylogs1,mylogs2,logs*"));
-            assertClusterStatusAndHasNullCounts(remote2Cluster, EsqlExecutionInfo.Cluster.Status.SKIPPED);
+            assertClusterStatusAndShardCounts(remote2Cluster, EsqlExecutionInfo.Cluster.Status.SKIPPED);
+        }
+
+        // skip_unavailable=false cluster is unavailable, throws Exception
+        {
+            EsqlExecutionInfo executionInfo = new EsqlExecutionInfo(true);
+            executionInfo.swapCluster(localClusterAlias, (k, v) -> new EsqlExecutionInfo.Cluster(localClusterAlias, "logs*", false));
+            executionInfo.swapCluster(remote1Alias, (k, v) -> new EsqlExecutionInfo.Cluster(remote1Alias, "*", true));
+            executionInfo.swapCluster(remote2Alias, (k, v) -> new EsqlExecutionInfo.Cluster(remote2Alias, "mylogs1,mylogs2,logs*", false));
+
+            var failure = new FieldCapabilitiesFailure(new String[] { "logs-a" }, new NoSeedNodeLeftException("unable to connect"));
+            RemoteTransportException e = expectThrows(
+                RemoteTransportException.class,
+                () -> EsqlSession.updateExecutionInfoWithUnavailableClusters(executionInfo, Map.of(remote2Alias, failure))
+            );
+            assertThat(e.status().getStatus(), equalTo(500));
+            assertThat(
+                e.getDetailedMessage(),
+                containsString("Remote cluster [remote2] (with setting skip_unavailable=false) is not available")
+            );
+            assertThat(e.getCause().getMessage(), containsString("unable to connect"));
         }
 
         // all clusters available, no Clusters in ExecutionInfo should be modified
         {
-            final String localClusterAlias = RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY;
-            final String remote1Alias = "remote1";
-            final String remote2Alias = "remote2";
             EsqlExecutionInfo executionInfo = new EsqlExecutionInfo(true);
             executionInfo.swapCluster(localClusterAlias, (k, v) -> new EsqlExecutionInfo.Cluster(localClusterAlias, "logs*", false));
             executionInfo.swapCluster(remote1Alias, (k, v) -> new EsqlExecutionInfo.Cluster(remote1Alias, "*", true));
             executionInfo.swapCluster(remote2Alias, (k, v) -> new EsqlExecutionInfo.Cluster(remote2Alias, "mylogs1,mylogs2,logs*", false));
 
-            EsqlSession.updateExecutionInfoWithUnavailableClusters(executionInfo, Set.of());
+            EsqlSession.updateExecutionInfoWithUnavailableClusters(executionInfo, Map.of());
 
             assertThat(executionInfo.clusterAliases(), equalTo(Set.of(localClusterAlias, remote1Alias, remote2Alias)));
             assertNull(executionInfo.overallTook());
 
             EsqlExecutionInfo.Cluster localCluster = executionInfo.getCluster(localClusterAlias);
             assertThat(localCluster.getIndexExpression(), equalTo("logs*"));
-            assertClusterStatusAndHasNullCounts(localCluster, EsqlExecutionInfo.Cluster.Status.RUNNING);
+            assertClusterStatusAndShardCounts(localCluster, EsqlExecutionInfo.Cluster.Status.RUNNING);
 
             EsqlExecutionInfo.Cluster remote1Cluster = executionInfo.getCluster(remote1Alias);
             assertThat(remote1Cluster.getIndexExpression(), equalTo("*"));
-            assertClusterStatusAndHasNullCounts(remote1Cluster, EsqlExecutionInfo.Cluster.Status.RUNNING);
+            assertClusterStatusAndShardCounts(remote1Cluster, EsqlExecutionInfo.Cluster.Status.RUNNING);
 
             EsqlExecutionInfo.Cluster remote2Cluster = executionInfo.getCluster(remote2Alias);
             assertThat(remote2Cluster.getIndexExpression(), equalTo("mylogs1,mylogs2,logs*"));
-            assertClusterStatusAndHasNullCounts(remote2Cluster, EsqlExecutionInfo.Cluster.Status.RUNNING);
+            assertClusterStatusAndShardCounts(remote2Cluster, EsqlExecutionInfo.Cluster.Status.RUNNING);
         }
     }
 
     public void testUpdateExecutionInfoWithClustersWithNoMatchingIndices() {
+        final String localClusterAlias = RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY;
+        final String remote1Alias = "remote1";
+        final String remote2Alias = "remote2";
         // all clusters present in EsIndex, so no updates to EsqlExecutionInfo should happen
         {
-            final String localClusterAlias = RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY;
-            final String remote1Alias = "remote1";
-            final String remote2Alias = "remote2";
             EsqlExecutionInfo executionInfo = new EsqlExecutionInfo(true);
             executionInfo.swapCluster(localClusterAlias, (k, v) -> new EsqlExecutionInfo.Cluster(localClusterAlias, "logs*", false));
             executionInfo.swapCluster(remote1Alias, (k, v) -> new EsqlExecutionInfo.Cluster(remote1Alias, "*", true));
@@ -139,28 +222,25 @@ public void testUpdateExecutionInfoWithClustersWithNoMatchingIndices() {
                     IndexMode.STANDARD
                 )
             );
-            IndexResolution indexResolution = IndexResolution.valid(esIndex, Set.of());
+            IndexResolution indexResolution = IndexResolution.valid(esIndex, Map.of());
 
             EsqlSession.updateExecutionInfoWithClustersWithNoMatchingIndices(executionInfo, indexResolution);
 
             EsqlExecutionInfo.Cluster localCluster = executionInfo.getCluster(localClusterAlias);
             assertThat(localCluster.getIndexExpression(), equalTo("logs*"));
-            assertClusterStatusAndHasNullCounts(localCluster, EsqlExecutionInfo.Cluster.Status.RUNNING);
+            assertClusterStatusAndShardCounts(localCluster, EsqlExecutionInfo.Cluster.Status.RUNNING);
 
             EsqlExecutionInfo.Cluster remote1Cluster = executionInfo.getCluster(remote1Alias);
             assertThat(remote1Cluster.getIndexExpression(), equalTo("*"));
-            assertClusterStatusAndHasNullCounts(remote1Cluster, EsqlExecutionInfo.Cluster.Status.RUNNING);
+            assertClusterStatusAndShardCounts(remote1Cluster, EsqlExecutionInfo.Cluster.Status.RUNNING);
 
             EsqlExecutionInfo.Cluster remote2Cluster = executionInfo.getCluster(remote2Alias);
             assertThat(remote2Cluster.getIndexExpression(), equalTo("mylogs1,mylogs2,logs*"));
-            assertClusterStatusAndHasNullCounts(remote2Cluster, EsqlExecutionInfo.Cluster.Status.RUNNING);
+            assertClusterStatusAndShardCounts(remote2Cluster, EsqlExecutionInfo.Cluster.Status.RUNNING);
         }
 
         // remote1 is missing from EsIndex info, so it should be updated and marked as SKIPPED with 0 total shards, 0 took time, etc.
         {
-            final String localClusterAlias = RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY;
-            final String remote1Alias = "remote1";
-            final String remote2Alias = "remote2";
             EsqlExecutionInfo executionInfo = new EsqlExecutionInfo(true);
             executionInfo.swapCluster(localClusterAlias, (k, v) -> new EsqlExecutionInfo.Cluster(localClusterAlias, "logs*", false));
             executionInfo.swapCluster(remote1Alias, (k, v) -> new EsqlExecutionInfo.Cluster(remote1Alias, "*", true));
@@ -180,13 +260,13 @@ public void testUpdateExecutionInfoWithClustersWithNoMatchingIndices() {
                     IndexMode.STANDARD
                 )
             );
-            IndexResolution indexResolution = IndexResolution.valid(esIndex, Set.of());
+            IndexResolution indexResolution = IndexResolution.valid(esIndex, Map.of());
 
             EsqlSession.updateExecutionInfoWithClustersWithNoMatchingIndices(executionInfo, indexResolution);
 
             EsqlExecutionInfo.Cluster localCluster = executionInfo.getCluster(localClusterAlias);
             assertThat(localCluster.getIndexExpression(), equalTo("logs*"));
-            assertClusterStatusAndHasNullCounts(localCluster, EsqlExecutionInfo.Cluster.Status.RUNNING);
+            assertClusterStatusAndShardCounts(localCluster, EsqlExecutionInfo.Cluster.Status.RUNNING);
 
             EsqlExecutionInfo.Cluster remote1Cluster = executionInfo.getCluster(remote1Alias);
             assertThat(remote1Cluster.getIndexExpression(), equalTo("*"));
@@ -199,14 +279,11 @@ public void testUpdateExecutionInfoWithClustersWithNoMatchingIndices() {
 
             EsqlExecutionInfo.Cluster remote2Cluster = executionInfo.getCluster(remote2Alias);
             assertThat(remote2Cluster.getIndexExpression(), equalTo("mylogs1,mylogs2,logs*"));
-            assertClusterStatusAndHasNullCounts(remote2Cluster, EsqlExecutionInfo.Cluster.Status.RUNNING);
+            assertClusterStatusAndShardCounts(remote2Cluster, EsqlExecutionInfo.Cluster.Status.RUNNING);
         }
 
         // all remotes are missing from EsIndex info, so they should be updated and marked as SKIPPED with 0 total shards, 0 took time, etc.
         {
-            final String localClusterAlias = RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY;
-            final String remote1Alias = "remote1";
-            final String remote2Alias = "remote2";
             EsqlExecutionInfo executionInfo = new EsqlExecutionInfo(true);
             executionInfo.swapCluster(localClusterAlias, (k, v) -> new EsqlExecutionInfo.Cluster(localClusterAlias, "logs*", false));
             executionInfo.swapCluster(remote1Alias, (k, v) -> new EsqlExecutionInfo.Cluster(remote1Alias, "*", true));
@@ -217,21 +294,21 @@ public void testUpdateExecutionInfoWithClustersWithNoMatchingIndices() {
                 randomMapping(),
                 Map.of("logs-a", IndexMode.STANDARD)
             );
-            // mark remote1 as unavailable
-            IndexResolution indexResolution = IndexResolution.valid(esIndex, Set.of(remote1Alias));
+            // remote1 is unavailable
+            var failure = new FieldCapabilitiesFailure(new String[] { "logs-a" }, new NoSeedNodeLeftException("unable to connect"));
+            IndexResolution indexResolution = IndexResolution.valid(esIndex, Map.of(remote1Alias, failure));
 
             EsqlSession.updateExecutionInfoWithClustersWithNoMatchingIndices(executionInfo, indexResolution);
 
             EsqlExecutionInfo.Cluster localCluster = executionInfo.getCluster(localClusterAlias);
             assertThat(localCluster.getIndexExpression(), equalTo("logs*"));
-            assertClusterStatusAndHasNullCounts(localCluster, EsqlExecutionInfo.Cluster.Status.RUNNING);
+            assertClusterStatusAndShardCounts(localCluster, EsqlExecutionInfo.Cluster.Status.RUNNING);
 
             EsqlExecutionInfo.Cluster remote1Cluster = executionInfo.getCluster(remote1Alias);
             assertThat(remote1Cluster.getIndexExpression(), equalTo("*"));
-            // remote1 is left as RUNNING, since another method (updateExecutionInfoWithUnavailableClusters) not under test changes status
+            // since remote1 is in the unavailable Map (passed to IndexResolution.valid), it's status will not be changed
+            // by updateExecutionInfoWithClustersWithNoMatchingIndices (it is handled in updateExecutionInfoWithUnavailableClusters)
             assertThat(remote1Cluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.RUNNING));
-            assertNull(remote1Cluster.getTook());
-            assertNull(remote1Cluster.getTotalShards());
 
             EsqlExecutionInfo.Cluster remote2Cluster = executionInfo.getCluster(remote2Alias);
             assertThat(remote2Cluster.getIndexExpression(), equalTo("mylogs1,mylogs2,logs*"));
@@ -242,6 +319,25 @@ public void testUpdateExecutionInfoWithClustersWithNoMatchingIndices() {
             assertThat(remote2Cluster.getSkippedShards(), equalTo(0));
             assertThat(remote2Cluster.getFailedShards(), equalTo(0));
         }
+
+        // all remotes are missing from EsIndex info. Since one is configured with skip_unavailable=false,
+        // an exception should be thrown
+        {
+            EsqlExecutionInfo executionInfo = new EsqlExecutionInfo(true);
+            executionInfo.swapCluster(localClusterAlias, (k, v) -> new EsqlExecutionInfo.Cluster(localClusterAlias, "logs*"));
+            executionInfo.swapCluster(remote1Alias, (k, v) -> new EsqlExecutionInfo.Cluster(remote1Alias, "*", true));
+            executionInfo.swapCluster(remote2Alias, (k, v) -> new EsqlExecutionInfo.Cluster(remote2Alias, "mylogs1,mylogs2,logs*", false));
+
+            EsIndex esIndex = new EsIndex(
+                "logs*,remote2:mylogs1,remote2:mylogs2,remote2:logs*",
+                randomMapping(),
+                Map.of("logs-a", IndexMode.STANDARD)
+            );
+
+            var failure = new FieldCapabilitiesFailure(new String[] { "logs-a" }, new NoSeedNodeLeftException("unable to connect"));
+            IndexResolution indexResolution = IndexResolution.valid(esIndex, Map.of(remote1Alias, failure));
+            EsqlSession.updateExecutionInfoWithClustersWithNoMatchingIndices(executionInfo, indexResolution);
+        }
     }
 
     public void testUpdateExecutionInfoAtEndOfPlanning() {
@@ -288,13 +384,22 @@ public void testUpdateExecutionInfoAtEndOfPlanning() {
         assertNull(remote2Cluster.getTook());
     }
 
-    private void assertClusterStatusAndHasNullCounts(EsqlExecutionInfo.Cluster cluster, EsqlExecutionInfo.Cluster.Status status) {
+    private void assertClusterStatusAndShardCounts(EsqlExecutionInfo.Cluster cluster, EsqlExecutionInfo.Cluster.Status status) {
         assertThat(cluster.getStatus(), equalTo(status));
         assertNull(cluster.getTook());
-        assertNull(cluster.getTotalShards());
-        assertNull(cluster.getSuccessfulShards());
-        assertNull(cluster.getSkippedShards());
-        assertNull(cluster.getFailedShards());
+        if (status == EsqlExecutionInfo.Cluster.Status.RUNNING) {
+            assertNull(cluster.getTotalShards());
+            assertNull(cluster.getSuccessfulShards());
+            assertNull(cluster.getSkippedShards());
+            assertNull(cluster.getFailedShards());
+        } else if (status == EsqlExecutionInfo.Cluster.Status.SKIPPED) {
+            assertThat(cluster.getTotalShards(), equalTo(0));
+            assertThat(cluster.getSuccessfulShards(), equalTo(0));
+            assertThat(cluster.getSkippedShards(), equalTo(0));
+            assertThat(cluster.getFailedShards(), equalTo(0));
+        } else {
+            fail("Unexpected status: " + status);
+        }
     }
 
     private static Map<String, EsField> randomMapping() {
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverTests.java
index 51497b5ca5093..d6e410305afaa 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverTests.java
@@ -15,6 +15,7 @@
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 
 import static org.hamcrest.Matchers.equalTo;
@@ -33,8 +34,8 @@ public void testDetermineUnavailableRemoteClusters() {
                 )
             );
 
-            Set<String> unavailableClusters = IndexResolver.determineUnavailableRemoteClusters(failures);
-            assertThat(unavailableClusters, equalTo(Set.of("remote1", "remote2")));
+            Map<String, FieldCapabilitiesFailure> unavailableClusters = IndexResolver.determineUnavailableRemoteClusters(failures);
+            assertThat(unavailableClusters.keySet(), equalTo(Set.of("remote1", "remote2")));
         }
 
         // one cluster with "remote unavailable" with two failures
@@ -43,8 +44,8 @@ public void testDetermineUnavailableRemoteClusters() {
             failures.add(new FieldCapabilitiesFailure(new String[] { "remote2:mylogs1" }, new NoSuchRemoteClusterException("remote2")));
             failures.add(new FieldCapabilitiesFailure(new String[] { "remote2:mylogs1" }, new NoSeedNodeLeftException("no seed node")));
 
-            Set<String> unavailableClusters = IndexResolver.determineUnavailableRemoteClusters(failures);
-            assertThat(unavailableClusters, equalTo(Set.of("remote2")));
+            Map<String, FieldCapabilitiesFailure> unavailableClusters = IndexResolver.determineUnavailableRemoteClusters(failures);
+            assertThat(unavailableClusters.keySet(), equalTo(Set.of("remote2")));
         }
 
         // two clusters, one "remote unavailable" type exceptions and one with another type
@@ -57,23 +58,23 @@ public void testDetermineUnavailableRemoteClusters() {
                     new IllegalStateException("Unable to open any connections")
                 )
             );
-            Set<String> unavailableClusters = IndexResolver.determineUnavailableRemoteClusters(failures);
-            assertThat(unavailableClusters, equalTo(Set.of("remote2")));
+            Map<String, FieldCapabilitiesFailure> unavailableClusters = IndexResolver.determineUnavailableRemoteClusters(failures);
+            assertThat(unavailableClusters.keySet(), equalTo(Set.of("remote2")));
         }
 
         // one cluster1 with exception not known to indicate "remote unavailable"
         {
             List<FieldCapabilitiesFailure> failures = new ArrayList<>();
             failures.add(new FieldCapabilitiesFailure(new String[] { "remote1:mylogs1" }, new RuntimeException("foo")));
-            Set<String> unavailableClusters = IndexResolver.determineUnavailableRemoteClusters(failures);
-            assertThat(unavailableClusters, equalTo(Set.of()));
+            Map<String, FieldCapabilitiesFailure> unavailableClusters = IndexResolver.determineUnavailableRemoteClusters(failures);
+            assertThat(unavailableClusters.keySet(), equalTo(Set.of()));
         }
 
         // empty failures list
         {
             List<FieldCapabilitiesFailure> failures = new ArrayList<>();
-            Set<String> unavailableClusters = IndexResolver.determineUnavailableRemoteClusters(failures);
-            assertThat(unavailableClusters, equalTo(Set.of()));
+            Map<String, FieldCapabilitiesFailure> unavailableClusters = IndexResolver.determineUnavailableRemoteClusters(failures);
+            assertThat(unavailableClusters.keySet(), equalTo(Set.of()));
         }
     }
 }
diff --git a/x-pack/plugin/inference/build.gradle b/x-pack/plugin/inference/build.gradle
index 28e1405cf7b97..6791aad6619d3 100644
--- a/x-pack/plugin/inference/build.gradle
+++ b/x-pack/plugin/inference/build.gradle
@@ -32,7 +32,7 @@ versions << [
 ]
 
 dependencies {
-  implementation project(path: ':libs:elasticsearch-logging')
+  implementation project(path: ':libs:logging')
   compileOnly project(":server")
   compileOnly project(path: xpackModule('core'))
   testImplementation(testArtifact(project(xpackModule('core'))))
diff --git a/x-pack/plugin/ml-package-loader/build.gradle b/x-pack/plugin/ml-package-loader/build.gradle
index bdd1e54f20c86..122ad396b507d 100644
--- a/x-pack/plugin/ml-package-loader/build.gradle
+++ b/x-pack/plugin/ml-package-loader/build.gradle
@@ -18,7 +18,7 @@ esplugin {
 }
 
 dependencies {
-  implementation project(path: ':libs:elasticsearch-logging')
+  implementation project(path: ':libs:logging')
   compileOnly project(":server")
   compileOnly project(path: xpackModule('core'))
   testImplementation(testArtifact(project(xpackModule('core'))))
diff --git a/x-pack/plugin/ml/build.gradle b/x-pack/plugin/ml/build.gradle
index 706d7ea73aea9..e79a771293392 100644
--- a/x-pack/plugin/ml/build.gradle
+++ b/x-pack/plugin/ml/build.gradle
@@ -76,7 +76,7 @@ dependencies {
   testImplementation(testArtifact(project(xpackModule('security'))))
   testImplementation project(path: xpackModule('wildcard'))
   // ml deps
-  api project(':libs:elasticsearch-grok')
+  api project(':libs:grok')
   api project(':modules:lang-mustache')
   api "org.apache.commons:commons-math3:3.6.1"
   api "com.ibm.icu:icu4j:${versions.icu4j}"
diff --git a/x-pack/plugin/searchable-snapshots/build.gradle b/x-pack/plugin/searchable-snapshots/build.gradle
index 4e309499445e6..747e94b0e8d8d 100644
--- a/x-pack/plugin/searchable-snapshots/build.gradle
+++ b/x-pack/plugin/searchable-snapshots/build.gradle
@@ -15,7 +15,7 @@ base {
 dependencies {
   compileOnly project(path: xpackModule('core'))
   compileOnly project(path: xpackModule('blob-cache'))
-  compileOnly project(path: ':libs:elasticsearch-native')
+  compileOnly project(path: ':libs:native')
   testImplementation(testArtifact(project(xpackModule('blob-cache'))))
   internalClusterTestImplementation(testArtifact(project(xpackModule('core'))))
   internalClusterTestImplementation(project(path: xpackModule('shutdown')))
diff --git a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityEsqlIT.java b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityEsqlIT.java
index 1a236ccb6aa06..d5b3141b539eb 100644
--- a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityEsqlIT.java
+++ b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityEsqlIT.java
@@ -34,9 +34,12 @@
 import java.util.Base64;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
@@ -491,6 +494,10 @@ public void testCrossClusterQueryWithRemoteDLSAndFLS() throws Exception {
         assertThat(flatList, containsInAnyOrder("engineering"));
     }
 
+    /**
+     * Note: invalid_remote is "invalid" because it has a bogus API key and the cluster does not exist (cannot be connected to)
+     */
+    @SuppressWarnings("unchecked")
     public void testCrossClusterQueryAgainstInvalidRemote() throws Exception {
         configureRemoteCluster();
         populateData();
@@ -512,22 +519,53 @@ public void testCrossClusterQueryAgainstInvalidRemote() throws Exception {
         );
 
         // invalid remote with local index should return local results
-        var q = "FROM invalid_remote:employees,employees |  SORT emp_id DESC | LIMIT 10";
-        Response response = performRequestWithRemoteSearchUser(esqlRequest(q));
-        assertLocalOnlyResults(response);
-
-        // only calling an invalid remote should error
-        ResponseException error = expectThrows(ResponseException.class, () -> {
-            var q2 = "FROM invalid_remote:employees |  SORT emp_id DESC | LIMIT 10";
-            performRequestWithRemoteSearchUser(esqlRequest(q2));
-        });
-
-        if (skipUnavailable == false) {
-            assertThat(error.getResponse().getStatusLine().getStatusCode(), equalTo(401));
-            assertThat(error.getMessage(), containsString("unable to find apikey"));
-        } else {
-            assertThat(error.getResponse().getStatusLine().getStatusCode(), equalTo(500));
-            assertThat(error.getMessage(), containsString("Unable to connect to [invalid_remote]"));
+        {
+            var q = "FROM invalid_remote:employees,employees | SORT emp_id DESC | LIMIT 10";
+            Response response = performRequestWithRemoteSearchUser(esqlRequest(q));
+            // TODO: when skip_unavailable=false for invalid_remote, a fatal exception should be thrown
+            // this does not yet happen because field-caps returns nothing for this cluster, rather
+            // than an error, so the current code cannot detect that error. Follow on PR will handle this.
+            assertLocalOnlyResults(response);
+        }
+
+        {
+            var q = "FROM invalid_remote:employees | SORT emp_id DESC | LIMIT 10";
+            // errors from invalid remote should be ignored if the cluster is marked with skip_unavailable=true
+            if (skipUnavailable) {
+                // expected response:
+                // {"took":1,"columns":[],"values":[],"_clusters":{"total":1,"successful":0,"running":0,"skipped":1,"partial":0,
+                // "failed":0,"details":{"invalid_remote":{"status":"skipped","indices":"employees","took":1,"_shards":
+                // {"total":0,"successful":0,"skipped":0,"failed":0},"failures":[{"shard":-1,"index":null,"reason":
+                // {"type":"remote_transport_exception",
+                // "reason":"[connect_transport_exception - unable to connect to remote cluster]"}}]}}}}
+                Response response = performRequestWithRemoteSearchUser(esqlRequest(q));
+                assertOK(response);
+                Map<String, Object> responseAsMap = entityAsMap(response);
+                List<?> columns = (List<?>) responseAsMap.get("columns");
+                List<?> values = (List<?>) responseAsMap.get("values");
+                assertThat(columns.size(), equalTo(1));
+                Map<String, ?> column1 = (Map<String, ?>) columns.get(0);
+                assertThat(column1.get("name").toString(), equalTo("<no-fields>"));
+                assertThat(values.size(), equalTo(0));
+                Map<String, ?> clusters = (Map<String, ?>) responseAsMap.get("_clusters");
+                Map<String, ?> details = (Map<String, ?>) clusters.get("details");
+                Map<String, ?> invalidRemoteEntry = (Map<String, ?>) details.get("invalid_remote");
+                assertThat(invalidRemoteEntry.get("status").toString(), equalTo("skipped"));
+                List<?> failures = (List<?>) invalidRemoteEntry.get("failures");
+                assertThat(failures.size(), equalTo(1));
+                Map<String, ?> failuresMap = (Map<String, ?>) failures.get(0);
+                Map<String, ?> reason = (Map<String, ?>) failuresMap.get("reason");
+                assertThat(reason.get("type").toString(), equalTo("remote_transport_exception"));
+                assertThat(reason.get("reason").toString(), containsString("unable to connect to remote cluster"));
+
+            } else {
+                // errors from invalid remote should throw an exception if the cluster is marked with skip_unavailable=false
+                ResponseException error = expectThrows(ResponseException.class, () -> {
+                    final Response response1 = performRequestWithRemoteSearchUser(esqlRequest(q));
+                });
+                assertThat(error.getResponse().getStatusLine().getStatusCode(), equalTo(401));
+                assertThat(error.getMessage(), containsString("unable to find apikey"));
+            }
         }
     }
 
@@ -887,7 +925,16 @@ public void testAlias() throws Exception {
             Request request = esqlRequest("FROM " + index + " | KEEP emp_id | SORT emp_id | LIMIT 100");
             ResponseException error = expectThrows(ResponseException.class, () -> performRequestWithRemoteSearchUser(request));
             assertThat(error.getResponse().getStatusLine().getStatusCode(), equalTo(400));
-            assertThat(error.getMessage(), containsString(" Unknown index [" + index + "]"));
+            String expectedIndexExpressionInError = index.replace("*", "my_remote_cluster");
+            Pattern p = Pattern.compile("Unknown index \\[([^\\]]+)\\]");
+            Matcher m = p.matcher(error.getMessage());
+            assertTrue("Pattern matcher to parse error message did not find matching string: " + error.getMessage(), m.find());
+            String unknownIndexExpressionInErrorMessage = m.group(1);
+            Set<String> actualUnknownIndexes = org.elasticsearch.common.Strings.commaDelimitedListToSet(
+                unknownIndexExpressionInErrorMessage
+            );
+            Set<String> expectedUnknownIndexes = org.elasticsearch.common.Strings.commaDelimitedListToSet(expectedIndexExpressionInError);
+            assertThat(actualUnknownIndexes, equalTo(expectedUnknownIndexes));
         }
 
         for (var index : List.of(
@@ -920,6 +967,7 @@ protected Request esqlRequest(String command) throws IOException {
         XContentBuilder body = JsonXContent.contentBuilder();
         body.startObject();
         body.field("query", command);
+        body.field("include_ccs_metadata", true);
         if (Build.current().isSnapshot() && randomBoolean()) {
             Settings.Builder settings = Settings.builder();
             if (randomBoolean()) {
diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/RoleMappingFileSettingsIT.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/RoleMappingFileSettingsIT.java
index fdd854e7a9673..9e36055e917a6 100644
--- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/RoleMappingFileSettingsIT.java
+++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/RoleMappingFileSettingsIT.java
@@ -204,6 +204,29 @@ public void clusterChanged(ClusterChangedEvent event) {
         return new Tuple<>(savedClusterState, metadataVersion);
     }
 
+    // Wait for any file metadata
+    public static Tuple<CountDownLatch, AtomicLong> setupClusterStateListener(String node) {
+        ClusterService clusterService = internalCluster().clusterService(node);
+        CountDownLatch savedClusterState = new CountDownLatch(1);
+        AtomicLong metadataVersion = new AtomicLong(-1);
+        clusterService.addListener(new ClusterStateListener() {
+            @Override
+            public void clusterChanged(ClusterChangedEvent event) {
+                ReservedStateMetadata reservedState = event.state().metadata().reservedStateMetadata().get(FileSettingsService.NAMESPACE);
+                if (reservedState != null) {
+                    ReservedStateHandlerMetadata handlerMetadata = reservedState.handlers().get(ReservedRoleMappingAction.NAME);
+                    if (handlerMetadata != null) {
+                        clusterService.removeListener(this);
+                        metadataVersion.set(event.state().metadata().version());
+                        savedClusterState.countDown();
+                    }
+                }
+            }
+        });
+
+        return new Tuple<>(savedClusterState, metadataVersion);
+    }
+
     public static Tuple<CountDownLatch, AtomicLong> setupClusterStateListenerForCleanup(String node) {
         ClusterService clusterService = internalCluster().clusterService(node);
         CountDownLatch savedClusterState = new CountDownLatch(1);
diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/support/CleanupRoleMappingDuplicatesMigrationIT.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/support/CleanupRoleMappingDuplicatesMigrationIT.java
new file mode 100644
index 0000000000000..63c510062bdad
--- /dev/null
+++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/support/CleanupRoleMappingDuplicatesMigrationIT.java
@@ -0,0 +1,417 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.security.support;
+
+import org.elasticsearch.action.ActionFuture;
+import org.elasticsearch.cluster.ClusterChangedEvent;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ClusterStateListener;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.reservedstate.service.FileSettingsService;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.test.SecurityIntegTestCase;
+import org.elasticsearch.xpack.core.security.action.UpdateIndexMigrationVersionAction;
+import org.elasticsearch.xpack.core.security.action.rolemapping.DeleteRoleMappingAction;
+import org.elasticsearch.xpack.core.security.action.rolemapping.DeleteRoleMappingRequest;
+import org.elasticsearch.xpack.core.security.action.rolemapping.DeleteRoleMappingResponse;
+import org.elasticsearch.xpack.core.security.action.rolemapping.GetRoleMappingsAction;
+import org.elasticsearch.xpack.core.security.action.rolemapping.GetRoleMappingsRequest;
+import org.elasticsearch.xpack.core.security.action.rolemapping.GetRoleMappingsResponse;
+import org.elasticsearch.xpack.core.security.action.rolemapping.PutRoleMappingAction;
+import org.elasticsearch.xpack.core.security.action.rolemapping.PutRoleMappingRequest;
+import org.elasticsearch.xpack.core.security.action.rolemapping.PutRoleMappingResponse;
+import org.elasticsearch.xpack.core.security.authc.support.mapper.ExpressionRoleMapping;
+import org.elasticsearch.xpack.core.security.authc.support.mapper.expressiondsl.FieldExpression;
+import org.junit.Before;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+
+import static org.elasticsearch.integration.RoleMappingFileSettingsIT.setupClusterStateListener;
+import static org.elasticsearch.integration.RoleMappingFileSettingsIT.writeJSONFile;
+import static org.elasticsearch.xpack.core.security.action.UpdateIndexMigrationVersionAction.MIGRATION_VERSION_CUSTOM_DATA_KEY;
+import static org.elasticsearch.xpack.core.security.action.UpdateIndexMigrationVersionAction.MIGRATION_VERSION_CUSTOM_KEY;
+import static org.elasticsearch.xpack.core.security.test.TestRestrictedIndices.INTERNAL_SECURITY_MAIN_INDEX_7;
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.lessThan;
+import static org.hamcrest.Matchers.lessThanOrEqualTo;
+
+@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, autoManageMasterNodes = false)
+public class CleanupRoleMappingDuplicatesMigrationIT extends SecurityIntegTestCase {
+
+    private final AtomicLong versionCounter = new AtomicLong(1);
+
+    @Before
+    public void resetVersion() {
+        versionCounter.set(1);
+    }
+
+    private static final String TEST_JSON_WITH_ROLE_MAPPINGS = """
+        {
+             "metadata": {
+                 "version": "%s",
+                 "compatibility": "8.4.0"
+             },
+             "state": {
+                 "role_mappings": {
+                       "everyone_kibana_alone": {
+                          "enabled": true,
+                          "roles": [ "kibana_user" ],
+                          "rules": { "field": { "username": "*" } },
+                          "metadata": {
+                             "uuid" : "b9a59ba9-6b92-4be2-bb8d-02bb270cb3a7",
+                             "_foo": "something"
+                          }
+                       },
+                       "everyone_fleet_alone": {
+                          "enabled": false,
+                          "roles": [ "fleet_user" ],
+                          "rules": { "field": { "username": "*" } },
+                          "metadata": {
+                             "uuid" : "b9a59ba9-6b92-4be3-bb8d-02bb270cb3a7",
+                             "_foo": "something_else"
+                          }
+                       }
+                 }
+             }
+        }""";
+
+    private static final String TEST_JSON_WITH_FALLBACK_NAME = """
+        {
+             "metadata": {
+                 "version": "%s",
+                 "compatibility": "8.4.0"
+             },
+             "state": {
+                 "role_mappings": {
+                       "name_not_available_after_deserialization": {
+                          "enabled": true,
+                          "roles": [ "kibana_user", "kibana_admin" ],
+                          "rules": { "field": { "username": "*" } },
+                          "metadata": {
+                             "uuid" : "b9a59ba9-6b92-4be2-bb8d-02bb270cb3a7",
+                             "_foo": "something"
+                          }
+                       }
+                 }
+             }
+        }""";
+
+    private static final String TEST_JSON_WITH_EMPTY_ROLE_MAPPINGS = """
+        {
+             "metadata": {
+                 "version": "%s",
+                 "compatibility": "8.4.0"
+             },
+             "state": {
+                 "role_mappings": {}
+             }
+        }""";
+
+    public void testMigrationSuccessful() throws Exception {
+        internalCluster().setBootstrapMasterNodeIndex(0);
+        ensureGreen();
+        final String masterNode = internalCluster().getMasterName();
+
+        // Create a native role mapping to create security index and trigger migration (skipped initially)
+        createNativeRoleMapping("everyone_kibana_alone");
+        createNativeRoleMapping("everyone_fleet_alone");
+        createNativeRoleMapping("dont_clean_this_up");
+        assertAllRoleMappings("everyone_kibana_alone", "everyone_fleet_alone", "dont_clean_this_up");
+
+        // Wait for file watcher to start
+        awaitFileSettingsWatcher();
+        // Setup listener to wait for role mapping
+        var fileBasedRoleMappingsWrittenListener = setupClusterStateListener(masterNode, "everyone_kibana_alone");
+        // Write role mappings
+        writeJSONFile(masterNode, TEST_JSON_WITH_ROLE_MAPPINGS, logger, versionCounter);
+        assertTrue(fileBasedRoleMappingsWrittenListener.v1().await(20, TimeUnit.SECONDS));
+        waitForMigrationCompletion(SecurityMigrations.CLEANUP_ROLE_MAPPING_DUPLICATES_MIGRATION_VERSION);
+
+        // First migration is on a new index, so should skip all migrations. If we reset, it should re-trigger and run all migrations
+        resetMigration();
+
+        // Wait for the first migration to finish
+        waitForMigrationCompletion(SecurityMigrations.CLEANUP_ROLE_MAPPING_DUPLICATES_MIGRATION_VERSION);
+
+        assertAllRoleMappings(
+            "everyone_kibana_alone" + ExpressionRoleMapping.READ_ONLY_ROLE_MAPPING_SUFFIX,
+            "everyone_fleet_alone" + ExpressionRoleMapping.READ_ONLY_ROLE_MAPPING_SUFFIX,
+            "dont_clean_this_up"
+        );
+    }
+
+    public void testMigrationSuccessfulNoOverlap() throws Exception {
+        internalCluster().setBootstrapMasterNodeIndex(0);
+        ensureGreen();
+        final String masterNode = internalCluster().getMasterName();
+
+        // Create a native role mapping to create security index and trigger migration (skipped initially)
+        createNativeRoleMapping("some_native_mapping");
+        createNativeRoleMapping("some_other_native_mapping");
+        assertAllRoleMappings("some_native_mapping", "some_other_native_mapping");
+
+        // Wait for file watcher to start
+        awaitFileSettingsWatcher();
+        // Setup listener to wait for role mapping
+        var fileBasedRoleMappingsWrittenListener = setupClusterStateListener(masterNode, "everyone_kibana_alone");
+        // Write role mappings with fallback name, this should block any security migration
+        writeJSONFile(masterNode, TEST_JSON_WITH_ROLE_MAPPINGS, logger, versionCounter);
+        assertTrue(fileBasedRoleMappingsWrittenListener.v1().await(20, TimeUnit.SECONDS));
+        waitForMigrationCompletion(SecurityMigrations.CLEANUP_ROLE_MAPPING_DUPLICATES_MIGRATION_VERSION);
+
+        // First migration is on a new index, so should skip all migrations. If we reset, it should re-trigger and run all migrations
+        resetMigration();
+
+        // Wait for the first migration to finish
+        waitForMigrationCompletion(SecurityMigrations.CLEANUP_ROLE_MAPPING_DUPLICATES_MIGRATION_VERSION);
+
+        assertAllRoleMappings(
+            "everyone_kibana_alone" + ExpressionRoleMapping.READ_ONLY_ROLE_MAPPING_SUFFIX,
+            "everyone_fleet_alone" + ExpressionRoleMapping.READ_ONLY_ROLE_MAPPING_SUFFIX,
+            "some_native_mapping",
+            "some_other_native_mapping"
+        );
+    }
+
+    public void testMigrationSuccessfulNoNative() throws Exception {
+        internalCluster().setBootstrapMasterNodeIndex(0);
+        ensureGreen();
+        final String masterNode = internalCluster().getMasterName();
+
+        // Create a native role mapping to create security index and trigger migration (skipped initially)
+        // Then delete it to test an empty role mapping store
+        createNativeRoleMapping("some_native_mapping");
+        deleteNativeRoleMapping("some_native_mapping");
+        // Wait for file watcher to start
+        awaitFileSettingsWatcher();
+        // Setup listener to wait for role mapping
+        var fileBasedRoleMappingsWrittenListener = setupClusterStateListener(masterNode, "everyone_kibana_alone");
+        // Write role mappings with fallback name, this should block any security migration
+        writeJSONFile(masterNode, TEST_JSON_WITH_ROLE_MAPPINGS, logger, versionCounter);
+        assertTrue(fileBasedRoleMappingsWrittenListener.v1().await(20, TimeUnit.SECONDS));
+        waitForMigrationCompletion(SecurityMigrations.CLEANUP_ROLE_MAPPING_DUPLICATES_MIGRATION_VERSION);
+
+        // First migration is on a new index, so should skip all migrations. If we reset, it should re-trigger and run all migrations
+        resetMigration();
+
+        // Wait for the first migration to finish
+        waitForMigrationCompletion(SecurityMigrations.CLEANUP_ROLE_MAPPING_DUPLICATES_MIGRATION_VERSION);
+
+        assertAllRoleMappings(
+            "everyone_kibana_alone" + ExpressionRoleMapping.READ_ONLY_ROLE_MAPPING_SUFFIX,
+            "everyone_fleet_alone" + ExpressionRoleMapping.READ_ONLY_ROLE_MAPPING_SUFFIX
+        );
+    }
+
+    public void testMigrationFallbackNamePreCondition() throws Exception {
+        internalCluster().setBootstrapMasterNodeIndex(0);
+        ensureGreen();
+        final String masterNode = internalCluster().getMasterName();
+        // Wait for file watcher to start
+        awaitFileSettingsWatcher();
+
+        // Setup listener to wait for role mapping
+        var nameNotAvailableListener = setupClusterStateListener(masterNode, "name_not_available_after_deserialization");
+        // Write role mappings with fallback name, this should block any security migration
+        writeJSONFile(masterNode, TEST_JSON_WITH_FALLBACK_NAME, logger, versionCounter);
+        assertTrue(nameNotAvailableListener.v1().await(20, TimeUnit.SECONDS));
+
+        // Create a native role mapping to create security index and trigger migration
+        createNativeRoleMapping("everyone_fleet_alone");
+        waitForMigrationCompletion(SecurityMigrations.CLEANUP_ROLE_MAPPING_DUPLICATES_MIGRATION_VERSION);
+        // First migration is on a new index, so should skip all migrations. If we reset, it should re-trigger and run all migrations
+        resetMigration();
+        // Wait for the first migration to finish
+        waitForMigrationCompletion(SecurityMigrations.CLEANUP_ROLE_MAPPING_DUPLICATES_MIGRATION_VERSION - 1);
+
+        // Make sure migration didn't run yet (blocked by the fallback name)
+        assertMigrationLessThan(SecurityMigrations.CLEANUP_ROLE_MAPPING_DUPLICATES_MIGRATION_VERSION);
+        ClusterService clusterService = internalCluster().getInstance(ClusterService.class);
+        SecurityIndexManager.RoleMappingsCleanupMigrationStatus status = SecurityIndexManager.getRoleMappingsCleanupMigrationStatus(
+            clusterService.state(),
+            SecurityMigrations.CLEANUP_ROLE_MAPPING_DUPLICATES_MIGRATION_VERSION - 1
+        );
+        assertThat(status, equalTo(SecurityIndexManager.RoleMappingsCleanupMigrationStatus.NOT_READY));
+
+        // Write file without fallback name in it to unblock migration
+        writeJSONFile(masterNode, TEST_JSON_WITH_ROLE_MAPPINGS, logger, versionCounter);
+        waitForMigrationCompletion(SecurityMigrations.CLEANUP_ROLE_MAPPING_DUPLICATES_MIGRATION_VERSION);
+    }
+
+    public void testSkipMigrationNoFileBasedMappings() throws Exception {
+        internalCluster().setBootstrapMasterNodeIndex(0);
+        ensureGreen();
+        // Create a native role mapping to create security index and trigger migration (skipped initially)
+        createNativeRoleMapping("everyone_kibana_alone");
+        createNativeRoleMapping("everyone_fleet_alone");
+        assertAllRoleMappings("everyone_kibana_alone", "everyone_fleet_alone");
+
+        waitForMigrationCompletion(SecurityMigrations.CLEANUP_ROLE_MAPPING_DUPLICATES_MIGRATION_VERSION);
+
+        // First migration is on a new index, so should skip all migrations. If we reset, it should re-trigger and run all migrations
+        resetMigration();
+
+        // Wait for the first migration to finish
+        waitForMigrationCompletion(SecurityMigrations.CLEANUP_ROLE_MAPPING_DUPLICATES_MIGRATION_VERSION);
+
+        assertAllRoleMappings("everyone_kibana_alone", "everyone_fleet_alone");
+    }
+
+    public void testSkipMigrationEmptyFileBasedMappings() throws Exception {
+        internalCluster().setBootstrapMasterNodeIndex(0);
+        ensureGreen();
+        final String masterNode = internalCluster().getMasterName();
+
+        // Wait for file watcher to start
+        awaitFileSettingsWatcher();
+        // Setup listener to wait for any role mapping
+        var fileBasedRoleMappingsWrittenListener = setupClusterStateListener(masterNode);
+        // Write role mappings
+        writeJSONFile(masterNode, TEST_JSON_WITH_EMPTY_ROLE_MAPPINGS, logger, versionCounter);
+        assertTrue(fileBasedRoleMappingsWrittenListener.v1().await(20, TimeUnit.SECONDS));
+
+        // Create a native role mapping to create security index and trigger migration (skipped initially)
+        createNativeRoleMapping("everyone_kibana_alone");
+        createNativeRoleMapping("everyone_fleet_alone");
+        assertAllRoleMappings("everyone_kibana_alone", "everyone_fleet_alone");
+
+        waitForMigrationCompletion(SecurityMigrations.CLEANUP_ROLE_MAPPING_DUPLICATES_MIGRATION_VERSION);
+
+        // First migration is on a new index, so should skip all migrations. If we reset, it should re-trigger and run all migrations
+        resetMigration();
+
+        // Wait for the first migration to finish
+        waitForMigrationCompletion(SecurityMigrations.CLEANUP_ROLE_MAPPING_DUPLICATES_MIGRATION_VERSION);
+
+        assertAllRoleMappings("everyone_kibana_alone", "everyone_fleet_alone");
+    }
+
+    public void testNewIndexSkipMigration() {
+        internalCluster().setBootstrapMasterNodeIndex(0);
+        final String masterNode = internalCluster().getMasterName();
+        ensureGreen();
+        CountDownLatch awaitMigrations = awaitMigrationVersionUpdates(
+            masterNode,
+            SecurityMigrations.CLEANUP_ROLE_MAPPING_DUPLICATES_MIGRATION_VERSION
+        );
+        // Create a native role mapping to create security index and trigger migration
+        createNativeRoleMapping("everyone_kibana_alone");
+        // Make sure no migration ran (set to current version without applying prior migrations)
+        safeAwait(awaitMigrations);
+    }
+
+    /**
+     * Make sure all versions are applied to cluster state sequentially
+     */
+    private CountDownLatch awaitMigrationVersionUpdates(String node, final int... versions) {
+        final ClusterService clusterService = internalCluster().clusterService(node);
+        final CountDownLatch allVersionsCountDown = new CountDownLatch(1);
+        final AtomicInteger currentVersionIdx = new AtomicInteger(0);
+        clusterService.addListener(new ClusterStateListener() {
+            @Override
+            public void clusterChanged(ClusterChangedEvent event) {
+                int currentMigrationVersion = getCurrentMigrationVersion(event.state());
+                if (currentMigrationVersion > 0) {
+                    assertThat(versions[currentVersionIdx.get()], lessThanOrEqualTo(currentMigrationVersion));
+                    if (versions[currentVersionIdx.get()] == currentMigrationVersion) {
+                        currentVersionIdx.incrementAndGet();
+                    }
+
+                    if (currentVersionIdx.get() >= versions.length) {
+                        clusterService.removeListener(this);
+                        allVersionsCountDown.countDown();
+                    }
+                }
+            }
+        });
+
+        return allVersionsCountDown;
+    }
+
+    private void assertAllRoleMappings(String... roleMappingNames) {
+        GetRoleMappingsResponse response = client().execute(GetRoleMappingsAction.INSTANCE, new GetRoleMappingsRequest()).actionGet();
+
+        assertTrue(response.hasMappings());
+        assertThat(response.mappings().length, equalTo(roleMappingNames.length));
+
+        assertThat(
+            Arrays.stream(response.mappings()).map(ExpressionRoleMapping::getName).toList(),
+            containsInAnyOrder(
+                roleMappingNames
+
+            )
+        );
+    }
+
+    private void awaitFileSettingsWatcher() throws Exception {
+        final String masterNode = internalCluster().getMasterName();
+        FileSettingsService masterFileSettingsService = internalCluster().getInstance(FileSettingsService.class, masterNode);
+        assertBusy(() -> assertTrue(masterFileSettingsService.watching()));
+    }
+
+    private void resetMigration() {
+        client().execute(
+            UpdateIndexMigrationVersionAction.INSTANCE,
+            // -1 is a hack, since running a migration on version 0 on a new cluster will cause all migrations to be skipped (not needed)
+            new UpdateIndexMigrationVersionAction.Request(TimeValue.MAX_VALUE, -1, INTERNAL_SECURITY_MAIN_INDEX_7)
+        ).actionGet();
+    }
+
+    private void createNativeRoleMapping(String name) {
+        PutRoleMappingRequest request = new PutRoleMappingRequest();
+        request.setName(name);
+        request.setRules(new FieldExpression("username", Collections.singletonList(new FieldExpression.FieldValue("*"))));
+        request.setRoles(List.of("superuser"));
+
+        ActionFuture<PutRoleMappingResponse> response = client().execute(PutRoleMappingAction.INSTANCE, request);
+        response.actionGet();
+    }
+
+    private void deleteNativeRoleMapping(String name) {
+        DeleteRoleMappingRequest request = new DeleteRoleMappingRequest();
+        request.setName(name);
+
+        ActionFuture<DeleteRoleMappingResponse> response = client().execute(DeleteRoleMappingAction.INSTANCE, request);
+        response.actionGet();
+    }
+
+    private void assertMigrationVersionAtLeast(int expectedVersion) {
+        assertThat(getCurrentMigrationVersion(), greaterThanOrEqualTo(expectedVersion));
+    }
+
+    private void assertMigrationLessThan(int expectedVersion) {
+        assertThat(getCurrentMigrationVersion(), lessThan(expectedVersion));
+    }
+
+    private int getCurrentMigrationVersion(ClusterState state) {
+        IndexMetadata indexMetadata = state.metadata().getIndices().get(INTERNAL_SECURITY_MAIN_INDEX_7);
+        if (indexMetadata == null || indexMetadata.getCustomData(MIGRATION_VERSION_CUSTOM_KEY) == null) {
+            return 0;
+        }
+        return Integer.parseInt(indexMetadata.getCustomData(MIGRATION_VERSION_CUSTOM_KEY).get(MIGRATION_VERSION_CUSTOM_DATA_KEY));
+    }
+
+    private int getCurrentMigrationVersion() {
+        ClusterService clusterService = internalCluster().getInstance(ClusterService.class);
+        return getCurrentMigrationVersion(clusterService.state());
+    }
+
+    private void waitForMigrationCompletion(int version) throws Exception {
+        assertBusy(() -> assertMigrationVersionAtLeast(version));
+    }
+}
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/SecurityFeatures.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/SecurityFeatures.java
index c1fe553f41334..d0292f32cd75f 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/SecurityFeatures.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/SecurityFeatures.java
@@ -17,13 +17,14 @@
 import static org.elasticsearch.xpack.security.support.SecuritySystemIndices.SECURITY_MIGRATION_FRAMEWORK;
 import static org.elasticsearch.xpack.security.support.SecuritySystemIndices.SECURITY_PROFILE_ORIGIN_FEATURE;
 import static org.elasticsearch.xpack.security.support.SecuritySystemIndices.SECURITY_ROLES_METADATA_FLATTENED;
+import static org.elasticsearch.xpack.security.support.SecuritySystemIndices.SECURITY_ROLE_MAPPING_CLEANUP;
 import static org.elasticsearch.xpack.security.support.SecuritySystemIndices.VERSION_SECURITY_PROFILE_ORIGIN;
 
 public class SecurityFeatures implements FeatureSpecification {
 
     @Override
     public Set<NodeFeature> getFeatures() {
-        return Set.of(SECURITY_ROLES_METADATA_FLATTENED, SECURITY_MIGRATION_FRAMEWORK);
+        return Set.of(SECURITY_ROLE_MAPPING_CLEANUP, SECURITY_ROLES_METADATA_FLATTENED, SECURITY_MIGRATION_FRAMEWORK);
     }
 
     @Override
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java
index 6d9b0ef6aeebe..12ef800a7aae7 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java
@@ -31,6 +31,7 @@
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.MappingMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
+import org.elasticsearch.cluster.metadata.ReservedStateMetadata;
 import org.elasticsearch.cluster.routing.IndexRoutingTable;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.core.TimeValue;
@@ -46,7 +47,9 @@
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.threadpool.Scheduler;
 import org.elasticsearch.xcontent.XContentType;
+import org.elasticsearch.xpack.core.security.authz.RoleMappingMetadata;
 import org.elasticsearch.xpack.security.SecurityFeatures;
+import org.elasticsearch.xpack.security.action.rolemapping.ReservedRoleMappingAction;
 
 import java.time.Instant;
 import java.util.List;
@@ -75,7 +78,7 @@
 public class SecurityIndexManager implements ClusterStateListener {
 
     public static final String SECURITY_VERSION_STRING = "security-version";
-
+    protected static final String FILE_SETTINGS_METADATA_NAMESPACE = "file_settings";
     private static final Logger logger = LogManager.getLogger(SecurityIndexManager.class);
 
     /**
@@ -86,6 +89,13 @@ public enum Availability {
         PRIMARY_SHARDS
     }
 
+    public enum RoleMappingsCleanupMigrationStatus {
+        READY,
+        NOT_READY,
+        SKIP,
+        DONE
+    }
+
     private final Client client;
     private final SystemIndexDescriptor systemIndexDescriptor;
 
@@ -195,10 +205,6 @@ public boolean isMigrationsVersionAtLeast(Integer expectedMigrationsVersion) {
         return indexExists() && this.state.migrationsVersion.compareTo(expectedMigrationsVersion) >= 0;
     }
 
-    public boolean isCreatedOnLatestVersion() {
-        return this.state.createdOnLatestVersion;
-    }
-
     public ElasticsearchException getUnavailableReason(Availability availability) {
         // ensure usage of a local copy so all checks execute against the same state!
         if (defensiveCopy == false) {
@@ -261,6 +267,7 @@ private SystemIndexDescriptor.MappingsVersion getMinSecurityIndexMappingVersion(
     /**
      * Check if the index was created on the latest index version available in the cluster
      */
+
     private static boolean isCreatedOnLatestVersion(IndexMetadata indexMetadata) {
         final IndexVersion indexVersionCreated = indexMetadata != null
             ? SETTING_INDEX_VERSION_CREATED.get(indexMetadata.getSettings())
@@ -268,6 +275,50 @@ private static boolean isCreatedOnLatestVersion(IndexMetadata indexMetadata) {
         return indexVersionCreated != null && indexVersionCreated.onOrAfter(IndexVersion.current());
     }
 
+    /**
+     * Check if a role mappings cleanup migration is needed or has already been performed and if the cluster is ready for a cleanup
+     * migration
+     *
+     * @param clusterState current cluster state
+     * @param migrationsVersion current migration version
+     *
+     * @return RoleMappingsCleanupMigrationStatus
+     */
+    static RoleMappingsCleanupMigrationStatus getRoleMappingsCleanupMigrationStatus(ClusterState clusterState, int migrationsVersion) {
+        // Migration already finished
+        if (migrationsVersion >= SecurityMigrations.CLEANUP_ROLE_MAPPING_DUPLICATES_MIGRATION_VERSION) {
+            return RoleMappingsCleanupMigrationStatus.DONE;
+        }
+
+        ReservedStateMetadata fileSettingsMetadata = clusterState.metadata().reservedStateMetadata().get(FILE_SETTINGS_METADATA_NAMESPACE);
+        boolean hasFileSettingsMetadata = fileSettingsMetadata != null;
+        // If there is no fileSettingsMetadata, there should be no reserved state (this is to catch bugs related to
+        // name changes to FILE_SETTINGS_METADATA_NAMESPACE)
+        assert hasFileSettingsMetadata || clusterState.metadata().reservedStateMetadata().isEmpty()
+            : "ReservedStateMetadata contains unknown namespace";
+
+        // If no file based role mappings available -> migration not needed
+        if (hasFileSettingsMetadata == false || fileSettingsMetadata.keys(ReservedRoleMappingAction.NAME).isEmpty()) {
+            return RoleMappingsCleanupMigrationStatus.SKIP;
+        }
+
+        RoleMappingMetadata roleMappingMetadata = RoleMappingMetadata.getFromClusterState(clusterState);
+
+        // If there are file based role mappings, make sure they have the latest format (name available) and that they have all been
+        // synced to cluster state (same size as the reserved state keys)
+        if (roleMappingMetadata.getRoleMappings().size() == fileSettingsMetadata.keys(ReservedRoleMappingAction.NAME).size()
+            && roleMappingMetadata.hasAnyMappingWithFallbackName() == false) {
+            return RoleMappingsCleanupMigrationStatus.READY;
+        }
+
+        // If none of the above conditions are met, wait for a state change to re-evaluate if the cluster is ready for migration
+        return RoleMappingsCleanupMigrationStatus.NOT_READY;
+    }
+
+    public RoleMappingsCleanupMigrationStatus getRoleMappingsCleanupMigrationStatus() {
+        return state.roleMappingsCleanupMigrationStatus;
+    }
+
     @Override
     public void clusterChanged(ClusterChangedEvent event) {
         if (event.state().blocks().hasGlobalBlock(GatewayService.STATE_NOT_RECOVERED_BLOCK)) {
@@ -285,8 +336,12 @@ public void clusterChanged(ClusterChangedEvent event) {
         Tuple<Boolean, Boolean> available = checkIndexAvailable(event.state());
         final boolean indexAvailableForWrite = available.v1();
         final boolean indexAvailableForSearch = available.v2();
-        final boolean mappingIsUpToDate = indexMetadata == null || checkIndexMappingUpToDate(event.state());
         final int migrationsVersion = getMigrationVersionFromIndexMetadata(indexMetadata);
+        final RoleMappingsCleanupMigrationStatus roleMappingsCleanupMigrationStatus = getRoleMappingsCleanupMigrationStatus(
+            event.state(),
+            migrationsVersion
+        );
+        final boolean mappingIsUpToDate = indexMetadata == null || checkIndexMappingUpToDate(event.state());
         final SystemIndexDescriptor.MappingsVersion minClusterMappingVersion = getMinSecurityIndexMappingVersion(event.state());
         final int indexMappingVersion = loadIndexMappingVersion(systemIndexDescriptor.getAliasName(), event.state());
         final String concreteIndexName = indexMetadata == null
@@ -315,6 +370,7 @@ public void clusterChanged(ClusterChangedEvent event) {
             indexAvailableForWrite,
             mappingIsUpToDate,
             createdOnLatestVersion,
+            roleMappingsCleanupMigrationStatus,
             migrationsVersion,
             minClusterMappingVersion,
             indexMappingVersion,
@@ -474,7 +530,8 @@ private Tuple<Boolean, Boolean> checkIndexAvailable(ClusterState state) {
 
     public boolean isEligibleSecurityMigration(SecurityMigrations.SecurityMigration securityMigration) {
         return state.securityFeatures.containsAll(securityMigration.nodeFeaturesRequired())
-            && state.indexMappingVersion >= securityMigration.minMappingVersion();
+            && state.indexMappingVersion >= securityMigration.minMappingVersion()
+            && securityMigration.checkPreConditions(state);
     }
 
     public boolean isReadyForSecurityMigration(SecurityMigrations.SecurityMigration securityMigration) {
@@ -680,6 +737,10 @@ public void onFailure(Exception e) {
         }
     }
 
+    public boolean isCreatedOnLatestVersion() {
+        return state.createdOnLatestVersion;
+    }
+
     /**
      * Return true if the state moves from an unhealthy ("RED") index state to a healthy ("non-RED") state.
      */
@@ -714,6 +775,7 @@ public static class State {
             null,
             null,
             null,
+            null,
             Set.of()
         );
         public final Instant creationTime;
@@ -722,6 +784,7 @@ public static class State {
         public final boolean indexAvailableForWrite;
         public final boolean mappingUpToDate;
         public final boolean createdOnLatestVersion;
+        public final RoleMappingsCleanupMigrationStatus roleMappingsCleanupMigrationStatus;
         public final Integer migrationsVersion;
         // Min mapping version supported by the descriptors in the cluster
         public final SystemIndexDescriptor.MappingsVersion minClusterMappingVersion;
@@ -740,6 +803,7 @@ public State(
             boolean indexAvailableForWrite,
             boolean mappingUpToDate,
             boolean createdOnLatestVersion,
+            RoleMappingsCleanupMigrationStatus roleMappingsCleanupMigrationStatus,
             Integer migrationsVersion,
             SystemIndexDescriptor.MappingsVersion minClusterMappingVersion,
             Integer indexMappingVersion,
@@ -756,6 +820,7 @@ public State(
             this.mappingUpToDate = mappingUpToDate;
             this.migrationsVersion = migrationsVersion;
             this.createdOnLatestVersion = createdOnLatestVersion;
+            this.roleMappingsCleanupMigrationStatus = roleMappingsCleanupMigrationStatus;
             this.minClusterMappingVersion = minClusterMappingVersion;
             this.indexMappingVersion = indexMappingVersion;
             this.concreteIndexName = concreteIndexName;
@@ -776,6 +841,7 @@ public boolean equals(Object o) {
                 && indexAvailableForWrite == state.indexAvailableForWrite
                 && mappingUpToDate == state.mappingUpToDate
                 && createdOnLatestVersion == state.createdOnLatestVersion
+                && roleMappingsCleanupMigrationStatus == state.roleMappingsCleanupMigrationStatus
                 && Objects.equals(indexMappingVersion, state.indexMappingVersion)
                 && Objects.equals(migrationsVersion, state.migrationsVersion)
                 && Objects.equals(minClusterMappingVersion, state.minClusterMappingVersion)
@@ -798,6 +864,7 @@ public int hashCode() {
                 indexAvailableForWrite,
                 mappingUpToDate,
                 createdOnLatestVersion,
+                roleMappingsCleanupMigrationStatus,
                 migrationsVersion,
                 minClusterMappingVersion,
                 indexMappingVersion,
@@ -822,6 +889,8 @@ public String toString() {
                 + mappingUpToDate
                 + ", createdOnLatestVersion="
                 + createdOnLatestVersion
+                + ", roleMappingsCleanupMigrationStatus="
+                + roleMappingsCleanupMigrationStatus
                 + ", migrationsVersion="
                 + migrationsVersion
                 + ", minClusterMappingVersion="
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityMigrations.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityMigrations.java
index 5cd8cba763d3d..203dec9e25b91 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityMigrations.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityMigrations.java
@@ -11,6 +11,8 @@
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.action.support.GroupedActionListener;
+import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.index.query.BoolQueryBuilder;
@@ -20,20 +22,35 @@
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.xpack.core.security.action.rolemapping.DeleteRoleMappingAction;
+import org.elasticsearch.xpack.core.security.action.rolemapping.DeleteRoleMappingRequestBuilder;
+import org.elasticsearch.xpack.core.security.action.rolemapping.DeleteRoleMappingResponse;
+import org.elasticsearch.xpack.core.security.action.rolemapping.GetRoleMappingsAction;
+import org.elasticsearch.xpack.core.security.action.rolemapping.GetRoleMappingsRequestBuilder;
+import org.elasticsearch.xpack.core.security.action.rolemapping.GetRoleMappingsResponse;
+import org.elasticsearch.xpack.core.security.authc.support.mapper.ExpressionRoleMapping;
 
+import java.util.Arrays;
 import java.util.Collections;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.TreeMap;
+import java.util.stream.Collectors;
 
+import static org.elasticsearch.xpack.core.ClientHelper.SECURITY_ORIGIN;
+import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin;
+import static org.elasticsearch.xpack.security.support.SecurityIndexManager.RoleMappingsCleanupMigrationStatus.READY;
+import static org.elasticsearch.xpack.security.support.SecurityIndexManager.RoleMappingsCleanupMigrationStatus.SKIP;
+import static org.elasticsearch.xpack.security.support.SecuritySystemIndices.SecurityMainIndexMappingVersion.ADD_MANAGE_ROLES_PRIVILEGE;
 import static org.elasticsearch.xpack.security.support.SecuritySystemIndices.SecurityMainIndexMappingVersion.ADD_REMOTE_CLUSTER_AND_DESCRIPTION_FIELDS;
 
-/**
- * Interface for creating SecurityMigrations that will be automatically applied once to existing .security indices
- * IMPORTANT: A new index version needs to be added to {@link org.elasticsearch.index.IndexVersions} for the migration to be triggered
- */
 public class SecurityMigrations {
 
+    /**
+     * Interface for creating SecurityMigrations that will be automatically applied once to existing .security indices
+     * IMPORTANT: A new index version needs to be added to {@link org.elasticsearch.index.IndexVersions} for the migration to be triggered
+     */
     public interface SecurityMigration {
         /**
          * Method that will execute the actual migration - needs to be idempotent and non-blocking
@@ -52,6 +69,16 @@ public interface SecurityMigration {
          */
         Set<NodeFeature> nodeFeaturesRequired();
 
+        /**
+         * Check that any pre-conditions are met before launching migration
+         *
+         * @param securityIndexManagerState current state of the security index
+         * @return true if pre-conditions met, otherwise false
+         */
+        default boolean checkPreConditions(SecurityIndexManager.State securityIndexManagerState) {
+            return true;
+        }
+
         /**
          * The min mapping version required to support this migration. This makes sure that the index has at least the min mapping that is
          * required to support the migration.
@@ -62,63 +89,163 @@ public interface SecurityMigration {
     }
 
     public static final Integer ROLE_METADATA_FLATTENED_MIGRATION_VERSION = 1;
+    public static final Integer CLEANUP_ROLE_MAPPING_DUPLICATES_MIGRATION_VERSION = 2;
+    private static final Logger logger = LogManager.getLogger(SecurityMigration.class);
 
     public static final TreeMap<Integer, SecurityMigration> MIGRATIONS_BY_VERSION = new TreeMap<>(
-        Map.of(ROLE_METADATA_FLATTENED_MIGRATION_VERSION, new SecurityMigration() {
-            private static final Logger logger = LogManager.getLogger(SecurityMigration.class);
-
-            @Override
-            public void migrate(SecurityIndexManager indexManager, Client client, ActionListener<Void> listener) {
-                BoolQueryBuilder filterQuery = new BoolQueryBuilder().filter(QueryBuilders.termQuery("type", "role"))
-                    .mustNot(QueryBuilders.existsQuery("metadata_flattened"));
-                SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().query(filterQuery).size(0).trackTotalHits(true);
-                SearchRequest countRequest = new SearchRequest(indexManager.getConcreteIndexName());
-                countRequest.source(searchSourceBuilder);
-
-                client.search(countRequest, ActionListener.wrap(response -> {
-                    // If there are no roles, skip migration
-                    if (response.getHits().getTotalHits().value() > 0) {
-                        logger.info("Preparing to migrate [" + response.getHits().getTotalHits().value() + "] roles");
-                        updateRolesByQuery(indexManager, client, filterQuery, listener);
-                    } else {
-                        listener.onResponse(null);
-                    }
+        Map.of(
+            ROLE_METADATA_FLATTENED_MIGRATION_VERSION,
+            new RoleMetadataFlattenedMigration(),
+            CLEANUP_ROLE_MAPPING_DUPLICATES_MIGRATION_VERSION,
+            new CleanupRoleMappingDuplicatesMigration()
+        )
+    );
+
+    public static class RoleMetadataFlattenedMigration implements SecurityMigration {
+        @Override
+        public void migrate(SecurityIndexManager indexManager, Client client, ActionListener<Void> listener) {
+            BoolQueryBuilder filterQuery = new BoolQueryBuilder().filter(QueryBuilders.termQuery("type", "role"))
+                .mustNot(QueryBuilders.existsQuery("metadata_flattened"));
+            SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().query(filterQuery).size(0).trackTotalHits(true);
+            SearchRequest countRequest = new SearchRequest(indexManager.getConcreteIndexName());
+            countRequest.source(searchSourceBuilder);
+
+            client.search(countRequest, ActionListener.wrap(response -> {
+                // If there are no roles, skip migration
+                if (response.getHits().getTotalHits().value() > 0) {
+                    logger.info("Preparing to migrate [" + response.getHits().getTotalHits().value() + "] roles");
+                    updateRolesByQuery(indexManager, client, filterQuery, listener);
+                } else {
+                    listener.onResponse(null);
+                }
+            }, listener::onFailure));
+        }
+
+        private void updateRolesByQuery(
+            SecurityIndexManager indexManager,
+            Client client,
+            BoolQueryBuilder filterQuery,
+            ActionListener<Void> listener
+        ) {
+            UpdateByQueryRequest updateByQueryRequest = new UpdateByQueryRequest(indexManager.getConcreteIndexName());
+            updateByQueryRequest.setQuery(filterQuery);
+            updateByQueryRequest.setScript(
+                new Script(ScriptType.INLINE, "painless", "ctx._source.metadata_flattened = ctx._source.metadata", Collections.emptyMap())
+            );
+            client.admin()
+                .cluster()
+                .execute(UpdateByQueryAction.INSTANCE, updateByQueryRequest, ActionListener.wrap(bulkByScrollResponse -> {
+                    logger.info("Migrated [" + bulkByScrollResponse.getTotal() + "] roles");
+                    listener.onResponse(null);
                 }, listener::onFailure));
+        }
+
+        @Override
+        public Set<NodeFeature> nodeFeaturesRequired() {
+            return Set.of(SecuritySystemIndices.SECURITY_ROLES_METADATA_FLATTENED);
+        }
+
+        @Override
+        public int minMappingVersion() {
+            return ADD_REMOTE_CLUSTER_AND_DESCRIPTION_FIELDS.id();
+        }
+    }
+
+    public static class CleanupRoleMappingDuplicatesMigration implements SecurityMigration {
+        @Override
+        public void migrate(SecurityIndexManager indexManager, Client client, ActionListener<Void> listener) {
+            if (indexManager.getRoleMappingsCleanupMigrationStatus() == SKIP) {
+                listener.onResponse(null);
+                return;
             }
+            assert indexManager.getRoleMappingsCleanupMigrationStatus() == READY;
 
-            private void updateRolesByQuery(
-                SecurityIndexManager indexManager,
-                Client client,
-                BoolQueryBuilder filterQuery,
-                ActionListener<Void> listener
-            ) {
-                UpdateByQueryRequest updateByQueryRequest = new UpdateByQueryRequest(indexManager.getConcreteIndexName());
-                updateByQueryRequest.setQuery(filterQuery);
-                updateByQueryRequest.setScript(
-                    new Script(
-                        ScriptType.INLINE,
-                        "painless",
-                        "ctx._source.metadata_flattened = ctx._source.metadata",
-                        Collections.emptyMap()
-                    )
+            getRoleMappings(client, ActionListener.wrap(roleMappings -> {
+                List<String> roleMappingsToDelete = getDuplicateRoleMappingNames(roleMappings.mappings());
+                if (roleMappingsToDelete.isEmpty() == false) {
+                    logger.info("Found [" + roleMappingsToDelete.size() + "] role mapping(s) to cleanup in .security index.");
+                    deleteNativeRoleMappings(client, roleMappingsToDelete, listener);
+                } else {
+                    listener.onResponse(null);
+                }
+            }, listener::onFailure));
+        }
+
+        private void getRoleMappings(Client client, ActionListener<GetRoleMappingsResponse> listener) {
+            executeAsyncWithOrigin(
+                client,
+                SECURITY_ORIGIN,
+                GetRoleMappingsAction.INSTANCE,
+                new GetRoleMappingsRequestBuilder(client).request(),
+                listener
+            );
+        }
+
+        private void deleteNativeRoleMappings(Client client, List<String> names, ActionListener<Void> listener) {
+            assert names.isEmpty() == false;
+            ActionListener<DeleteRoleMappingResponse> groupListener = new GroupedActionListener<>(
+                names.size(),
+                ActionListener.wrap(responses -> {
+                    long foundRoleMappings = responses.stream().filter(DeleteRoleMappingResponse::isFound).count();
+                    if (responses.size() > foundRoleMappings) {
+                        logger.warn(
+                            "[" + (responses.size() - foundRoleMappings) + "] Role mapping(s) not found during role mapping clean up."
+                        );
+                    }
+                    if (foundRoleMappings > 0) {
+                        logger.info("Deleted [" + foundRoleMappings + "] duplicated role mapping(s) from .security index");
+                    }
+                    listener.onResponse(null);
+                }, listener::onFailure)
+            );
+
+            for (String name : names) {
+                executeAsyncWithOrigin(
+                    client,
+                    SECURITY_ORIGIN,
+                    DeleteRoleMappingAction.INSTANCE,
+                    new DeleteRoleMappingRequestBuilder(client).name(name).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).request(),
+                    groupListener
                 );
-                client.admin()
-                    .cluster()
-                    .execute(UpdateByQueryAction.INSTANCE, updateByQueryRequest, ActionListener.wrap(bulkByScrollResponse -> {
-                        logger.info("Migrated [" + bulkByScrollResponse.getTotal() + "] roles");
-                        listener.onResponse(null);
-                    }, listener::onFailure));
             }
 
-            @Override
-            public Set<NodeFeature> nodeFeaturesRequired() {
-                return Set.of(SecuritySystemIndices.SECURITY_ROLES_METADATA_FLATTENED);
-            }
+        }
 
-            @Override
-            public int minMappingVersion() {
-                return ADD_REMOTE_CLUSTER_AND_DESCRIPTION_FIELDS.id();
-            }
-        })
-    );
+        @Override
+        public boolean checkPreConditions(SecurityIndexManager.State securityIndexManagerState) {
+            // Block migration until expected role mappings are in cluster state and in the correct format or skip if no role mappings
+            // are expected
+            return securityIndexManagerState.roleMappingsCleanupMigrationStatus == READY
+                || securityIndexManagerState.roleMappingsCleanupMigrationStatus == SKIP;
+        }
+
+        @Override
+        public Set<NodeFeature> nodeFeaturesRequired() {
+            return Set.of(SecuritySystemIndices.SECURITY_ROLE_MAPPING_CLEANUP);
+        }
+
+        @Override
+        public int minMappingVersion() {
+            return ADD_MANAGE_ROLES_PRIVILEGE.id();
+        }
+
+        // Visible for testing
+        protected static List<String> getDuplicateRoleMappingNames(ExpressionRoleMapping... roleMappings) {
+            // Partition role mappings on if they're cluster state role mappings (true) or native role mappings (false)
+            Map<Boolean, List<ExpressionRoleMapping>> partitionedRoleMappings = Arrays.stream(roleMappings)
+                .collect(Collectors.partitioningBy(ExpressionRoleMapping::isReadOnly));
+
+            Set<String> clusterStateRoleMappings = partitionedRoleMappings.get(true)
+                .stream()
+                .map(ExpressionRoleMapping::getName)
+                .map(ExpressionRoleMapping::removeReadOnlySuffixIfPresent)
+                .collect(Collectors.toSet());
+
+            return partitionedRoleMappings.get(false)
+                .stream()
+                .map(ExpressionRoleMapping::getName)
+                .filter(clusterStateRoleMappings::contains)
+                .toList();
+        }
+    }
 }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecuritySystemIndices.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecuritySystemIndices.java
index 36ea14c6e101b..77c7d19e94a9b 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecuritySystemIndices.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecuritySystemIndices.java
@@ -61,6 +61,7 @@ public class SecuritySystemIndices {
     public static final NodeFeature SECURITY_PROFILE_ORIGIN_FEATURE = new NodeFeature("security.security_profile_origin");
     public static final NodeFeature SECURITY_MIGRATION_FRAMEWORK = new NodeFeature("security.migration_framework");
     public static final NodeFeature SECURITY_ROLES_METADATA_FLATTENED = new NodeFeature("security.roles_metadata_flattened");
+    public static final NodeFeature SECURITY_ROLE_MAPPING_CLEANUP = new NodeFeature("security.role_mapping_cleanup");
 
     /**
      * Security managed index mappings used to be updated based on the product version. They are now updated based on per-index mappings
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java
index 8d580f10e5137..c0e55992df88f 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java
@@ -824,7 +824,7 @@ public void testSecurityRestHandlerInterceptorCanBeInstalled() throws IllegalAcc
                 null,
                 List.of(),
                 RestExtension.allowAll(),
-                new IncrementalBulkService(null, null, new ThreadContext(Settings.EMPTY))
+                new IncrementalBulkService(null, null)
             );
             actionModule.initRestHandlers(null, null);
 
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java
index e1c3b936e5a32..cd6c88cf525af 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java
@@ -2518,6 +2518,7 @@ private SecurityIndexManager.State dummyState(ClusterHealthStatus indexStatus) {
             null,
             null,
             null,
+            null,
             concreteSecurityIndexName,
             indexStatus,
             IndexMetadata.State.OPEN,
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmTests.java
index 2254c78a2910c..75d5959f351f0 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmTests.java
@@ -43,6 +43,7 @@ private SecurityIndexManager.State dummyState(ClusterHealthStatus indexStatus) {
             null,
             null,
             null,
+            null,
             concreteSecurityIndexName,
             indexStatus,
             IndexMetadata.State.OPEN,
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java
index 38f01d4d18bc7..ca84a9189d90a 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java
@@ -415,6 +415,7 @@ private SecurityIndexManager.State indexState(boolean isUpToDate, ClusterHealthS
             null,
             null,
             null,
+            null,
             concreteSecurityIndexName,
             healthStatus,
             IndexMetadata.State.OPEN,
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java
index 9587533d87d86..da903ff7f7177 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java
@@ -1702,6 +1702,7 @@ public SecurityIndexManager.State dummyIndexState(boolean isIndexUpToDate, Clust
             null,
             null,
             null,
+            null,
             concreteSecurityIndexName,
             healthStatus,
             IndexMetadata.State.OPEN,
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreTests.java
index f91cb567ba689..73a45dc20ac42 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreTests.java
@@ -904,6 +904,7 @@ private SecurityIndexManager.State dummyState(
             null,
             null,
             null,
+            null,
             concreteSecurityIndexName,
             healthStatus,
             IndexMetadata.State.OPEN,
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/CacheInvalidatorRegistryTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/CacheInvalidatorRegistryTests.java
index e3b00dfbcc6b8..d551dded4e566 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/CacheInvalidatorRegistryTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/CacheInvalidatorRegistryTests.java
@@ -63,6 +63,7 @@ public void testSecurityIndexStateChangeWillInvalidateAllRegisteredInvalidators(
             true,
             true,
             null,
+            null,
             new SystemIndexDescriptor.MappingsVersion(SecurityMainIndexMappingVersion.latest().id(), 0),
             null,
             ".security",
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java
index 493483a5e4a1b..0b98a595a6ab9 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java
@@ -23,6 +23,8 @@
 import org.elasticsearch.cluster.metadata.AliasMetadata;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
+import org.elasticsearch.cluster.metadata.ReservedStateHandlerMetadata;
+import org.elasticsearch.cluster.metadata.ReservedStateMetadata;
 import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.cluster.node.DiscoveryNodeUtils;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
@@ -51,8 +53,11 @@
 import org.elasticsearch.test.client.NoOpClient;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xpack.core.security.authc.support.mapper.ExpressionRoleMapping;
+import org.elasticsearch.xpack.core.security.authz.RoleMappingMetadata;
 import org.elasticsearch.xpack.core.security.test.TestRestrictedIndices;
 import org.elasticsearch.xpack.security.SecurityFeatures;
+import org.elasticsearch.xpack.security.action.rolemapping.ReservedRoleMappingAction;
 import org.elasticsearch.xpack.security.support.SecuritySystemIndices.SecurityMainIndexMappingVersion;
 import org.elasticsearch.xpack.security.test.SecurityTestUtils;
 import org.hamcrest.Matchers;
@@ -70,6 +75,7 @@
 import java.util.stream.Collectors;
 
 import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder;
+import static org.elasticsearch.xpack.security.support.SecurityIndexManager.FILE_SETTINGS_METADATA_NAMESPACE;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;
@@ -654,6 +660,138 @@ public int minMappingVersion() {
         }));
     }
 
+    public void testNotReadyForMigrationBecauseOfPrecondition() {
+        final ClusterState.Builder clusterStateBuilder = createClusterState(
+            TestRestrictedIndices.INTERNAL_SECURITY_MAIN_INDEX_7,
+            SecuritySystemIndices.SECURITY_MAIN_ALIAS,
+            IndexMetadata.State.OPEN
+        );
+        clusterStateBuilder.nodeFeatures(
+            Map.of("1", new SecurityFeatures().getFeatures().stream().map(NodeFeature::id).collect(Collectors.toSet()))
+        );
+        manager.clusterChanged(event(markShardsAvailable(clusterStateBuilder)));
+        assertFalse(manager.isReadyForSecurityMigration(new SecurityMigrations.SecurityMigration() {
+            @Override
+            public void migrate(SecurityIndexManager indexManager, Client client, ActionListener<Void> listener) {
+                listener.onResponse(null);
+            }
+
+            @Override
+            public Set<NodeFeature> nodeFeaturesRequired() {
+                return Set.of();
+            }
+
+            @Override
+            public int minMappingVersion() {
+                return 0;
+            }
+
+            @Override
+            public boolean checkPreConditions(SecurityIndexManager.State securityIndexManagerState) {
+                return false;
+            }
+        }));
+    }
+
+    private ClusterState.Builder clusterStateBuilderForMigrationTesting() {
+        return createClusterState(
+            TestRestrictedIndices.INTERNAL_SECURITY_MAIN_INDEX_7,
+            SecuritySystemIndices.SECURITY_MAIN_ALIAS,
+            IndexMetadata.State.OPEN
+        );
+    }
+
+    public void testGetRoleMappingsCleanupMigrationStatus() {
+        {
+            assertThat(
+                SecurityIndexManager.getRoleMappingsCleanupMigrationStatus(
+                    clusterStateBuilderForMigrationTesting().build(),
+                    SecurityMigrations.CLEANUP_ROLE_MAPPING_DUPLICATES_MIGRATION_VERSION
+                ),
+                equalTo(SecurityIndexManager.RoleMappingsCleanupMigrationStatus.DONE)
+            );
+        }
+        {
+            // Migration should be skipped
+            ClusterState.Builder clusterStateBuilder = clusterStateBuilderForMigrationTesting();
+            Metadata.Builder metadataBuilder = new Metadata.Builder();
+            metadataBuilder.put(ReservedStateMetadata.builder(FILE_SETTINGS_METADATA_NAMESPACE).build());
+            assertThat(
+                SecurityIndexManager.getRoleMappingsCleanupMigrationStatus(clusterStateBuilder.metadata(metadataBuilder).build(), 1),
+                equalTo(SecurityIndexManager.RoleMappingsCleanupMigrationStatus.SKIP)
+            );
+        }
+        {
+            // Not ready for migration
+            ClusterState.Builder clusterStateBuilder = clusterStateBuilderForMigrationTesting();
+            Metadata.Builder metadataBuilder = new Metadata.Builder();
+            ReservedStateMetadata.Builder builder = ReservedStateMetadata.builder(FILE_SETTINGS_METADATA_NAMESPACE);
+            // File settings role mappings exist
+            ReservedStateHandlerMetadata reservedStateHandlerMetadata = new ReservedStateHandlerMetadata(
+                ReservedRoleMappingAction.NAME,
+                Set.of("role_mapping_1")
+            );
+            builder.putHandler(reservedStateHandlerMetadata);
+            metadataBuilder.put(builder.build());
+
+            // No role mappings in cluster state yet
+            metadataBuilder.putCustom(RoleMappingMetadata.TYPE, new RoleMappingMetadata(Set.of()));
+
+            assertThat(
+                SecurityIndexManager.getRoleMappingsCleanupMigrationStatus(clusterStateBuilder.metadata(metadataBuilder).build(), 1),
+                equalTo(SecurityIndexManager.RoleMappingsCleanupMigrationStatus.NOT_READY)
+            );
+        }
+        {
+            // Old role mappings in cluster state
+            final ClusterState.Builder clusterStateBuilder = clusterStateBuilderForMigrationTesting();
+            Metadata.Builder metadataBuilder = new Metadata.Builder();
+            ReservedStateMetadata.Builder builder = ReservedStateMetadata.builder(FILE_SETTINGS_METADATA_NAMESPACE);
+            // File settings role mappings exist
+            ReservedStateHandlerMetadata reservedStateHandlerMetadata = new ReservedStateHandlerMetadata(
+                ReservedRoleMappingAction.NAME,
+                Set.of("role_mapping_1")
+            );
+            builder.putHandler(reservedStateHandlerMetadata);
+            metadataBuilder.put(builder.build());
+
+            // Role mappings in cluster state with fallback name
+            metadataBuilder.putCustom(
+                RoleMappingMetadata.TYPE,
+                new RoleMappingMetadata(Set.of(new ExpressionRoleMapping(RoleMappingMetadata.FALLBACK_NAME, null, null, null, null, true)))
+            );
+
+            assertThat(
+                SecurityIndexManager.getRoleMappingsCleanupMigrationStatus(clusterStateBuilder.metadata(metadataBuilder).build(), 1),
+                equalTo(SecurityIndexManager.RoleMappingsCleanupMigrationStatus.NOT_READY)
+            );
+        }
+        {
+            // Ready for migration
+            final ClusterState.Builder clusterStateBuilder = clusterStateBuilderForMigrationTesting();
+            Metadata.Builder metadataBuilder = new Metadata.Builder();
+            ReservedStateMetadata.Builder builder = ReservedStateMetadata.builder(FILE_SETTINGS_METADATA_NAMESPACE);
+            // File settings role mappings exist
+            ReservedStateHandlerMetadata reservedStateHandlerMetadata = new ReservedStateHandlerMetadata(
+                ReservedRoleMappingAction.NAME,
+                Set.of("role_mapping_1")
+            );
+            builder.putHandler(reservedStateHandlerMetadata);
+            metadataBuilder.put(builder.build());
+
+            // Role mappings in cluster state
+            metadataBuilder.putCustom(
+                RoleMappingMetadata.TYPE,
+                new RoleMappingMetadata(Set.of(new ExpressionRoleMapping("role_mapping_1", null, null, null, null, true)))
+            );
+
+            assertThat(
+                SecurityIndexManager.getRoleMappingsCleanupMigrationStatus(clusterStateBuilder.metadata(metadataBuilder).build(), 1),
+                equalTo(SecurityIndexManager.RoleMappingsCleanupMigrationStatus.READY)
+            );
+        }
+    }
+
     public void testProcessClosedIndexState() {
         // Index initially exists
         final ClusterState.Builder indexAvailable = createClusterState(
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityMigrationsTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityMigrationsTests.java
new file mode 100644
index 0000000000000..3d3cc47b55cf6
--- /dev/null
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityMigrationsTests.java
@@ -0,0 +1,174 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.security.support;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.client.internal.Client;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.threadpool.TestThreadPool;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.xpack.core.security.action.rolemapping.DeleteRoleMappingAction;
+import org.elasticsearch.xpack.core.security.action.rolemapping.DeleteRoleMappingRequest;
+import org.elasticsearch.xpack.core.security.action.rolemapping.DeleteRoleMappingResponse;
+import org.elasticsearch.xpack.core.security.action.rolemapping.GetRoleMappingsAction;
+import org.elasticsearch.xpack.core.security.action.rolemapping.GetRoleMappingsRequest;
+import org.elasticsearch.xpack.core.security.action.rolemapping.GetRoleMappingsResponse;
+import org.elasticsearch.xpack.core.security.authc.support.mapper.ExpressionRoleMapping;
+import org.junit.After;
+import org.junit.Before;
+
+import java.util.List;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.empty;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasSize;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class SecurityMigrationsTests extends ESTestCase {
+    private ThreadPool threadPool;
+    private Client client;
+
+    public void testGetDuplicateRoleMappingNames() {
+        assertThat(SecurityMigrations.CleanupRoleMappingDuplicatesMigration.getDuplicateRoleMappingNames(), empty());
+        assertThat(
+            SecurityMigrations.CleanupRoleMappingDuplicatesMigration.getDuplicateRoleMappingNames(
+                nativeRoleMapping("roleMapping1"),
+                nativeRoleMapping("roleMapping2")
+            ),
+            empty()
+        );
+        assertThat(
+            SecurityMigrations.CleanupRoleMappingDuplicatesMigration.getDuplicateRoleMappingNames(
+                nativeRoleMapping("roleMapping1"),
+                reservedRoleMapping("roleMapping1")
+            ),
+            equalTo(List.of("roleMapping1"))
+        );
+
+        {
+            List<String> duplicates = SecurityMigrations.CleanupRoleMappingDuplicatesMigration.getDuplicateRoleMappingNames(
+                nativeRoleMapping("roleMapping1"),
+                nativeRoleMapping("roleMapping2"),
+                reservedRoleMapping("roleMapping1"),
+                reservedRoleMapping("roleMapping2")
+            );
+            assertThat(duplicates, hasSize(2));
+            assertThat(duplicates, containsInAnyOrder("roleMapping1", "roleMapping2"));
+        }
+        {
+            List<String> duplicates = SecurityMigrations.CleanupRoleMappingDuplicatesMigration.getDuplicateRoleMappingNames(
+                nativeRoleMapping("roleMapping1"),
+                nativeRoleMapping("roleMapping2"),
+                nativeRoleMapping("roleMapping3"),
+                reservedRoleMapping("roleMapping1"),
+                reservedRoleMapping("roleMapping2"),
+                reservedRoleMapping("roleMapping4")
+            );
+            assertThat(duplicates, hasSize(2));
+            assertThat(duplicates, containsInAnyOrder("roleMapping1", "roleMapping2"));
+        }
+        {
+            List<String> duplicates = SecurityMigrations.CleanupRoleMappingDuplicatesMigration.getDuplicateRoleMappingNames(
+                nativeRoleMapping("roleMapping1" + ExpressionRoleMapping.READ_ONLY_ROLE_MAPPING_SUFFIX),
+                nativeRoleMapping("roleMapping2" + ExpressionRoleMapping.READ_ONLY_ROLE_MAPPING_SUFFIX),
+                nativeRoleMapping("roleMapping3"),
+                reservedRoleMapping("roleMapping1"),
+                reservedRoleMapping("roleMapping2"),
+                reservedRoleMapping("roleMapping3")
+            );
+            assertThat(duplicates, hasSize(1));
+            assertThat(duplicates, containsInAnyOrder("roleMapping3"));
+        }
+    }
+
+    private static ExpressionRoleMapping reservedRoleMapping(String name) {
+        return new ExpressionRoleMapping(
+            name + ExpressionRoleMapping.READ_ONLY_ROLE_MAPPING_SUFFIX,
+            null,
+            null,
+            null,
+            Map.of(ExpressionRoleMapping.READ_ONLY_ROLE_MAPPING_METADATA_FLAG, true),
+            true
+        );
+    }
+
+    private static ExpressionRoleMapping nativeRoleMapping(String name) {
+        return new ExpressionRoleMapping(name, null, null, null, randomBoolean() ? null : Map.of(), true);
+    }
+
+    public void testCleanupRoleMappingDuplicatesMigrationPartialFailure() {
+        // Make sure migration continues even if a duplicate is not found
+        SecurityIndexManager securityIndexManager = mock(SecurityIndexManager.class);
+        when(securityIndexManager.getRoleMappingsCleanupMigrationStatus()).thenReturn(
+            SecurityIndexManager.RoleMappingsCleanupMigrationStatus.READY
+        );
+        doAnswer(inv -> {
+            final Object[] args = inv.getArguments();
+            @SuppressWarnings("unchecked")
+            ActionListener<GetRoleMappingsResponse> listener = (ActionListener<GetRoleMappingsResponse>) args[2];
+            listener.onResponse(
+                new GetRoleMappingsResponse(
+                    nativeRoleMapping("duplicate-0"),
+                    reservedRoleMapping("duplicate-0"),
+                    nativeRoleMapping("duplicate-1"),
+                    reservedRoleMapping("duplicate-1"),
+                    nativeRoleMapping("duplicate-2"),
+                    reservedRoleMapping("duplicate-2")
+                )
+            );
+            return null;
+        }).when(client).execute(eq(GetRoleMappingsAction.INSTANCE), any(GetRoleMappingsRequest.class), any());
+
+        final boolean[] duplicatesDeleted = new boolean[3];
+        doAnswer(inv -> {
+            final Object[] args = inv.getArguments();
+            @SuppressWarnings("unchecked")
+            ActionListener<DeleteRoleMappingResponse> listener = (ActionListener<DeleteRoleMappingResponse>) args[2];
+            DeleteRoleMappingRequest request = (DeleteRoleMappingRequest) args[1];
+            if (request.getName().equals("duplicate-0")) {
+                duplicatesDeleted[0] = true;
+            }
+            if (request.getName().equals("duplicate-1")) {
+                if (randomBoolean()) {
+                    listener.onResponse(new DeleteRoleMappingResponse(false));
+                } else {
+                    listener.onFailure(new IllegalStateException("bad state"));
+                }
+            }
+            if (request.getName().equals("duplicate-2")) {
+                duplicatesDeleted[2] = true;
+            }
+            return null;
+        }).when(client).execute(eq(DeleteRoleMappingAction.INSTANCE), any(DeleteRoleMappingRequest.class), any());
+
+        SecurityMigrations.SecurityMigration securityMigration = new SecurityMigrations.CleanupRoleMappingDuplicatesMigration();
+        securityMigration.migrate(securityIndexManager, client, ActionListener.noop());
+
+        assertTrue(duplicatesDeleted[0]);
+        assertFalse(duplicatesDeleted[1]);
+        assertTrue(duplicatesDeleted[2]);
+    }
+
+    @Before
+    public void createClientAndThreadPool() {
+        threadPool = new TestThreadPool("cleanup role mappings test pool");
+        client = mock(Client.class);
+        when(client.threadPool()).thenReturn(threadPool);
+    }
+
+    @After
+    public void stopThreadPool() {
+        terminate(threadPool);
+    }
+
+}
diff --git a/x-pack/plugin/spatial/build.gradle b/x-pack/plugin/spatial/build.gradle
index e111949724844..5bcec68c227ce 100644
--- a/x-pack/plugin/spatial/build.gradle
+++ b/x-pack/plugin/spatial/build.gradle
@@ -15,7 +15,7 @@ dependencies {
   compileOnly project(':modules:lang-painless:spi')
   compileOnly project(path: xpackModule('core'))
   api "org.apache.lucene:lucene-spatial3d:${versions.lucene}"
-  api project(":libs:elasticsearch-h3")
+  api project(":libs:h3")
   testImplementation(testArtifact(project(xpackModule('core'))))
   testImplementation project(path: ':modules:percolator')
   testImplementation project(path: xpackModule('vector-tile'))
diff --git a/x-pack/plugin/sql/sql-action/build.gradle b/x-pack/plugin/sql/sql-action/build.gradle
index 9a0aefac4e434..60e809df00ae0 100644
--- a/x-pack/plugin/sql/sql-action/build.gradle
+++ b/x-pack/plugin/sql/sql-action/build.gradle
@@ -11,10 +11,10 @@ dependencies {
   api(project(':server')) {
     transitive = false
   }
-  api(project(':libs:elasticsearch-core')) {
+  api(project(':libs:core')) {
     transitive = false
   }
-  api(project(':libs:elasticsearch-x-content')) {
+  api(project(':libs:x-content')) {
     transitive = false
   }
   api project(':x-pack:plugin:core')
@@ -33,4 +33,4 @@ tasks.named('forbiddenApisMain').configure {
 tasks.named("dependencyLicenses").configure {
   mapping from: /jackson-.*/, to: 'jackson'
   mapping from: /lucene-.*/, to: 'lucene'
-}
\ No newline at end of file
+}
diff --git a/x-pack/plugin/sql/sql-cli/build.gradle b/x-pack/plugin/sql/sql-cli/build.gradle
index 1d3a63ec13c98..b9713bcb8e7a3 100644
--- a/x-pack/plugin/sql/sql-cli/build.gradle
+++ b/x-pack/plugin/sql/sql-cli/build.gradle
@@ -29,7 +29,7 @@ dependencies {
   api "org.jline:jline-style:${jlineVersion}"
 
   api project(':x-pack:plugin:sql:sql-client')
-  api project(":libs:elasticsearch-cli")
+  api project(":libs:cli")
   implementation "net.java.dev.jna:jna:${versions.jna}"
   testImplementation project(":test:framework")
 }
diff --git a/x-pack/plugin/sql/sql-proto/build.gradle b/x-pack/plugin/sql/sql-proto/build.gradle
index de3f3462da85e..2cb1cfa89f033 100644
--- a/x-pack/plugin/sql/sql-proto/build.gradle
+++ b/x-pack/plugin/sql/sql-proto/build.gradle
@@ -10,9 +10,9 @@ dependencies {
   api "com.fasterxml.jackson.core:jackson-core:${versions.jackson}"
   api "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:${versions.jackson}"
 
-  testImplementation project(":libs:elasticsearch-x-content")
+  testImplementation project(":libs:x-content")
   testImplementation(project(":test:framework")) {
-    exclude group: 'org.elasticsearch', module: 'elasticsearch-x-content'
+    exclude group: 'org.elasticsearch', module: 'x-content'
   }
 }
 
diff --git a/x-pack/plugin/text-structure/build.gradle b/x-pack/plugin/text-structure/build.gradle
index cab7f3ceeaa13..5bb6d8ef50274 100644
--- a/x-pack/plugin/text-structure/build.gradle
+++ b/x-pack/plugin/text-structure/build.gradle
@@ -12,7 +12,7 @@ base {
 dependencies {
   compileOnly project(path: xpackModule('core'))
   testImplementation(testArtifact(project(xpackModule('core'))))
-  api project(':libs:elasticsearch-grok')
+  api project(':libs:grok')
   api "com.ibm.icu:icu4j:${versions.icu4j}"
   api "net.sf.supercsv:super-csv:${versions.supercsv}"
 }
diff --git a/x-pack/plugin/transform/qa/common/build.gradle b/x-pack/plugin/transform/qa/common/build.gradle
index 9e7abfa2f977e..28e4068d31c6b 100644
--- a/x-pack/plugin/transform/qa/common/build.gradle
+++ b/x-pack/plugin/transform/qa/common/build.gradle
@@ -1,7 +1,7 @@
 apply plugin: 'elasticsearch.internal-java-rest-test'
 
 dependencies {
-  api project(':libs:elasticsearch-x-content')
+  api project(':libs:x-content')
   api project(':test:framework')
   api project(xpackModule('core'))
 }
diff --git a/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/feature/FeatureFactory.java b/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/feature/FeatureFactory.java
index 0c4ff1780ae1e..b5f9088edc4be 100644
--- a/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/feature/FeatureFactory.java
+++ b/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/feature/FeatureFactory.java
@@ -307,8 +307,11 @@ private static org.locationtech.jts.geom.Geometry clipGeometry(
                         return null;
                     }
                 } catch (TopologyException ex) {
-                    // we should never get here but just to be super safe because a TopologyException will kill the node
-                    throw new IllegalArgumentException(ex);
+                    // Note we should never throw a TopologyException as it kill the node
+                    // unfortunately the intersection method is not perfect and it will throw this error for complex
+                    // geometries even when valid. We can still simplify such geometry so we just return the original and
+                    // let the simplification process handle it.
+                    return geometry;
                 }
             }
         }
diff --git a/x-pack/qa/rolling-upgrade/build.gradle b/x-pack/qa/rolling-upgrade/build.gradle
index b9b0531fa5b68..38fbf99068a9b 100644
--- a/x-pack/qa/rolling-upgrade/build.gradle
+++ b/x-pack/qa/rolling-upgrade/build.gradle
@@ -88,6 +88,8 @@ BuildParams.bwcVersions.withWireCompatible { bwcVersion, baseName ->
     keystore 'xpack.watcher.encryption_key', file("${project.projectDir}/src/test/resources/system_key")
     setting 'xpack.watcher.encrypt_sensitive_data', 'true'
 
+    extraConfigFile 'operator/settings.json', file("${project.projectDir}/src/test/resources/operator_defined_role_mappings.json")
+
     // Old versions of the code contain an invalid assertion that trips
     // during tests.  Versions 5.6.9 and 6.2.4 have been fixed by removing
     // the assertion, but this is impossible for released versions.
diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/AbstractUpgradeTestCase.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/AbstractUpgradeTestCase.java
index 4324aed5fee18..b17644cd1c2a9 100644
--- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/AbstractUpgradeTestCase.java
+++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/AbstractUpgradeTestCase.java
@@ -9,11 +9,13 @@
 import org.elasticsearch.Build;
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
+import org.elasticsearch.client.RestClient;
 import org.elasticsearch.common.io.Streams;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.core.Booleans;
+import org.elasticsearch.test.XContentTestUtils;
 import org.elasticsearch.test.rest.ESRestTestCase;
 import org.elasticsearch.xpack.test.SecuritySettingsSourceField;
 import org.junit.Before;
@@ -21,6 +23,7 @@
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
+import java.util.Map;
 import java.util.stream.Collectors;
 
 public abstract class AbstractUpgradeTestCase extends ESRestTestCase {
@@ -149,4 +152,22 @@ public void setupForTests() throws Exception {
             }
         });
     }
+
+    protected static void waitForSecurityMigrationCompletion(RestClient adminClient, int version) throws Exception {
+        final Request request = new Request("GET", "_cluster/state/metadata/.security-7");
+        assertBusy(() -> {
+            Map<String, Object> indices = new XContentTestUtils.JsonMapView(entityAsMap(adminClient.performRequest(request))).get(
+                "metadata.indices"
+            );
+            assertNotNull(indices);
+            assertTrue(indices.containsKey(".security-7"));
+            // JsonMapView doesn't support . prefixed indices (splits on .)
+            @SuppressWarnings("unchecked")
+            String responseVersion = new XContentTestUtils.JsonMapView((Map<String, Object>) indices.get(".security-7")).get(
+                "migration_version.version"
+            );
+            assertNotNull(responseVersion);
+            assertTrue(Integer.parseInt(responseVersion) >= version);
+        });
+    }
 }
diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/SecurityIndexRoleMappingCleanupIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/SecurityIndexRoleMappingCleanupIT.java
new file mode 100644
index 0000000000000..82d4050c044b1
--- /dev/null
+++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/SecurityIndexRoleMappingCleanupIT.java
@@ -0,0 +1,146 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.upgrades;
+
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.RestClient;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.xpack.core.security.authc.support.mapper.ExpressionRoleMapping;
+import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+
+import static org.elasticsearch.TransportVersions.V_8_15_0;
+import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder;
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.Matchers.containsInAnyOrder;
+
+public class SecurityIndexRoleMappingCleanupIT extends AbstractUpgradeTestCase {
+
+    public void testCleanupDuplicateMappings() throws Exception {
+        if (CLUSTER_TYPE == ClusterType.OLD) {
+            // If we're in a state where the same operator-defined role mappings can exist both in cluster state and the native store
+            // (V_8_15_0 transport added to security.role_mapping_cleanup feature added), create a state
+            // where the native store will need to be cleaned up
+            assumeTrue(
+                "Cleanup only needed before security.role_mapping_cleanup feature available in cluster",
+                clusterHasFeature("security.role_mapping_cleanup") == false
+            );
+            assumeTrue(
+                "If role mappings are in cluster state but cleanup has not been performed yet, create duplicated role mappings",
+                minimumTransportVersion().onOrAfter(V_8_15_0)
+            );
+            // Since the old cluster has role mappings in cluster state, but doesn't check duplicates, create duplicates
+            createNativeRoleMapping("operator_role_mapping_1", Map.of("meta", "test"), true);
+            createNativeRoleMapping("operator_role_mapping_2", Map.of("meta", "test"), true);
+        } else if (CLUSTER_TYPE == ClusterType.MIXED) {
+            // Create a native role mapping that doesn't conflict with anything before the migration run
+            createNativeRoleMapping("no_name_conflict", Map.of("meta", "test"));
+        } else if (CLUSTER_TYPE == ClusterType.UPGRADED) {
+            waitForSecurityMigrationCompletion(adminClient(), 2);
+            assertAllRoleMappings(
+                client(),
+                "operator_role_mapping_1" + ExpressionRoleMapping.READ_ONLY_ROLE_MAPPING_SUFFIX,
+                "operator_role_mapping_2" + ExpressionRoleMapping.READ_ONLY_ROLE_MAPPING_SUFFIX,
+                "no_name_conflict"
+            );
+            // In the old cluster we might have created these (depending on the node features), so make sure they were removed
+            assertFalse(roleMappingExistsInSecurityIndex("operator_role_mapping_1"));
+            assertFalse(roleMappingExistsInSecurityIndex("operator_role_mapping_2"));
+            assertTrue(roleMappingExistsInSecurityIndex("no_name_conflict"));
+            // Make sure we can create and delete a conflicting role mapping again
+            createNativeRoleMapping("operator_role_mapping_1", Map.of("meta", "test"), true);
+            deleteNativeRoleMapping("operator_role_mapping_1", true);
+        }
+    }
+
+    @SuppressWarnings("unchecked")
+    private boolean roleMappingExistsInSecurityIndex(String mappingName) throws IOException {
+        final Request request = new Request("POST", "/.security/_search");
+        request.setJsonEntity(String.format(Locale.ROOT, """
+            {"query":{"bool":{"must":[{"term":{"_id":"%s_%s"}}]}}}""", "role-mapping", mappingName));
+
+        request.setOptions(
+            expectWarnings(
+                "this request accesses system indices: [.security-7],"
+                    + " but in a future major version, direct access to system indices will be prevented by default"
+            )
+        );
+
+        Response response = adminClient().performRequest(request);
+        assertOK(response);
+        final Map<String, Object> responseMap = responseAsMap(response);
+
+        Map<String, Object> hits = ((Map<String, Object>) responseMap.get("hits"));
+        return ((List<Object>) hits.get("hits")).isEmpty() == false;
+    }
+
+    private void createNativeRoleMapping(String roleMappingName, Map<String, Object> metadata) throws IOException {
+        createNativeRoleMapping(roleMappingName, metadata, false);
+    }
+
+    private void createNativeRoleMapping(String roleMappingName, Map<String, Object> metadata, boolean expectWarning) throws IOException {
+        final Request request = new Request("POST", "/_security/role_mapping/" + roleMappingName);
+        if (expectWarning) {
+            request.setOptions(
+                expectWarnings(
+                    "A read-only role mapping with the same name ["
+                        + roleMappingName
+                        + "] has been previously defined in a configuration file. "
+                        + "Both role mappings will be used to determine role assignments."
+                )
+            );
+        }
+
+        BytesReference source = BytesReference.bytes(
+            jsonBuilder().map(
+                Map.of(
+                    ExpressionRoleMapping.Fields.ROLES.getPreferredName(),
+                    List.of("superuser"),
+                    ExpressionRoleMapping.Fields.ENABLED.getPreferredName(),
+                    true,
+                    ExpressionRoleMapping.Fields.RULES.getPreferredName(),
+                    Map.of("field", Map.of("username", "role-mapping-test-user")),
+                    RoleDescriptor.Fields.METADATA.getPreferredName(),
+                    metadata
+                )
+            )
+        );
+        request.setJsonEntity(source.utf8ToString());
+        assertOK(client().performRequest(request));
+    }
+
+    private void deleteNativeRoleMapping(String roleMappingName, boolean expectWarning) throws IOException {
+        final Request request = new Request("DELETE", "/_security/role_mapping/" + roleMappingName);
+        if (expectWarning) {
+            request.setOptions(
+                expectWarnings(
+                    "A read-only role mapping with the same name ["
+                        + roleMappingName
+                        + "] has previously been defined in a configuration file. "
+                        + "The native role mapping was deleted, but the read-only mapping will remain active "
+                        + "and will be used to determine role assignments."
+                )
+            );
+        }
+        assertOK(client().performRequest(request));
+    }
+
+    private void assertAllRoleMappings(RestClient client, String... roleNames) throws IOException {
+        Request request = new Request("GET", "/_security/role_mapping");
+        Response response = client.performRequest(request);
+        assertOK(response);
+        Map<String, Object> responseMap = responseAsMap(response);
+
+        assertThat(responseMap.keySet(), containsInAnyOrder(roleNames));
+        assertThat(responseMap.size(), is(roleNames.length));
+    }
+}
diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/SecurityIndexRolesMetadataMigrationIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/SecurityIndexRolesMetadataMigrationIT.java
index d31130e970f03..6c34e68297aa0 100644
--- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/SecurityIndexRolesMetadataMigrationIT.java
+++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/SecurityIndexRolesMetadataMigrationIT.java
@@ -58,7 +58,7 @@ public void testRoleMigration() throws Exception {
         } else if (CLUSTER_TYPE == ClusterType.UPGRADED) {
             createRoleWithMetadata(upgradedTestRole, Map.of("meta", "test"));
             assertTrue(canRolesBeMigrated());
-            waitForMigrationCompletion(adminClient());
+            waitForSecurityMigrationCompletion(adminClient(), 1);
             assertMigratedDocInSecurityIndex(oldTestRole, "meta", "test");
             assertMigratedDocInSecurityIndex(mixed1TestRole, "meta", "test");
             assertMigratedDocInSecurityIndex(mixed2TestRole, "meta", "test");
@@ -136,23 +136,6 @@ private static void assertNoMigration(RestClient adminClient) throws Exception {
         );
     }
 
-    @SuppressWarnings("unchecked")
-    private static void waitForMigrationCompletion(RestClient adminClient) throws Exception {
-        final Request request = new Request("GET", "_cluster/state/metadata/" + INTERNAL_SECURITY_MAIN_INDEX_7);
-        assertBusy(() -> {
-            Response response = adminClient.performRequest(request);
-            assertOK(response);
-            Map<String, Object> responseMap = responseAsMap(response);
-            Map<String, Object> indicesMetadataMap = (Map<String, Object>) ((Map<String, Object>) responseMap.get("metadata")).get(
-                "indices"
-            );
-            assertTrue(indicesMetadataMap.containsKey(INTERNAL_SECURITY_MAIN_INDEX_7));
-            assertTrue(
-                ((Map<String, Object>) indicesMetadataMap.get(INTERNAL_SECURITY_MAIN_INDEX_7)).containsKey(MIGRATION_VERSION_CUSTOM_KEY)
-            );
-        });
-    }
-
     private void createRoleWithMetadata(String roleName, Map<String, Object> metadata) throws IOException {
         final Request request = new Request("POST", "/_security/role/" + roleName);
         BytesReference source = BytesReference.bytes(
diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/operator_defined_role_mappings.json b/x-pack/qa/rolling-upgrade/src/test/resources/operator_defined_role_mappings.json
new file mode 100644
index 0000000000000..d897cabb8ab01
--- /dev/null
+++ b/x-pack/qa/rolling-upgrade/src/test/resources/operator_defined_role_mappings.json
@@ -0,0 +1,38 @@
+{
+  "metadata": {
+    "version": "2",
+    "compatibility": "8.4.0"
+  },
+  "state": {
+    "role_mappings": {
+      "operator_role_mapping_1": {
+        "enabled": true,
+        "roles": [
+          "kibana_user"
+        ],
+        "metadata": {
+          "from_file": true
+        },
+        "rules": {
+          "field": {
+            "username": "role-mapping-test-user"
+          }
+        }
+      },
+      "operator_role_mapping_2": {
+        "enabled": true,
+        "roles": [
+          "fleet_user"
+        ],
+        "metadata": {
+          "from_file": true
+        },
+        "rules": {
+          "field": {
+            "username": "role-mapping-test-user"
+          }
+        }
+      }
+    }
+  }
+}