From 04bf642d9f3c169032558d14d717f804e3e749e8 Mon Sep 17 00:00:00 2001
From: Ignacio Vera <ignacio.vera@elastic.co>
Date: Wed, 24 Apr 2024 16:30:21 +0200
Subject: [PATCH 01/58] Validate stats formatting in standard InternalStats
 constructor (#107678)

We want to validate stats formatting before we serialize to XContent, as chunked x-content serialization
 assumes that we don't throw exceptions at that point. It is not necessary to do it in the StreamInput constructor
as this one has been serialise from an already checked object.

This commit adds starts formatting validation to the standard InternalStats constructor.
---
 docs/changelog/107678.yaml                    |  6 +++
 .../stats_metric_fail_formatting.yml          | 42 +++++++++++++++++++
 .../aggregations/metrics/InternalStats.java   | 18 ++++++++
 3 files changed, 66 insertions(+)
 create mode 100644 docs/changelog/107678.yaml
 create mode 100644 modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/stats_metric_fail_formatting.yml

diff --git a/docs/changelog/107678.yaml b/docs/changelog/107678.yaml
new file mode 100644
index 0000000000000..9be55dd4d6b96
--- /dev/null
+++ b/docs/changelog/107678.yaml
@@ -0,0 +1,6 @@
+pr: 107678
+summary: Validate stats formatting in standard `InternalStats` constructor
+area: Aggregations
+type: bug
+issues:
+ - 107671
diff --git a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/stats_metric_fail_formatting.yml b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/stats_metric_fail_formatting.yml
new file mode 100644
index 0000000000000..650c8447c5b10
--- /dev/null
+++ b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/stats_metric_fail_formatting.yml
@@ -0,0 +1,42 @@
+setup:
+  - do:
+      indices.create:
+          index: test_date
+          body:
+            mappings:
+              properties:
+                date_field:
+                   type : date
+                   format: date_hour_minute_second_millis
+
+
+  - do:
+       bulk:
+         refresh: true
+         body:
+           - index:
+               _index: test_date
+               _id:    "1"
+           - date_field: 9999-01-01T00:00:00.000
+           - index:
+               _index: test_date
+               _id:    "2"
+           - date_field: 9999-01-01T00:00:00.000
+
+---
+"fail formatting":
+
+  - skip:
+      version: "- 8.14.99"
+      reason: fixed in 8.15.0
+  - do:
+      catch: /Cannot format stat \[sum\] with format \[DocValueFormat.DateTime\(format\[date_hour_minute_second_millis\] locale\[\], Z, MILLISECONDS\)\]/
+      search:
+        index: test_date
+        body:
+          size: 0
+          aggs:
+            the_stats:
+              stats:
+                field: date_field
+
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalStats.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalStats.java
index 362708ec1f624..9d2079b9f1e96 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalStats.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalStats.java
@@ -70,6 +70,24 @@ public InternalStats(
         this.sum = sum;
         this.min = min;
         this.max = max;
+        verifyFormattingStats();
+    }
+
+    private void verifyFormattingStats() {
+        if (format != DocValueFormat.RAW) {
+            verifyFormattingStat(Fields.MIN, format, min);
+            verifyFormattingStat(Fields.MAX, format, max);
+            verifyFormattingStat(Fields.AVG, format, getAvg());
+            verifyFormattingStat(Fields.SUM, format, sum);
+        }
+    }
+
+    private static void verifyFormattingStat(String stat, DocValueFormat format, double value) {
+        try {
+            format.format(value);
+        } catch (Exception e) {
+            throw new IllegalArgumentException("Cannot format stat [" + stat + "] with format [" + format.toString() + "]", e);
+        }
     }
 
     /**

From 62c3d9d2a1017cec941be64cfb8b7f4a18990868 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Lorenzo=20Dematt=C3=A9?= <lorenzo.dematte@elastic.co>
Date: Wed, 24 Apr 2024 17:06:48 +0200
Subject: [PATCH 02/58] Introduce time limit to
 CacheTests#testDependentKeyDeadlock (#107845)

---
 .../org/elasticsearch/common/cache/CacheTests.java   | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/server/src/test/java/org/elasticsearch/common/cache/CacheTests.java b/server/src/test/java/org/elasticsearch/common/cache/CacheTests.java
index 6f74972e93988..0fb40583a4d79 100644
--- a/server/src/test/java/org/elasticsearch/common/cache/CacheTests.java
+++ b/server/src/test/java/org/elasticsearch/common/cache/CacheTests.java
@@ -644,7 +644,7 @@ public void testComputeIfAbsentThrowsExceptionIfLoaderReturnsANullValue() {
         }
     }
 
-    public void testDependentKeyDeadlock() {
+    public void testDependentKeyDeadlock() throws InterruptedException {
         class Key {
             private final int key;
 
@@ -673,6 +673,7 @@ public int hashCode() {
         final Cache<Key, Integer> cache = CacheBuilder.<Key, Integer>builder().build();
 
         CopyOnWriteArrayList<ExecutionException> failures = new CopyOnWriteArrayList<>();
+        AtomicBoolean reachedTimeLimit = new AtomicBoolean();
 
         CyclicBarrier barrier = new CyclicBarrier(1 + numberOfThreads);
         CountDownLatch deadlockLatch = new CountDownLatch(numberOfThreads);
@@ -682,7 +683,7 @@ public int hashCode() {
                 try {
                     safeAwait(barrier);
                     Random random = new Random(random().nextLong());
-                    for (int j = 0; j < numberOfEntries; j++) {
+                    for (int j = 0; j < numberOfEntries && reachedTimeLimit.get() == false; j++) {
                         Key key = new Key(random.nextInt(numberOfEntries));
                         try {
                             cache.computeIfAbsent(key, k -> {
@@ -734,7 +735,12 @@ public int hashCode() {
         // everything is setup, release the hounds
         safeAwait(barrier);
 
-        // wait for either deadlock to be detected or the threads to terminate
+        // run the test for a limited amount of time; if threads are still running after that, let them know and exit gracefully
+        if (deadlockLatch.await(1, TimeUnit.SECONDS) == false) {
+            reachedTimeLimit.set(true);
+        }
+
+        // wait for either deadlock to be detected or the threads to terminate (end operations or time limit reached)
         safeAwait(deadlockLatch);
 
         // shutdown the watchdog service

From a002b8962360723a143be23485c4e2b1304cc015 Mon Sep 17 00:00:00 2001
From: Dianna Hohensee <dianna.hohensee@elastic.co>
Date: Wed, 24 Apr 2024 11:33:24 -0400
Subject: [PATCH 03/58] Document REST layer in architecture guide (#107714)

---
 docs/internal/DistributedArchitectureGuide.md | 40 ++++++++++++++++++-
 1 file changed, 39 insertions(+), 1 deletion(-)

diff --git a/docs/internal/DistributedArchitectureGuide.md b/docs/internal/DistributedArchitectureGuide.md
index a89956721a481..b8fb92b1ea15d 100644
--- a/docs/internal/DistributedArchitectureGuide.md
+++ b/docs/internal/DistributedArchitectureGuide.md
@@ -79,10 +79,48 @@ caller timeouts.
 
 ### REST Layer
 
-(including how REST and Transport layers are bound together through the ActionModule)
+The REST and Transport layers are bound together through the `ActionModule`. `ActionModule#initRestHandlers` registers all the
+rest actions with a `RestController` that matches incoming requests to particular REST actions. `RestController#registerHandler`
+uses each `Rest*Action`'s `#routes()` implementation to match HTTP requests to that particular `Rest*Action`. Typically, REST
+actions follow the class naming convention `Rest*Action`, which makes them easier to find, but not always; the `#routes()`
+definition can also be helpful in finding a REST action. `RestController#dispatchRequest` eventually calls `#handleRequest` on a
+`RestHandler` implementation. `RestHandler` is the base class for `BaseRestHandler`, which most `Rest*Action` instances extend to
+implement a particular REST action.
+
+`BaseRestHandler#handleRequest` calls into `BaseRestHandler#prepareRequest`, which children `Rest*Action` classes extend to
+define the behavior for a particular action. `RestController#dispatchRequest` passes a `RestChannel` to the `Rest*Action` via
+`RestHandler#handleRequest`: `Rest*Action#prepareRequest` implementations return a `RestChannelConsumer` defining how to execute
+the action and reply on the channel (usually in the form of completing an ActionListener wrapper). `Rest*Action#prepareRequest`
+implementations are responsible for parsing the incoming request, and verifying that the structure of the request is valid.
+`BaseRestHandler#handleRequest` will then check that all the request parameters have been consumed: unexpected request parameters
+result in an error.
+
+### How REST Actions Connect to Transport Actions
+
+The Rest layer uses an implementation of `AbstractClient`. `BaseRestHandler#prepareRequest` takes a `NodeClient`: this client
+knows how to connect to a specified TransportAction. A `Rest*Action` implementation will return a `RestChannelConsumer` that
+most often invokes a call into a method on the `NodeClient` to pass through to the TransportAction. Along the way from
+`BaseRestHandler#prepareRequest` through the `AbstractClient` and `NodeClient` code, `NodeClient#executeLocally` is called: this
+method calls into `TaskManager#registerAndExecute`, registering the operation with the `TaskManager` so it can be found in Task
+API requests, before moving on to execute the specified TransportAction.
+
+`NodeClient` has a `NodeClient#actions` map from `ActionType` to `TransportAction`. `ActionModule#setupActions` registers all the
+core TransportActions, as well as those defined in any plugins that are being used: plugins can override `Plugin#getActions()` to
+define additional TransportActions. Note that not all TransportActions will be mapped back to a REST action: many TransportActions
+are only used for internode operations/communications.
 
 ### Transport Layer
 
+(Managed by the TransportService, TransportActions must be registered there, too)
+
+(Executing a TransportAction (either locally via NodeClient or remotely via TransportService) is where most of the authorization & other security logic runs)
+
+(What actions, and why, are registered in TransportService but not NodeClient?)
+
+### Direct Node to Node Transport Layer
+
+(TransportService maps incoming requests to TransportActions)
+
 ### Chunk Encoding
 
 #### XContent

From 137d333d74075c44eca1035f5eca4814879eaba1 Mon Sep 17 00:00:00 2001
From: Tommaso Teofili <tommaso.teofili@gmail.com>
Date: Wed, 24 Apr 2024 17:53:33 +0200
Subject: [PATCH 04/58] Vector formats in segments supported for all codecs
 (#107833)

---
 .../elasticsearch/index/engine/Engine.java    | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/index/engine/Engine.java b/server/src/main/java/org/elasticsearch/index/engine/Engine.java
index e55a9ff6b3291..2bba2a85a518e 100644
--- a/server/src/main/java/org/elasticsearch/index/engine/Engine.java
+++ b/server/src/main/java/org/elasticsearch/index/engine/Engine.java
@@ -58,7 +58,8 @@
 import org.elasticsearch.core.UpdateForV9;
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.VersionType;
-import org.elasticsearch.index.codec.PerFieldMapperCodec;
+import org.elasticsearch.index.codec.Elasticsearch814Codec;
+import org.elasticsearch.index.codec.LegacyPerFieldMapperCodec;
 import org.elasticsearch.index.mapper.DocumentParser;
 import org.elasticsearch.index.mapper.IdFieldMapper;
 import org.elasticsearch.index.mapper.LuceneDocument;
@@ -1096,14 +1097,14 @@ private void fillSegmentInfo(
         segment.attributes.putAll(info.info.getAttributes());
         Codec codec = info.info.getCodec();
         Map<String, List<String>> knnFormats = null;
-        if (includeVectorFormatsInfo && codec instanceof PerFieldMapperCodec) {
+        if (includeVectorFormatsInfo) {
             try {
                 FieldInfos fieldInfos = segmentReader.getFieldInfos();
                 if (fieldInfos.hasVectorValues()) {
                     for (FieldInfo fieldInfo : fieldInfos) {
                         String name = fieldInfo.getName();
                         if (fieldInfo.hasVectorValues()) {
-                            KnnVectorsFormat knnVectorsFormatForField = ((PerFieldMapperCodec) codec).getKnnVectorsFormatForField(name);
+                            KnnVectorsFormat knnVectorsFormatForField = getKnnVectorsFormatForField(codec, name);
                             if (knnFormats == null) {
                                 knnFormats = new HashMap<>();
                             }
@@ -1130,6 +1131,18 @@ private void fillSegmentInfo(
         segments.put(info.info.name, segment);
     }
 
+    private static KnnVectorsFormat getKnnVectorsFormatForField(Codec codec, String name) {
+        KnnVectorsFormat format;
+        if (codec instanceof Elasticsearch814Codec esCodec) {
+            format = esCodec.getKnnVectorsFormatForField(name);
+        } else if (codec instanceof LegacyPerFieldMapperCodec legacy) {
+            format = legacy.getKnnVectorsFormatForField(name);
+        } else {
+            format = codec.knnVectorsFormat();
+        }
+        return format;
+    }
+
     /**
      * The list of segments in the engine.
      */

From 6290f8de910396a22f0485f121be94100b20740b Mon Sep 17 00:00:00 2001
From: Oleksandr Kolomiiets <oleksandr.kolomiiets@elastic.co>
Date: Wed, 24 Apr 2024 08:56:47 -0700
Subject: [PATCH 05/58] Consider copy_to in synthetic source implementation for
 binary field (#107784)

---
 .../org/elasticsearch/index/mapper/BinaryFieldMapper.java    | 5 +++++
 .../elasticsearch/index/mapper/BinaryFieldMapperTests.java   | 5 -----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java
index 3d72d5f5d55a0..cfee70fb0000b 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java
@@ -202,6 +202,11 @@ protected String contentType() {
 
     @Override
     public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
+        if (copyTo.copyToFields().isEmpty() != true) {
+            throw new IllegalArgumentException(
+                "field [" + name() + "] of type [" + typeName() + "] doesn't support synthetic source because it declares copy_to"
+            );
+        }
         if (hasDocValues == false) {
             throw new IllegalArgumentException(
                 "field [" + name() + "] of type [" + typeName() + "] doesn't support synthetic source because it doesn't have doc values"
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BinaryFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BinaryFieldMapperTests.java
index aef02494b9a15..ae04963dd1e23 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/BinaryFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/BinaryFieldMapperTests.java
@@ -199,11 +199,6 @@ protected boolean supportsIgnoreMalformed() {
         return false;
     }
 
-    @Override
-    protected boolean supportsCopyTo() {
-        return false;
-    }
-
     @Override
     protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed) {
         return new SyntheticSourceSupport() {

From 0ac10c9d1f97839c9502512b32a51bcb1a2f74b3 Mon Sep 17 00:00:00 2001
From: Kostas Krikellas <131142368+kkrik-es@users.noreply.github.com>
Date: Wed, 24 Apr 2024 19:11:04 +0300
Subject: [PATCH 06/58] [TEST] wait for ILM in testRollupNonTSIndex (#107855)

look_ahead_time is 1m, need to wait for longer than that.

Related to #103981
---
 .../elasticsearch/xpack/ilm/actions/DownsampleActionIT.java | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/DownsampleActionIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/DownsampleActionIT.java
index 5ca5da555718b..15a370e994583 100644
--- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/DownsampleActionIT.java
+++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/DownsampleActionIT.java
@@ -406,7 +406,11 @@ public void testRollupNonTSIndex() throws Exception {
         updatePolicy(client(), index, policy);
 
         try {
-            assertBusy(() -> assertThat(getStepKeyForIndex(client(), index), equalTo(PhaseCompleteStep.finalStep(phaseName).getKey())));
+            assertBusy(
+                () -> assertThat(getStepKeyForIndex(client(), index), equalTo(PhaseCompleteStep.finalStep(phaseName).getKey())),
+                120,
+                TimeUnit.SECONDS
+            );
             String rollupIndex = getRollupIndexName(client(), index, fixedInterval);
             assertNull("Rollup index should not have been created", rollupIndex);
             assertTrue("Source index should not have been deleted", indexExists(index));

From cde894a5ce33a9b682200e858044f7fcf2bf8a8b Mon Sep 17 00:00:00 2001
From: Oleksandr Kolomiiets <oleksandr.kolomiiets@elastic.co>
Date: Wed, 24 Apr 2024 11:32:20 -0700
Subject: [PATCH 07/58] Implement synthetic source support for range fields
 (#107081)

* Implement synthetic source support for range fields

This PR adds basic synthetic source support for range fields. There are
following notable properties of synthetic source produced:
* Ranges are always normalized to be inclusive on both ends (this is how
 they are stored).
* Original order of ranges is not preserved.
* Date ranges are always expressed in epoch millis, format is not
preserved.
* IP ranges are always expressed as a range of IPs while it could
have been originally provided as a CIDR.

This PR only implements retrieval of data for source reconstruction from
 doc values.
---
 docs/changelog/107081.yaml                    |   5 +
 .../mapping/fields/synthetic-source.asciidoc  |   5 +
 docs/reference/mapping/types/range.asciidoc   | 206 ++++++++
 .../test/range/20_synthetic_source.yml        | 473 ++++++++++++++++++
 .../common/network/InetAddresses.java         |  33 ++
 .../index/mapper/BinaryRangeUtil.java         |   4 +
 .../index/mapper/RangeFieldMapper.java        | 188 ++++---
 .../elasticsearch/index/mapper/RangeType.java |  31 +-
 .../mapper/DateRangeFieldMapperTests.java     |  51 +-
 .../mapper/DoubleRangeFieldMapperTests.java   |  70 ++-
 .../mapper/FloatRangeFieldMapperTests.java    |  70 ++-
 .../mapper/IntegerRangeFieldMapperTests.java  |  21 +-
 .../index/mapper/IpRangeFieldMapperTests.java | 151 +++++-
 .../mapper/LongRangeFieldMapperTests.java     |  21 +-
 .../index/mapper/RangeFieldMapperTests.java   | 168 ++++++-
 15 files changed, 1416 insertions(+), 81 deletions(-)
 create mode 100644 docs/changelog/107081.yaml
 create mode 100644 rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/range/20_synthetic_source.yml

diff --git a/docs/changelog/107081.yaml b/docs/changelog/107081.yaml
new file mode 100644
index 0000000000000..2acd2f919b476
--- /dev/null
+++ b/docs/changelog/107081.yaml
@@ -0,0 +1,5 @@
+pr: 107081
+summary: Implement synthetic source support for range fields
+area: Mapping
+type: feature
+issues: []
diff --git a/docs/reference/mapping/fields/synthetic-source.asciidoc b/docs/reference/mapping/fields/synthetic-source.asciidoc
index 8b84c73eb6586..ec6f51f78eda5 100644
--- a/docs/reference/mapping/fields/synthetic-source.asciidoc
+++ b/docs/reference/mapping/fields/synthetic-source.asciidoc
@@ -57,6 +57,7 @@ types:
 ** <<ip-synthetic-source,`ip`>>
 ** <<keyword-synthetic-source,`keyword`>>
 ** <<numeric-synthetic-source,`long`>>
+** <<range-synthetic-source,`range` types>>
 ** <<numeric-synthetic-source,`scaled_float`>>
 ** <<numeric-synthetic-source,`short`>>
 ** <<text-synthetic-source,`text`>>
@@ -170,3 +171,7 @@ https://www.rfc-editor.org/rfc/rfc7159.html[JSON RFC] defines objects as
 shouldn't care but without synthetic `_source` the original ordering is
 preserved and some applications may, counter to the spec, do something with
 that ordering.
+
+[[synthetic-source-modifications-ranges]]
+====== Representation of ranges
+Range field vales (e.g. `long_range`) are always represented as inclusive on both sides with bounds adjusted accordingly.  See <<range-synthetic-source-inclusive, examples>>.
diff --git a/docs/reference/mapping/types/range.asciidoc b/docs/reference/mapping/types/range.asciidoc
index 3b1ce6a108178..3716b4b346209 100644
--- a/docs/reference/mapping/types/range.asciidoc
+++ b/docs/reference/mapping/types/range.asciidoc
@@ -220,6 +220,12 @@ The following parameters are accepted by range types:
     Try to convert strings to numbers and truncate fractions for integers.
     Accepts `true` (default) and `false`.
 
+<<doc-values,`doc_values`>>::
+
+    Should the field be stored on disk in a column-stride fashion, so that it
+    can later be used for sorting, aggregations, or scripting? Accepts `true`
+    (default) or `false`.
+
 <<mapping-index,`index`>>::
 
     Should the field be searchable? Accepts `true` (default) and `false`.
@@ -229,3 +235,203 @@ The following parameters are accepted by range types:
     Whether the field value should be stored and retrievable separately from
     the <<mapping-source-field,`_source`>> field. Accepts `true` or `false`
     (default).
+
+[[range-synthetic-source]]
+==== Synthetic `_source`
+
+IMPORTANT: Synthetic `_source` is Generally Available only for TSDB indices
+(indices that have `index.mode` set to `time_series`). For other indices
+synthetic `_source` is in technical preview. Features in technical preview may
+be changed or removed in a future release. Elastic will work to fix
+any issues, but features in technical preview are not subject to the support SLA
+of official GA features.
+
+`range` fields support <<synthetic-source,synthetic `_source`>> in their default
+configuration. Synthetic `_source` cannot be used with <<doc-values,`doc_values`>> disabled.
+
+Synthetic source always sorts values and removes duplicates for all `range` fields except `ip_range` . Ranges are sorted by their lower bound and then by upper bound. For example:
+[source,console,id=synthetic-source-range-sorting-example]
+----
+PUT idx
+{
+  "mappings": {
+    "_source": { "mode": "synthetic" },
+    "properties": {
+      "my_range": { "type": "long_range" }
+    }
+  }
+}
+
+PUT idx/_doc/1
+{
+  "my_range": [
+    {
+        "gte": 200,
+        "lte": 300
+    },
+    {
+        "gte": 1,
+        "lte": 100
+    },
+    {
+        "gte": 200,
+        "lte": 300
+    },
+    {
+        "gte": 200,
+        "lte": 500
+    }
+  ]
+}
+----
+// TEST[s/$/\nGET idx\/_doc\/1?filter_path=_source\n/]
+
+Will become:
+
+[source,console-result]
+----
+{
+  "my_range": [
+    {
+        "gte": 1,
+        "lte": 100
+    },
+    {
+        "gte": 200,
+        "lte": 300
+    },
+    {
+        "gte": 200,
+        "lte": 500
+    }
+  ]
+}
+----
+// TEST[s/^/{"_source":/ s/\n$/}/]
+
+Values of `ip_range` fields are not sorted but original order is not preserved. Duplicate ranges are removed. If `ip_range` field value is provided as a CIDR, it will be represented as a range of IP addresses in synthetic source.
+
+For example:
+[source,console,id=synthetic-source-range-ip-example]
+----
+PUT idx
+{
+  "mappings": {
+    "_source": { "mode": "synthetic" },
+    "properties": {
+      "my_range": { "type": "ip_range" }
+    }
+  }
+}
+
+PUT idx/_doc/1
+{
+  "my_range": [
+    "10.0.0.0/24",
+    {
+      "gte": "10.0.0.0",
+      "lte": "10.0.0.255"
+    }
+  ]
+}
+----
+// TEST[s/$/\nGET idx\/_doc\/1?filter_path=_source\n/]
+
+Will become:
+
+[source,console-result]
+----
+{
+  "my_range": {
+      "gte": "10.0.0.0",
+      "lte": "10.0.0.255"
+    }
+
+}
+----
+// TEST[s/^/{"_source":/ s/\n$/}/]
+
+[[range-synthetic-source-inclusive]]
+Range field vales are always represented as inclusive on both sides with bounds adjusted accordingly. For example:
+[source,console,id=synthetic-source-range-normalization-example]
+----
+PUT idx
+{
+  "mappings": {
+    "_source": { "mode": "synthetic" },
+    "properties": {
+      "my_range": { "type": "long_range" }
+    }
+  }
+}
+
+PUT idx/_doc/1
+{
+  "my_range": {
+    "gt": 200,
+    "lt": 300
+  }
+}
+----
+// TEST[s/$/\nGET idx\/_doc\/1?filter_path=_source\n/]
+
+Will become:
+
+[source,console-result]
+----
+{
+  "my_range": {
+    "gte": 201,
+    "lte": 299
+  }
+}
+----
+// TEST[s/^/{"_source":/ s/\n$/}/]
+
+`date` ranges are formatted using provided `format` or by default using `yyyy-MM-dd'T'HH:mm:ss.SSSZ` format. For example:
+[source,console,id=synthetic-source-range-date-example]
+----
+PUT idx
+{
+  "mappings": {
+    "_source": { "mode": "synthetic" },
+    "properties": {
+      "my_range": { "type": "date_range" }
+    }
+  }
+}
+
+PUT idx/_doc/1
+{
+  "my_range": [
+    {
+      "gte": 1504224000000,
+      "lte": 1504569600000
+    },
+    {
+      "gte": "2017-09-01",
+      "lte": "2017-09-10"
+    }
+  ]
+}
+----
+// TEST[s/$/\nGET idx\/_doc\/1?filter_path=_source\n/]
+
+Will become:
+
+[source,console-result]
+----
+{
+  "my_range": [
+    {
+      "gte": "2017-09-01T00:00:00.000Z",
+      "lte": "2017-09-05T00:00:00.000Z"
+    },
+    {
+      "gte": "2017-09-01T00:00:00.000Z",
+      "lte": "2017-09-10T00:00:00.000Z"
+    }
+  ]
+}
+----
+// TEST[s/^/{"_source":/ s/\n$/}/]
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/range/20_synthetic_source.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/range/20_synthetic_source.yml
new file mode 100644
index 0000000000000..eac0fb9a52aa2
--- /dev/null
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/range/20_synthetic_source.yml
@@ -0,0 +1,473 @@
+setup:
+  - skip:
+      version: " - 8.14.99"
+      reason: synthetic source support added in 8.15
+
+  - do:
+      indices.create:
+        index: synthetic_source_test
+        body:
+          mappings:
+            "_source":
+              "mode": "synthetic"
+            "properties":
+              "integer_range":
+                "type" : "integer_range"
+              "long_range":
+                "type" : "long_range"
+              "float_range":
+                "type" : "float_range"
+              "double_range":
+                "type" : "double_range"
+              "date_range":
+                "type" : "date_range"
+              "ip_range":
+                "type" : "ip_range"
+
+---
+"Integer range":
+
+  - do:
+      index:
+        index: synthetic_source_test
+        id: "1"
+        body: { "integer_range" : { "gte": 1, "lte": 5 } }
+
+  - do:
+      index:
+        index: synthetic_source_test
+        id: "2"
+        body: { "integer_range" : { "gt": 1, "lte": 3 } }
+
+  - do:
+      index:
+        index: synthetic_source_test
+        id: "3"
+        body: { "integer_range" : [ { "gte": 4, "lt": 5 } ] }
+
+  - do:
+      index:
+        index: synthetic_source_test
+        id: "4"
+        body: { "integer_range" : [ { "gt": 4, "lt": 8 }, { "gt": 4, "lt": 7 } ] }
+
+  - do:
+      index:
+        index: synthetic_source_test
+        id: "5"
+        body: { "integer_range" : null }
+
+  - do:
+      index:
+        index: synthetic_source_test
+        id: "6"
+        body: { "integer_range": { "gte": null, "lte": 10 } }
+
+  - do:
+      index:
+        index: synthetic_source_test
+        id: "7"
+        body: { "integer_range": { "gte": 1 } }
+
+  - do:
+      indices.refresh: {}
+
+  - do:
+      search:
+        index: synthetic_source_test
+  - match: { hits.total.value: 7 }
+  - match:
+      hits.hits.0._source:
+        integer_range:  { "gte": 1, "lte": 5 }
+  - match:
+      hits.hits.1._source:
+        integer_range: { "gte": 2, "lte": 3 }
+  - match:
+      hits.hits.2._source:
+        integer_range: { "gte": 4, "lte": 4 }
+  - match:
+      hits.hits.3._source:
+        integer_range: [ { "gte": 5, "lte": 6 }, { "gte": 5, "lte": 7 } ]
+  - match:
+      hits.hits.4._source: {}
+  - match:
+      hits.hits.5._source:
+        integer_range: { "gte": -2147483648, "lte": 10 }
+  - match:
+      hits.hits.6._source:
+        integer_range: { "gte": 1, "lte": 2147483647 }
+
+---
+"Long range":
+
+  - do:
+      index:
+        index: synthetic_source_test
+        id: "1"
+        body: { "long_range" : { "gte": 1, "lte": 5 } }
+
+  - do:
+      index:
+        index: synthetic_source_test
+        id: "2"
+        body: { "long_range" : { "gt": 1, "lte": 3 } }
+
+  - do:
+      index:
+        index: synthetic_source_test
+        id: "3"
+        body: { "long_range" : [ { "gte": 4, "lt": 5 } ] }
+
+  - do:
+      index:
+        index: synthetic_source_test
+        id: "4"
+        body: { "long_range" : [ { "gt": 4, "lt": 8 }, { "gt": 4, "lt": 7 } ] }
+
+  - do:
+      index:
+        index: synthetic_source_test
+        id: "5"
+        body: { "long_range" : null }
+
+  - do:
+      index:
+        index: synthetic_source_test
+        id: "6"
+        body: { "long_range": { "gte": null, "lte": 10 } }
+
+  - do:
+      index:
+        index: synthetic_source_test
+        id: "7"
+        body: { "long_range": { "gte": 1 } }
+
+  - do:
+      indices.refresh: {}
+
+  - do:
+      search:
+        index: synthetic_source_test
+  - match: { hits.total.value: 7 }
+  - match:
+      hits.hits.0._source:
+        long_range:  { "gte": 1, "lte": 5 }
+  - match:
+      hits.hits.1._source:
+        long_range: { "gte": 2, "lte": 3 }
+  - match:
+      hits.hits.2._source:
+        long_range: { "gte": 4, "lte": 4 }
+  - match:
+      hits.hits.3._source:
+        long_range: [ { "gte": 5, "lte": 6 }, { "gte": 5, "lte": 7 } ]
+  - match:
+      hits.hits.4._source: {}
+  - match:
+      hits.hits.5._source:
+        long_range: { "gte": -9223372036854775808, "lte": 10 }
+  - match:
+      hits.hits.6._source:
+        long_range: { "gte": 1, "lte": 9223372036854775807 }
+
+---
+"Float range":
+
+  - do:
+      index:
+        index: synthetic_source_test
+        id: "1"
+        body: { "float_range" : { "gte": 1, "lte": 5 } }
+
+  - do:
+      index:
+        index: synthetic_source_test
+        id: "2"
+        body: { "float_range" : [ { "gte": 4.0, "lte": 5.0 } ] }
+
+  - do:
+      index:
+        index: synthetic_source_test
+        id: "3"
+        body: { "float_range" : [ { "gte": 4, "lte": 8 }, { "gte": 4, "lte": 7 } ] }
+
+  - do:
+      index:
+        index: synthetic_source_test
+        id: "4"
+        body: { "float_range" : null }
+
+  - do:
+      index:
+        index: synthetic_source_test
+        id: "5"
+        body: { "float_range": { "gte": null, "lte": 10 } }
+
+  - do:
+      index:
+        index: synthetic_source_test
+        id: "6"
+        body: { "float_range": { "gte": 1 } }
+
+  - do:
+      indices.refresh: {}
+
+  - do:
+      search:
+        index: synthetic_source_test
+  - match: { hits.total.value: 6 }
+  - match:
+      hits.hits.0._source:
+        float_range:  { "gte": 1.0, "lte": 5.0 }
+  - match:
+      hits.hits.1._source:
+        float_range: { "gte": 4.0, "lte": 5.0 }
+  - match:
+      hits.hits.2._source:
+        float_range: [ { "gte": 4.0, "lte": 7.0 }, { "gte": 4.0, "lte": 8.0 } ]
+  - match:
+      hits.hits.3._source: {}
+  - match:
+      hits.hits.4._source:
+        float_range: { "gte": "-Infinity", "lte": 10.0 }
+  - match:
+      hits.hits.5._source:
+        float_range: { "gte": 1.0, "lte": "Infinity" }
+
+---
+"Double range":
+
+  - do:
+      index:
+        index: synthetic_source_test
+        id: "1"
+        body: { "double_range" : { "gte": 1, "lte": 5 } }
+
+  - do:
+      index:
+        index: synthetic_source_test
+        id: "2"
+        body: { "double_range" : [ { "gte": 4.0, "lte": 5.0 } ] }
+
+  - do:
+      index:
+        index: synthetic_source_test
+        id: "3"
+        body: { "double_range" : [ { "gte": 4, "lte": 8 }, { "gte": 4, "lte": 7 } ] }
+
+  - do:
+      index:
+        index: synthetic_source_test
+        id: "4"
+        body: { "double_range" : null }
+
+  - do:
+      index:
+        index: synthetic_source_test
+        id: "5"
+        body: { "double_range": { "gte": null, "lte": 10 } }
+
+  - do:
+      index:
+        index: synthetic_source_test
+        id: "6"
+        body: { "double_range": { "gte": 1 } }
+
+  - do:
+      indices.refresh: {}
+
+  - do:
+      search:
+        index: synthetic_source_test
+  - match: { hits.total.value: 6 }
+  - match:
+      hits.hits.0._source:
+        double_range:  { "gte": 1.0, "lte": 5.0 }
+  - match:
+      hits.hits.1._source:
+        double_range: { "gte": 4.0, "lte": 5.0 }
+  - match:
+      hits.hits.2._source:
+        double_range: [ { "gte": 4.0, "lte": 7.0 }, { "gte": 4.0, "lte": 8.0 } ]
+  - match:
+      hits.hits.3._source: {}
+  - match:
+      hits.hits.4._source:
+        double_range: { "gte": "-Infinity", "lte": 10.0 }
+  - match:
+      hits.hits.5._source:
+        double_range: { "gte": 1.0, "lte": "Infinity" }
+
+---
+"IP range":
+
+  - do:
+      index:
+        index: synthetic_source_test
+        id: "1"
+        body: { "ip_range" : { "gte": "192.168.0.1", "lte": "192.168.0.5" } }
+
+  - do:
+      index:
+        index: synthetic_source_test
+        id: "2"
+        body: { "ip_range" : { "gt": "192.168.0.1", "lte": "192.168.0.3" } }
+
+  - do:
+      index:
+        index: synthetic_source_test
+        id: "3"
+        body: { "ip_range" : [ { "gte": "192.168.0.4", "lt": "192.168.0.5" } ] }
+
+  - do:
+      index:
+        index: synthetic_source_test
+        id: "4"
+        body: { "ip_range" : { "gt": "2001:db8::", "lt": "200a:100::" } }
+
+  - do:
+      index:
+        index: synthetic_source_test
+        id: "5"
+        body: { "ip_range" : "74.125.227.0/25" }
+
+  - do:
+      index:
+        index: synthetic_source_test
+        id: "6"
+        body: { "ip_range" : null }
+
+  - do:
+      index:
+        index: synthetic_source_test
+        id: "7"
+        body: { "ip_range": { "gte": null, "lte": "10.10.10.10" } }
+
+  - do:
+      index:
+        index: synthetic_source_test
+        id: "8"
+        body: { "ip_range": { "gte": "2001:db8::" } }
+
+  - do:
+      indices.refresh: {}
+
+  - do:
+      search:
+        index: synthetic_source_test
+  - match: { hits.total.value: 8 }
+  - match:
+      hits.hits.0._source:
+        ip_range:  { "gte": "192.168.0.1", "lte": "192.168.0.5" }
+  - match:
+      hits.hits.1._source:
+        ip_range: { "gte": "192.168.0.2", "lte": "192.168.0.3" }
+  - match:
+      hits.hits.2._source:
+        ip_range: { "gte": "192.168.0.4", "lte": "192.168.0.4" }
+  - match:
+      hits.hits.3._source:
+        ip_range: { "gte": "2001:db8::1", "lte": "200a:ff:ffff:ffff:ffff:ffff:ffff:ffff" }
+  - match:
+      hits.hits.4._source:
+        ip_range: { "gte": "74.125.227.0", "lte": "74.125.227.127" }
+  - match:
+      hits.hits.5._source: {}
+  - match:
+      hits.hits.6._source:
+        ip_range: { "gte": "0.0.0.0", "lte": "10.10.10.10" }
+  - match:
+      hits.hits.7._source:
+        ip_range: { "gte": "2001:db8::", "lte": "ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff" }
+
+---
+"Date range":
+
+  - do:
+      index:
+        index: synthetic_source_test
+        id: "1"
+        body: { "date_range" : { "gte": "2017-09-01", "lte": "2017-09-05" } }
+
+  - do:
+      index:
+        index: synthetic_source_test
+        id: "2"
+        body: { "date_range" : { "gt": "2017-09-01", "lte": "2017-09-03" } }
+
+  - do:
+      index:
+        index: synthetic_source_test
+        id: "3"
+        body: { "date_range" : [ { "gte": "2017-09-04", "lt": "2017-09-05" } ] }
+
+  - do:
+      index:
+        index: synthetic_source_test
+        id: "4"
+        body: { "date_range" : [ { "gt": "2017-09-04", "lt": "2017-09-08" }, { "gt": "2017-09-04", "lt": "2017-09-07" } ] }
+
+  - do:
+      index:
+        index: synthetic_source_test
+        id: "5"
+        body: { "date_range" : { "gte": 1504224000000, "lte": 1504569600000 } }
+
+  - do:
+      index:
+        index: synthetic_source_test
+        id: "6"
+        body: { "date_range" : { "gte": "2017-09-01T10:20:30.123Z", "lte": "2017-09-05T03:04:05.789Z" } }
+
+  - do:
+      index:
+        index: synthetic_source_test
+        id: "7"
+        body: { "date_range" : null }
+
+  - do:
+      index:
+        index: synthetic_source_test
+        id: "8"
+        body: { "date_range": { "gte": null, "lte": "2017-09-05" } }
+
+  - do:
+      index:
+        index: synthetic_source_test
+        id: "9"
+        body: { "date_range": { "gte": "2017-09-05" } }
+
+  - do:
+      indices.refresh: {}
+
+  - do:
+      search:
+        index: synthetic_source_test
+  - match: { hits.total.value: 9 }
+  - match:
+      hits.hits.0._source:
+        date_range:  { "gte": "2017-09-01T00:00:00.000Z", "lte": "2017-09-05T00:00:00.000Z" }
+  - match:
+      hits.hits.1._source:
+        date_range: { "gte": "2017-09-01T00:00:00.001Z", "lte": "2017-09-03T00:00:00.000Z" }
+  - match:
+      hits.hits.2._source:
+        date_range: { "gte": "2017-09-04T00:00:00.000Z", "lte": "2017-09-04T23:59:59.999Z" }
+  - match:
+      hits.hits.3._source:
+        date_range: [ { "gte": "2017-09-04T00:00:00.001Z", "lte": "2017-09-06T23:59:59.999Z" }, { "gte": "2017-09-04T00:00:00.001Z", "lte": "2017-09-07T23:59:59.999Z" } ]
+  - match:
+      hits.hits.4._source:
+        date_range: { "gte": "2017-09-01T00:00:00.000Z", "lte": "2017-09-05T00:00:00.000Z" }
+  - match:
+      hits.hits.5._source:
+        date_range: { "gte": "2017-09-01T10:20:30.123Z", "lte": "2017-09-05T03:04:05.789Z" }
+  - match:
+      hits.hits.6._source: {}
+  - match:
+      hits.hits.7._source:
+        date_range: { "gte": "-292275055-05-16T16:47:04.192Z", "lte": "2017-09-05T00:00:00.000Z" }
+  - match:
+      hits.hits.8._source:
+        date_range: { "gte": "2017-09-05T00:00:00.000Z", "lte": "+292278994-08-17T07:12:55.807Z" }
+
diff --git a/server/src/main/java/org/elasticsearch/common/network/InetAddresses.java b/server/src/main/java/org/elasticsearch/common/network/InetAddresses.java
index 0f5b746b95967..e902d55848406 100644
--- a/server/src/main/java/org/elasticsearch/common/network/InetAddresses.java
+++ b/server/src/main/java/org/elasticsearch/common/network/InetAddresses.java
@@ -417,4 +417,37 @@ public static Tuple<InetAddress, Integer> parseCidr(String maskedAddress) {
     public static String toCidrString(InetAddress address, int prefixLength) {
         return new StringBuilder().append(toAddrString(address)).append("/").append(prefixLength).toString();
     }
+
+    /**
+     * Represents a range of IP addresses
+     * @param lowerBound start of the ip range (inclusive)
+     * @param upperBound end of the ip range (inclusive)
+     */
+    public record IpRange(InetAddress lowerBound, InetAddress upperBound) {}
+
+    /**
+     * Parse an IP address and its prefix length using the CIDR notation
+     * into a range of ip addresses corresponding to it.
+     * @param maskedAddress ip address range in a CIDR notation
+     * @throws IllegalArgumentException if the string is not formatted as {@code ip_address/prefix_length}
+     * @throws IllegalArgumentException if the IP address is an IPv6-mapped ipv4 address
+     * @throws IllegalArgumentException if the prefix length is not in 0-32 for IPv4 addresses and 0-128 for IPv6 addresses
+     * @throws NumberFormatException if the prefix length is not an integer
+     */
+    public static IpRange parseIpRangeFromCidr(String maskedAddress) {
+        final Tuple<InetAddress, Integer> cidr = InetAddresses.parseCidr(maskedAddress);
+        // create the lower value by zeroing out the host portion, upper value by filling it with all ones.
+        byte[] lower = cidr.v1().getAddress();
+        byte[] upper = lower.clone();
+        for (int i = cidr.v2(); i < 8 * lower.length; i++) {
+            int m = 1 << 7 - (i & 7);
+            lower[i >> 3] &= (byte) ~m;
+            upper[i >> 3] |= (byte) m;
+        }
+        try {
+            return new IpRange(InetAddress.getByAddress(lower), InetAddress.getByAddress(upper));
+        } catch (UnknownHostException bogus) {
+            throw new AssertionError(bogus);
+        }
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BinaryRangeUtil.java b/server/src/main/java/org/elasticsearch/index/mapper/BinaryRangeUtil.java
index 6c31b684fa791..9eec1b10a0635 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/BinaryRangeUtil.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/BinaryRangeUtil.java
@@ -98,6 +98,10 @@ static List<RangeFieldMapper.Range> decodeFloatRanges(BytesRef encodedRanges) th
         return decodeRanges(encodedRanges, RangeType.FLOAT, BinaryRangeUtil::decodeFloat);
     }
 
+    static List<RangeFieldMapper.Range> decodeDateRanges(BytesRef encodedRanges) throws IOException {
+        return decodeRanges(encodedRanges, RangeType.DATE, BinaryRangeUtil::decodeLong);
+    }
+
     static List<RangeFieldMapper.Range> decodeRanges(
         BytesRef encodedRanges,
         RangeType rangeType,
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java
index 3836915e65753..885e0c9f2642d 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java
@@ -26,16 +26,17 @@
 import org.elasticsearch.index.query.SearchExecutionContext;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
+import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentParser;
 
 import java.io.IOException;
 import java.net.InetAddress;
-import java.net.UnknownHostException;
 import java.time.ZoneId;
 import java.time.ZoneOffset;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
@@ -380,83 +381,116 @@ protected boolean supportsParsingObject() {
 
     @Override
     protected void parseCreateField(DocumentParserContext context) throws IOException {
-        Range range;
         XContentParser parser = context.parser();
-        final XContentParser.Token start = parser.currentToken();
-        if (start == XContentParser.Token.VALUE_NULL) {
+        if (parser.currentToken() == XContentParser.Token.VALUE_NULL) {
             return;
-        } else if (start == XContentParser.Token.START_OBJECT) {
-            RangeFieldType fieldType = fieldType();
-            RangeType rangeType = fieldType.rangeType;
-            String fieldName = null;
-            Object from = rangeType.minValue();
-            Object to = rangeType.maxValue();
-            boolean includeFrom = DEFAULT_INCLUDE_LOWER;
-            boolean includeTo = DEFAULT_INCLUDE_UPPER;
-            XContentParser.Token token;
-            while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-                if (token == XContentParser.Token.FIELD_NAME) {
-                    fieldName = parser.currentName();
-                } else {
-                    if (fieldName.equals(GT_FIELD.getPreferredName())) {
-                        includeFrom = false;
-                        if (parser.currentToken() != XContentParser.Token.VALUE_NULL) {
-                            from = rangeType.parseFrom(fieldType, parser, coerce.value(), includeFrom);
-                        }
-                    } else if (fieldName.equals(GTE_FIELD.getPreferredName())) {
-                        includeFrom = true;
-                        if (parser.currentToken() != XContentParser.Token.VALUE_NULL) {
-                            from = rangeType.parseFrom(fieldType, parser, coerce.value(), includeFrom);
-                        }
-                    } else if (fieldName.equals(LT_FIELD.getPreferredName())) {
-                        includeTo = false;
-                        if (parser.currentToken() != XContentParser.Token.VALUE_NULL) {
-                            to = rangeType.parseTo(fieldType, parser, coerce.value(), includeTo);
-                        }
-                    } else if (fieldName.equals(LTE_FIELD.getPreferredName())) {
-                        includeTo = true;
-                        if (parser.currentToken() != XContentParser.Token.VALUE_NULL) {
-                            to = rangeType.parseTo(fieldType, parser, coerce.value(), includeTo);
-                        }
-                    } else {
-                        throw new DocumentParsingException(
-                            parser.getTokenLocation(),
-                            "error parsing field [" + name() + "], with unknown parameter [" + fieldName + "]"
-                        );
-                    }
-                }
-            }
-            range = new Range(rangeType, from, to, includeFrom, includeTo);
-        } else if (fieldType().rangeType == RangeType.IP && start == XContentParser.Token.VALUE_STRING) {
-            range = parseIpRangeFromCidr(parser);
-        } else {
+        }
+
+        Range range = parseRange(parser);
+        context.doc().addAll(fieldType().rangeType.createFields(context, name(), range, index, hasDocValues, store));
+
+        if (hasDocValues == false && (index || store)) {
+            context.addToFieldNames(fieldType().name());
+        }
+    }
+
+    private Range parseRange(XContentParser parser) throws IOException {
+        final XContentParser.Token start = parser.currentToken();
+        if (fieldType().rangeType == RangeType.IP && start == XContentParser.Token.VALUE_STRING) {
+            return parseIpRangeFromCidr(parser);
+        }
+
+        if (start != XContentParser.Token.START_OBJECT) {
             throw new DocumentParsingException(
                 parser.getTokenLocation(),
                 "error parsing field [" + name() + "], expected an object but got " + parser.currentName()
             );
         }
-        context.doc().addAll(fieldType().rangeType.createFields(context, name(), range, index, hasDocValues, store));
 
-        if (hasDocValues == false && (index || store)) {
-            context.addToFieldNames(fieldType().name());
+        RangeFieldType fieldType = fieldType();
+        RangeType rangeType = fieldType.rangeType;
+        String fieldName = null;
+        Object parsedFrom = null;
+        Object parsedTo = null;
+        boolean includeFrom = DEFAULT_INCLUDE_LOWER;
+        boolean includeTo = DEFAULT_INCLUDE_UPPER;
+        XContentParser.Token token;
+        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+            if (token == XContentParser.Token.FIELD_NAME) {
+                fieldName = parser.currentName();
+            } else {
+                if (fieldName.equals(GT_FIELD.getPreferredName())) {
+                    includeFrom = false;
+                    if (parser.currentToken() != XContentParser.Token.VALUE_NULL) {
+                        parsedFrom = rangeType.parseFrom(fieldType, parser, coerce.value(), includeFrom);
+                    }
+                } else if (fieldName.equals(GTE_FIELD.getPreferredName())) {
+                    includeFrom = true;
+                    if (parser.currentToken() != XContentParser.Token.VALUE_NULL) {
+                        parsedFrom = rangeType.parseFrom(fieldType, parser, coerce.value(), includeFrom);
+                    }
+                } else if (fieldName.equals(LT_FIELD.getPreferredName())) {
+                    includeTo = false;
+                    if (parser.currentToken() != XContentParser.Token.VALUE_NULL) {
+                        parsedTo = rangeType.parseTo(fieldType, parser, coerce.value(), includeTo);
+                    }
+                } else if (fieldName.equals(LTE_FIELD.getPreferredName())) {
+                    includeTo = true;
+                    if (parser.currentToken() != XContentParser.Token.VALUE_NULL) {
+                        parsedTo = rangeType.parseTo(fieldType, parser, coerce.value(), includeTo);
+                    }
+                } else {
+                    throw new DocumentParsingException(
+                        parser.getTokenLocation(),
+                        "error parsing field [" + name() + "], with unknown parameter [" + fieldName + "]"
+                    );
+                }
+            }
         }
+        Object from = parsedFrom != null ? parsedFrom : rangeType.defaultFrom(parsedTo);
+        Object to = parsedTo != null ? parsedTo : rangeType.defaultTo(parsedFrom);
+
+        return new Range(rangeType, from, to, includeFrom, includeTo);
     }
 
     private static Range parseIpRangeFromCidr(final XContentParser parser) throws IOException {
-        final Tuple<InetAddress, Integer> cidr = InetAddresses.parseCidr(parser.text());
-        // create the lower value by zeroing out the host portion, upper value by filling it with all ones.
-        byte[] lower = cidr.v1().getAddress();
-        byte[] upper = lower.clone();
-        for (int i = cidr.v2(); i < 8 * lower.length; i++) {
-            int m = 1 << 7 - (i & 7);
-            lower[i >> 3] &= (byte) ~m;
-            upper[i >> 3] |= (byte) m;
-        }
-        try {
-            return new Range(RangeType.IP, InetAddress.getByAddress(lower), InetAddress.getByAddress(upper), true, true);
-        } catch (UnknownHostException bogus) {
-            throw new AssertionError(bogus);
+        final InetAddresses.IpRange range = InetAddresses.parseIpRangeFromCidr(parser.text());
+        return new Range(RangeType.IP, range.lowerBound(), range.upperBound(), true, true);
+    }
+
+    @Override
+    public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
+        if (hasDocValues == false) {
+            throw new IllegalArgumentException(
+                "field [" + name() + "] of type [" + typeName() + "] doesn't support synthetic source because it doesn't have doc values"
+            );
+        }
+        if (copyTo.copyToFields().isEmpty() != true) {
+            throw new IllegalArgumentException(
+                "field [" + name() + "] of type [" + typeName() + "] doesn't support synthetic source because it declares copy_to"
+            );
         }
+        return new BinaryDocValuesSyntheticFieldLoader(name()) {
+            @Override
+            protected void writeValue(XContentBuilder b, BytesRef value) throws IOException {
+                List<Range> ranges = type.decodeRanges(value);
+
+                switch (ranges.size()) {
+                    case 0:
+                        return;
+                    case 1:
+                        b.field(simpleName());
+                        ranges.get(0).toXContent(b, fieldType().dateTimeFormatter);
+                        break;
+                    default:
+                        b.startArray(simpleName());
+                        for (var range : ranges) {
+                            range.toXContent(b, fieldType().dateTimeFormatter);
+                        }
+                        b.endArray();
+                }
+            }
+        };
     }
 
     /** Class defining a range */
@@ -516,6 +550,30 @@ public Object getFrom() {
         public Object getTo() {
             return to;
         }
+
+        public XContentBuilder toXContent(XContentBuilder builder, DateFormatter dateFormatter) throws IOException {
+            builder.startObject();
+
+            if (includeFrom) {
+                builder.field("gte");
+            } else {
+                builder.field("gt");
+            }
+            Object f = includeFrom || from.equals(type.minValue()) ? from : type.nextDown(from);
+            builder.value(type.formatValue(f, dateFormatter));
+
+            if (includeTo) {
+                builder.field("lte");
+            } else {
+                builder.field("lt");
+            }
+            Object t = includeTo || to.equals(type.maxValue()) ? to : type.nextUp(to);
+            builder.value(type.formatValue(t, dateFormatter));
+
+            builder.endObject();
+
+            return builder;
+        }
     }
 
     static class BinaryRangesDocValuesField extends CustomDocValuesField {
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RangeType.java b/server/src/main/java/org/elasticsearch/index/mapper/RangeType.java
index f339269d93636..24a1eb869cf25 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/RangeType.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/RangeType.java
@@ -31,6 +31,7 @@
 import org.elasticsearch.xcontent.XContentParser;
 
 import java.io.IOException;
+import java.net.Inet4Address;
 import java.net.InetAddress;
 import java.time.Instant;
 import java.time.ZoneId;
@@ -64,6 +65,26 @@ public InetAddress parseTo(RangeFieldMapper.RangeFieldType fieldType, XContentPa
             return included ? address : nextDown(address);
         }
 
+        public Object defaultFrom(Object parsedTo) {
+            if (parsedTo == null) {
+                return minValue();
+            }
+
+            // Make sure that we keep the range inside the same address family.
+            // `minValue()` is always IPv6 so we need to adjust it.
+            return parsedTo instanceof Inet4Address ? InetAddressPoint.decode(new byte[4]) : minValue();
+        }
+
+        public Object defaultTo(Object parsedFrom) {
+            if (parsedFrom == null) {
+                return maxValue();
+            }
+
+            // Make sure that we keep the range inside the same address family.
+            // `maxValue()` is always IPv6 so we need to adjust it.
+            return parsedFrom instanceof Inet4Address ? InetAddressPoint.decode(new byte[] { -1, -1, -1, -1 }) : maxValue();
+        }
+
         @Override
         public InetAddress parseValue(Object value, boolean coerce, @Nullable DateMathParser dateMathParser) {
             if (value instanceof InetAddress) {
@@ -249,7 +270,7 @@ public BytesRef encodeRanges(Set<RangeFieldMapper.Range> ranges) throws IOExcept
 
         @Override
         public List<RangeFieldMapper.Range> decodeRanges(BytesRef bytes) throws IOException {
-            return LONG.decodeRanges(bytes);
+            return BinaryRangeUtil.decodeDateRanges(bytes);
         }
 
         @Override
@@ -844,6 +865,14 @@ public Object parseTo(RangeFieldMapper.RangeFieldType fieldType, XContentParser
         return included ? value : (Number) nextDown(value);
     }
 
+    public Object defaultFrom(Object parsedTo) {
+        return minValue();
+    }
+
+    public Object defaultTo(Object parsedFrom) {
+        return maxValue();
+    }
+
     public abstract Object minValue();
 
     public abstract Object maxValue();
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DateRangeFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DateRangeFieldMapperTests.java
index f570c7431e01a..0d971d64a8fe3 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/DateRangeFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/DateRangeFieldMapperTests.java
@@ -8,16 +8,20 @@
 
 package org.elasticsearch.index.mapper;
 
+import org.elasticsearch.common.time.DateFormatter;
+import org.elasticsearch.common.time.DateUtils;
+import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.junit.AssumptionViolatedException;
 
 import java.io.IOException;
+import java.time.Instant;
+import java.util.Map;
+import java.util.stream.Collectors;
 
 import static org.hamcrest.Matchers.containsString;
 
 public class DateRangeFieldMapperTests extends RangeFieldMapperTests {
-
-    private static final String DATE_FORMAT = "yyyy-MM-dd HH:mm:ss||yyyy-MM-dd||epoch_millis";
     private static final String FROM_DATE = "2016-10-31";
     private static final String TO_DATE = "2016-11-01 20:00:00";
 
@@ -56,8 +60,47 @@ public void testIllegalFormatField() {
     }
 
     @Override
-    protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed) {
-        throw new AssumptionViolatedException("not supported");
+    @SuppressWarnings("unchecked")
+    protected TestRange<Long> randomRangeForSyntheticSourceTest() {
+        var includeFrom = randomBoolean();
+        Long from = rarely() ? null : randomLongBetween(0, DateUtils.MAX_MILLIS_BEFORE_9999 - 1);
+        var includeTo = randomBoolean();
+        Long to = rarely() ? null : randomLongBetween((from == null ? 0 : from) + 1, DateUtils.MAX_MILLIS_BEFORE_9999);
+
+        return new TestRange<>(rangeType(), from, to, includeFrom, includeTo) {
+            private final DateFormatter inputDateFormatter = DateFormatter.forPattern("yyyy-MM-dd HH:mm:ss.SSS");
+            private final DateFormatter expectedDateFormatter = DateFormatter.forPattern(DATE_FORMAT);
+
+            @Override
+            Object toInput() {
+                var fromKey = includeFrom ? "gte" : "gt";
+                var toKey = includeTo ? "lte" : "lt";
+
+                var fromFormatted = from != null && randomBoolean() ? inputDateFormatter.format(Instant.ofEpochMilli(from)) : from;
+                var toFormatted = to != null && randomBoolean() ? inputDateFormatter.format(Instant.ofEpochMilli(to)) : to;
+
+                return (ToXContent) (builder, params) -> builder.startObject()
+                    .field(fromKey, fromFormatted)
+                    .field(toKey, toFormatted)
+                    .endObject();
+            }
+
+            @Override
+            Object toExpectedSyntheticSource() {
+                Map<String, Object> expectedInMillis = (Map<String, Object>) super.toExpectedSyntheticSource();
+
+                return expectedInMillis.entrySet()
+                    .stream()
+                    .collect(
+                        Collectors.toMap(Map.Entry::getKey, e -> expectedDateFormatter.format(Instant.ofEpochMilli((long) e.getValue())))
+                    );
+            }
+        };
+    }
+
+    @Override
+    protected RangeType rangeType() {
+        return RangeType.DATE;
     }
 
     @Override
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DoubleRangeFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DoubleRangeFieldMapperTests.java
index 6287777396029..07addee5bb532 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/DoubleRangeFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/DoubleRangeFieldMapperTests.java
@@ -8,13 +8,69 @@
 
 package org.elasticsearch.index.mapper;
 
+import org.elasticsearch.core.CheckedConsumer;
+import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.junit.AssumptionViolatedException;
 
 import java.io.IOException;
+import java.util.LinkedHashMap;
+import java.util.List;
+
+import static org.hamcrest.Matchers.equalTo;
 
 public class DoubleRangeFieldMapperTests extends RangeFieldMapperTests {
 
+    public void testSyntheticSourceDefaultValues() throws IOException {
+        // Default range ends for double are negative and positive infinity
+        // and they can not pass `roundTripSyntheticSource` test.
+
+        CheckedConsumer<XContentBuilder, IOException> mapping = b -> {
+            b.startObject("field");
+            minimalMapping(b);
+            b.endObject();
+        };
+
+        var inputValues = List.of(
+            (builder, params) -> builder.startObject().field("gte", (Double) null).field("lte", 10).endObject(),
+            (builder, params) -> builder.startObject().field("lte", 20).endObject(),
+            (builder, params) -> builder.startObject().field("gte", 10).field("lte", (Double) null).endObject(),
+            (builder, params) -> builder.startObject().field("gte", 20).endObject(),
+            (ToXContent) (builder, params) -> builder.startObject().endObject()
+        );
+
+        var expected = List.of(new LinkedHashMap<>() {
+            {
+                put("gte", "-Infinity");
+                put("lte", 10.0);
+            }
+        }, new LinkedHashMap<>() {
+            {
+                put("gte", "-Infinity");
+                put("lte", 20.0);
+            }
+        }, new LinkedHashMap<>() {
+            {
+                put("gte", "-Infinity");
+                put("lte", "Infinity");
+            }
+        }, new LinkedHashMap<>() {
+            {
+                put("gte", 10.0);
+                put("lte", "Infinity");
+            }
+        }, new LinkedHashMap<>() {
+            {
+                put("gte", 20.0);
+                put("lte", "Infinity");
+            }
+        });
+
+        var source = getSourceFor(mapping, inputValues);
+        var actual = source.source().get("field");
+        assertThat(actual, equalTo(expected));
+    }
+
     @Override
     protected XContentBuilder rangeSource(XContentBuilder in) throws IOException {
         return rangeSource(in, "0.5", "2.7");
@@ -41,8 +97,18 @@ protected boolean supportsDecimalCoerce() {
     }
 
     @Override
-    protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed) {
-        throw new AssumptionViolatedException("not supported");
+    protected TestRange<Double> randomRangeForSyntheticSourceTest() {
+        var includeFrom = randomBoolean();
+        Double from = randomDoubleBetween(-Double.MAX_VALUE, Double.MAX_VALUE, true);
+        var includeTo = randomBoolean();
+        Double to = randomDoubleBetween(from, Double.MAX_VALUE, false);
+
+        return new TestRange<>(rangeType(), from, to, includeFrom, includeTo);
+    }
+
+    @Override
+    protected RangeType rangeType() {
+        return RangeType.DOUBLE;
     }
 
     @Override
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/FloatRangeFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/FloatRangeFieldMapperTests.java
index ab2af0c09aacc..d30e08ec8d90a 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/FloatRangeFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/FloatRangeFieldMapperTests.java
@@ -8,13 +8,69 @@
 
 package org.elasticsearch.index.mapper;
 
+import org.elasticsearch.core.CheckedConsumer;
+import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.junit.AssumptionViolatedException;
 
 import java.io.IOException;
+import java.util.LinkedHashMap;
+import java.util.List;
+
+import static org.hamcrest.Matchers.equalTo;
 
 public class FloatRangeFieldMapperTests extends RangeFieldMapperTests {
 
+    public void testSyntheticSourceDefaultValues() throws IOException {
+        // Default range ends for float are negative and positive infinity
+        // and they can not pass `roundTripSyntheticSource` test.
+
+        CheckedConsumer<XContentBuilder, IOException> mapping = b -> {
+            b.startObject("field");
+            minimalMapping(b);
+            b.endObject();
+        };
+
+        var inputValues = List.of(
+            (builder, params) -> builder.startObject().field("gte", (Float) null).field("lte", 10).endObject(),
+            (builder, params) -> builder.startObject().field("lte", 20).endObject(),
+            (builder, params) -> builder.startObject().field("gte", 10).field("lte", (Float) null).endObject(),
+            (builder, params) -> builder.startObject().field("gte", 20).endObject(),
+            (ToXContent) (builder, params) -> builder.startObject().endObject()
+        );
+
+        var expected = List.of(new LinkedHashMap<>() {
+            {
+                put("gte", "-Infinity");
+                put("lte", 10.0);
+            }
+        }, new LinkedHashMap<>() {
+            {
+                put("gte", "-Infinity");
+                put("lte", 20.0);
+            }
+        }, new LinkedHashMap<>() {
+            {
+                put("gte", "-Infinity");
+                put("lte", "Infinity");
+            }
+        }, new LinkedHashMap<>() {
+            {
+                put("gte", 10.0);
+                put("lte", "Infinity");
+            }
+        }, new LinkedHashMap<>() {
+            {
+                put("gte", 20.0);
+                put("lte", "Infinity");
+            }
+        });
+
+        var source = getSourceFor(mapping, inputValues);
+        var actual = source.source().get("field");
+        assertThat(actual, equalTo(expected));
+    }
+
     @Override
     protected XContentBuilder rangeSource(XContentBuilder in) throws IOException {
         return rangeSource(in, "0.5", "2.7");
@@ -41,8 +97,18 @@ protected boolean supportsDecimalCoerce() {
     }
 
     @Override
-    protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed) {
-        throw new AssumptionViolatedException("not supported");
+    protected TestRange<Float> randomRangeForSyntheticSourceTest() {
+        var includeFrom = randomBoolean();
+        Float from = (float) randomDoubleBetween(-Float.MAX_VALUE, Float.MAX_VALUE, true);
+        var includeTo = randomBoolean();
+        Float to = (float) randomDoubleBetween(from, Float.MAX_VALUE, false);
+
+        return new TestRange<>(rangeType(), from, to, includeFrom, includeTo);
+    }
+
+    @Override
+    protected RangeType rangeType() {
+        return RangeType.FLOAT;
     }
 
     @Override
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IntegerRangeFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IntegerRangeFieldMapperTests.java
index 82d792afc36fb..311f8af8f0189 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/IntegerRangeFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/IntegerRangeFieldMapperTests.java
@@ -35,8 +35,25 @@ protected void minimalMapping(XContentBuilder b) throws IOException {
     }
 
     @Override
-    protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed) {
-        throw new AssumptionViolatedException("not supported");
+    protected TestRange<Integer> randomRangeForSyntheticSourceTest() {
+        var includeFrom = randomBoolean();
+        Integer from = randomIntBetween(Integer.MIN_VALUE, Integer.MAX_VALUE - 1);
+        var includeTo = randomBoolean();
+        Integer to = randomIntBetween((from) + 1, Integer.MAX_VALUE);
+
+        if (rarely()) {
+            from = null;
+        }
+        if (rarely()) {
+            to = null;
+        }
+
+        return new TestRange<>(rangeType(), from, to, includeFrom, includeTo);
+    }
+
+    @Override
+    protected RangeType rangeType() {
+        return RangeType.INTEGER;
     }
 
     @Override
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IpRangeFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IpRangeFieldMapperTests.java
index 88a36ce307017..279c9263c98a9 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/IpRangeFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/IpRangeFieldMapperTests.java
@@ -10,15 +10,23 @@
 import org.apache.lucene.index.DocValuesType;
 import org.apache.lucene.index.IndexableField;
 import org.elasticsearch.common.network.InetAddresses;
+import org.elasticsearch.core.CheckedConsumer;
+import org.elasticsearch.core.Tuple;
+import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.junit.AssumptionViolatedException;
 
 import java.io.IOException;
+import java.net.InetAddress;
 import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 
 import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.instanceOf;
 
 public class IpRangeFieldMapperTests extends RangeFieldMapperTests {
 
@@ -79,9 +87,150 @@ public void testStoreCidr() throws Exception {
         }
     }
 
+    @Override
+    public void testNullBounds() throws IOException {
+        DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> {
+            minimalMapping(b);
+            b.field("store", true);
+        }));
+
+        ParsedDocument bothNull = mapper.parse(source(b -> b.startObject("field").nullField("gte").nullField("lte").endObject()));
+        assertThat(storedValue(bothNull), equalTo("[:: : ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff]"));
+
+        ParsedDocument onlyFromIPv4 = mapper.parse(
+            source(b -> b.startObject("field").field("gte", rangeValue()).nullField("lte").endObject())
+        );
+        assertThat(storedValue(onlyFromIPv4), equalTo("[192.168.1.7 : 255.255.255.255]"));
+
+        ParsedDocument onlyToIPv4 = mapper.parse(
+            source(b -> b.startObject("field").nullField("gte").field("lte", rangeValue()).endObject())
+        );
+        assertThat(storedValue(onlyToIPv4), equalTo("[0.0.0.0 : 192.168.1.7]"));
+
+        ParsedDocument onlyFromIPv6 = mapper.parse(
+            source(b -> b.startObject("field").field("gte", "2001:db8::").nullField("lte").endObject())
+        );
+        assertThat(storedValue(onlyFromIPv6), equalTo("[2001:db8:: : ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff]"));
+
+        ParsedDocument onlyToIPv6 = mapper.parse(
+            source(b -> b.startObject("field").nullField("gte").field("lte", "2001:db8::").endObject())
+        );
+        assertThat(storedValue(onlyToIPv6), equalTo("[:: : 2001:db8::]"));
+    }
+
+    @SuppressWarnings("unchecked")
+    public void testValidSyntheticSource() throws IOException {
+        CheckedConsumer<XContentBuilder, IOException> mapping = b -> {
+            b.startObject("field");
+            b.field("type", "ip_range");
+            if (rarely()) {
+                b.field("index", false);
+            }
+            if (rarely()) {
+                b.field("store", false);
+            }
+            b.endObject();
+        };
+
+        var values = randomList(1, 5, this::generateValue);
+        var inputValues = values.stream().map(Tuple::v1).toList();
+        var expectedValues = values.stream().map(Tuple::v2).toList();
+
+        var source = getSourceFor(mapping, inputValues);
+
+        // This is the main reason why we need custom logic.
+        // IP ranges are serialized into binary doc values in unpredictable order
+        // because API uses a set.
+        // So this assert needs to be not sensitive to order and in "reference"
+        // implementation of tests from MapperTestCase it is.
+        var actual = source.source().get("field");
+        if (inputValues.size() == 1) {
+            assertEquals(expectedValues.get(0), actual);
+        } else {
+            assertThat(actual, instanceOf(List.class));
+            assertTrue(((List<Object>) actual).containsAll(new HashSet<>(expectedValues)));
+        }
+    }
+
+    private Tuple<Object, Object> generateValue() {
+        String cidr = randomCidrBlock();
+        InetAddresses.IpRange range = InetAddresses.parseIpRangeFromCidr(cidr);
+
+        var includeFrom = randomBoolean();
+        var includeTo = randomBoolean();
+
+        Object input;
+        // "to" field always comes first.
+        Map<String, Object> output = new LinkedHashMap<>();
+        if (randomBoolean()) {
+            // CIDRs are always inclusive ranges.
+            input = cidr;
+            output.put("gte", InetAddresses.toAddrString(range.lowerBound()));
+            output.put("lte", InetAddresses.toAddrString(range.upperBound()));
+        } else {
+            var fromKey = includeFrom ? "gte" : "gt";
+            var toKey = includeTo ? "lte" : "lt";
+            var from = rarely() ? null : InetAddresses.toAddrString(range.lowerBound());
+            var to = rarely() ? null : InetAddresses.toAddrString(range.upperBound());
+            input = (ToXContent) (builder, params) -> builder.startObject().field(fromKey, from).field(toKey, to).endObject();
+
+            // When ranges are stored, they are always normalized to include both ends.
+            // `includeFrom` and `includeTo` here refers to user input.
+            //
+            // Range values are not properly normalized for default values
+            // which results in off by one error here.
+            // So "gte": null and "gt": null both result in "gte": MIN_VALUE.
+            // This is a bug, see #107282.
+            if (from == null) {
+                output.put("gte", InetAddresses.toAddrString((InetAddress) rangeType().minValue()));
+            } else {
+                var rawFrom = range.lowerBound();
+                var adjustedFrom = includeFrom ? rawFrom : (InetAddress) RangeType.IP.nextUp(rawFrom);
+                output.put("gte", InetAddresses.toAddrString(adjustedFrom));
+            }
+            if (to == null) {
+                output.put("lte", InetAddresses.toAddrString((InetAddress) rangeType().maxValue()));
+            } else {
+                var rawTo = range.upperBound();
+                var adjustedTo = includeTo ? rawTo : (InetAddress) RangeType.IP.nextDown(rawTo);
+                output.put("lte", InetAddresses.toAddrString(adjustedTo));
+            }
+        }
+
+        return Tuple.tuple(input, output);
+    }
+
+    public void testInvalidSyntheticSource() {
+        Exception e = expectThrows(IllegalArgumentException.class, () -> createDocumentMapper(syntheticSourceMapping(b -> {
+            b.startObject("field");
+            b.field("type", "ip_range");
+            b.field("doc_values", false);
+            b.endObject();
+        })));
+        assertThat(
+            e.getMessage(),
+            equalTo("field [field] of type [ip_range] doesn't support synthetic source because it doesn't have doc values")
+        );
+    }
+
     @Override
     protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed) {
-        throw new AssumptionViolatedException("not supported");
+        throw new AssumptionViolatedException("custom version of synthetic source tests is implemented");
+    }
+
+    private static String randomCidrBlock() {
+        boolean ipv4 = randomBoolean();
+
+        InetAddress address = randomIp(ipv4);
+        // exclude smallest prefix lengths to avoid empty ranges
+        int prefixLength = ipv4 ? randomIntBetween(0, 30) : randomIntBetween(0, 126);
+
+        return InetAddresses.toCidrString(address, prefixLength);
+    }
+
+    @Override
+    protected RangeType rangeType() {
+        return RangeType.IP;
     }
 
     @Override
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/LongRangeFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/LongRangeFieldMapperTests.java
index 2b7487f521fb3..7256e31e4a717 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/LongRangeFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/LongRangeFieldMapperTests.java
@@ -36,8 +36,25 @@ protected void minimalMapping(XContentBuilder b) throws IOException {
     }
 
     @Override
-    protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed) {
-        throw new AssumptionViolatedException("not supported");
+    protected TestRange<Long> randomRangeForSyntheticSourceTest() {
+        var includeFrom = randomBoolean();
+        Long from = randomLongBetween(Long.MIN_VALUE, Long.MAX_VALUE - 1);
+        var includeTo = randomBoolean();
+        Long to = randomLongBetween(from + 1, Long.MAX_VALUE);
+
+        if (rarely()) {
+            from = null;
+        }
+        if (rarely()) {
+            to = null;
+        }
+
+        return new TestRange<>(rangeType(), from, to, includeFrom, includeTo);
+    }
+
+    @Override
+    protected RangeType rangeType() {
+        return RangeType.LONG;
     }
 
     @Override
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java
index 6de36c37e8c22..d87d8dbc2bb4e 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java
@@ -8,14 +8,25 @@
 
 package org.elasticsearch.index.mapper;
 
+import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.DocValuesType;
 import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.tests.index.RandomIndexWriter;
 import org.elasticsearch.core.CheckedConsumer;
+import org.elasticsearch.search.lookup.Source;
+import org.elasticsearch.search.lookup.SourceProvider;
+import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContentBuilder;
+import org.junit.AssumptionViolatedException;
 
 import java.io.IOException;
 import java.util.Collections;
+import java.util.Comparator;
+import java.util.LinkedHashMap;
 import java.util.List;
+import java.util.Locale;
+import java.util.Map;
 
 import static org.elasticsearch.index.query.RangeQueryBuilder.GTE_FIELD;
 import static org.elasticsearch.index.query.RangeQueryBuilder.GT_FIELD;
@@ -23,10 +34,13 @@
 import static org.elasticsearch.index.query.RangeQueryBuilder.LT_FIELD;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.endsWith;
+import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.startsWith;
 
 public abstract class RangeFieldMapperTests extends MapperTestCase {
 
+    protected static final String DATE_FORMAT = "uuuu-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd||epoch_millis";
+
     @Override
     protected boolean supportsSearchLookup() {
         return false;
@@ -222,14 +236,14 @@ private void assertNullBounds(CheckedConsumer<XContentBuilder, IOException> toCh
         }
     }
 
-    private static String storedValue(ParsedDocument doc) {
+    protected static String storedValue(ParsedDocument doc) {
         assertEquals(3, doc.rootDoc().getFields("field").size());
         List<IndexableField> fields = doc.rootDoc().getFields("field");
         IndexableField storedField = fields.get(2);
         return storedField.stringValue();
     }
 
-    public final void testNullBounds() throws IOException {
+    public void testNullBounds() throws IOException {
 
         // null, null => min, max
         assertNullBounds(b -> b.startObject("field").nullField("gte").nullField("lte").endObject(), true, true);
@@ -242,6 +256,156 @@ public final void testNullBounds() throws IOException {
         assertNullBounds(b -> b.startObject("field").field("gte", val).nullField("lte").endObject(), false, true);
     }
 
+    @Override
+    protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed) {
+        assumeTrue("test setup only supports numeric ranges", rangeType().isNumeric());
+
+        return new SyntheticSourceSupport() {
+            @Override
+            public SyntheticSourceExample example(int maxValues) throws IOException {
+                if (randomBoolean()) {
+                    var range = randomRangeForSyntheticSourceTest();
+                    return new SyntheticSourceExample(range.toInput(), range.toExpectedSyntheticSource(), this::mapping);
+                }
+
+                var values = randomList(1, maxValues, () -> randomRangeForSyntheticSourceTest());
+                List<Object> in = values.stream().map(TestRange::toInput).toList();
+                List<Object> outList = values.stream().sorted(Comparator.naturalOrder()).map(TestRange::toExpectedSyntheticSource).toList();
+                Object out = outList.size() == 1 ? outList.get(0) : outList;
+
+                return new SyntheticSourceExample(in, out, this::mapping);
+            }
+
+            private void mapping(XContentBuilder b) throws IOException {
+                b.field("type", rangeType().name);
+                if (rarely()) {
+                    b.field("index", false);
+                }
+                if (rarely()) {
+                    b.field("store", false);
+                }
+                if (rangeType() == RangeType.DATE) {
+                    b.field("format", DATE_FORMAT);
+                }
+            }
+
+            @Override
+            public List<SyntheticSourceInvalidExample> invalidExample() throws IOException {
+                return List.of(
+                    new SyntheticSourceInvalidExample(
+                        equalTo(
+                            String.format(
+                                Locale.ROOT,
+                                "field [field] of type [%s] doesn't support synthetic source because it doesn't have doc values",
+                                rangeType().name
+                            )
+                        ),
+                        b -> b.field("type", rangeType().name).field("doc_values", false)
+                    )
+                );
+            }
+        };
+    }
+
+    /**
+     * Stores range information as if it was provided by user.
+     * Provides an expected value of provided range in synthetic source.
+     * @param <T>
+     */
+    protected class TestRange<T extends Comparable<T>> implements Comparable<TestRange<T>> {
+        private final RangeType type;
+        private final T from;
+        private final T to;
+        private final boolean includeFrom;
+        private final boolean includeTo;
+
+        public TestRange(RangeType type, T from, T to, boolean includeFrom, boolean includeTo) {
+            this.type = type;
+            this.from = from;
+            this.to = to;
+            this.includeFrom = includeFrom;
+            this.includeTo = includeTo;
+        }
+
+        Object toInput() {
+            var fromKey = includeFrom ? "gte" : "gt";
+            var toKey = includeTo ? "lte" : "lt";
+
+            return (ToXContent) (builder, params) -> builder.startObject().field(fromKey, from).field(toKey, to).endObject();
+        }
+
+        Object toExpectedSyntheticSource() {
+            // When ranges are stored, they are always normalized to include both ends.
+            // Also, "to" field always comes first.
+            Map<String, Object> output = new LinkedHashMap<>();
+
+            // Range values are not properly normalized for default values
+            // which results in off by one error here.
+            // So "gte": null and "gt": null both result in "gte": MIN_VALUE.
+            // This is a bug, see #107282.
+            if (from == null) {
+                output.put("gte", rangeType().minValue());
+            } else if (includeFrom) {
+                output.put("gte", from);
+            } else {
+                output.put("gte", type.nextUp(from));
+            }
+
+            if (to == null) {
+                output.put("lte", rangeType().maxValue());
+            } else if (includeTo) {
+                output.put("lte", to);
+            } else {
+                output.put("lte", type.nextDown(to));
+            }
+
+            return output;
+        }
+
+        @Override
+        public int compareTo(TestRange<T> o) {
+            return Comparator.comparing((TestRange<T> r) -> r.from, Comparator.nullsFirst(Comparator.naturalOrder()))
+                .thenComparing((TestRange<T> r) -> r.to)
+                .compare(this, o);
+        }
+    }
+
+    protected TestRange<?> randomRangeForSyntheticSourceTest() {
+        throw new AssumptionViolatedException("Should only be called for specific range types");
+    }
+
+    protected Source getSourceFor(CheckedConsumer<XContentBuilder, IOException> mapping, List<?> inputValues) throws IOException {
+        DocumentMapper mapper = createDocumentMapper(syntheticSourceMapping(mapping));
+
+        CheckedConsumer<XContentBuilder, IOException> input = b -> {
+            b.field("field");
+            if (inputValues.size() == 1) {
+                b.value(inputValues.get(0));
+            } else {
+                b.startArray();
+                for (var range : inputValues) {
+                    b.value(range);
+                }
+                b.endArray();
+            }
+        };
+
+        try (Directory directory = newDirectory()) {
+            RandomIndexWriter iw = new RandomIndexWriter(random(), directory);
+            LuceneDocument doc = mapper.parse(source(input)).rootDoc();
+            iw.addDocument(doc);
+            iw.close();
+            try (DirectoryReader reader = DirectoryReader.open(directory)) {
+                SourceProvider provider = SourceProvider.fromSyntheticSource(mapper.mapping());
+                Source syntheticSource = provider.getSource(getOnlyLeafReader(reader).getContext(), 0);
+
+                return syntheticSource;
+            }
+        }
+    }
+
+    protected abstract RangeType rangeType();
+
     @Override
     protected Object generateRandomInputValue(MappedFieldType ft) {
         // Doc value fetching crashes.

From dd741ba50cabea5598247a2de3c83046cfe8cfcc Mon Sep 17 00:00:00 2001
From: David Kyle <david.kyle@elastic.co>
Date: Wed, 24 Apr 2024 20:16:00 +0100
Subject: [PATCH 08/58] [ML] External inference service rolling upgrade tests
 (#107619)

Rolling upgrade tests for OpenAI, Cohere and Hugging Face.
The services are tested by setting the services' url to a local mock
 web server and mocking the responses.
---
 .../inference/qa/rolling-upgrade/build.gradle |  33 ++
 .../AzureOpenAiServiceUpgradeIT.java          | 112 ++++++
 .../application/CohereServiceUpgradeIT.java   | 354 ++++++++++++++++++
 .../HuggingFaceServiceUpgradeIT.java          | 194 ++++++++++
 .../application/InferenceUpgradeTestCase.java | 107 ++++++
 .../application/OpenAiServiceUpgradeIT.java   | 265 +++++++++++++
 .../inference/services/ServiceUtils.java      |   3 +-
 .../CohereEmbeddingsServiceSettings.java      |  59 ++-
 .../OpenAiEmbeddingsServiceSettings.java      |  14 +-
 .../CohereEmbeddingsServiceSettingsTests.java |  18 +
 .../OpenAiEmbeddingsServiceSettingsTests.java |  16 +-
 11 files changed, 1136 insertions(+), 39 deletions(-)
 create mode 100644 x-pack/plugin/inference/qa/rolling-upgrade/build.gradle
 create mode 100644 x-pack/plugin/inference/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/application/AzureOpenAiServiceUpgradeIT.java
 create mode 100644 x-pack/plugin/inference/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/application/CohereServiceUpgradeIT.java
 create mode 100644 x-pack/plugin/inference/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/application/HuggingFaceServiceUpgradeIT.java
 create mode 100644 x-pack/plugin/inference/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/application/InferenceUpgradeTestCase.java
 create mode 100644 x-pack/plugin/inference/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/application/OpenAiServiceUpgradeIT.java

diff --git a/x-pack/plugin/inference/qa/rolling-upgrade/build.gradle b/x-pack/plugin/inference/qa/rolling-upgrade/build.gradle
new file mode 100644
index 0000000000000..328444dacaf53
--- /dev/null
+++ b/x-pack/plugin/inference/qa/rolling-upgrade/build.gradle
@@ -0,0 +1,33 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import org.elasticsearch.gradle.Version
+import org.elasticsearch.gradle.VersionProperties
+import org.elasticsearch.gradle.internal.info.BuildParams
+import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask
+
+apply plugin: 'elasticsearch.internal-java-rest-test'
+apply plugin: 'elasticsearch.internal-test-artifact-base'
+apply plugin: 'elasticsearch.bwc-test'
+
+
+dependencies {
+  compileOnly project(':x-pack:plugin:core')
+  javaRestTestImplementation(testArtifact(project(xpackModule('core'))))
+  javaRestTestImplementation project(path: xpackModule('inference'))
+  javaRestTestImplementation(testArtifact(project(":qa:rolling-upgrade"), "javaRestTest"))
+}
+
+// Inference API added in 8.11
+BuildParams.bwcVersions.withWireCompatible(v -> v.after("8.11.0")) { bwcVersion, baseName ->
+  tasks.register(bwcTaskName(bwcVersion), StandaloneRestIntegTestTask) {
+    usesBwcDistribution(bwcVersion)
+    systemProperty("tests.old_cluster_version", bwcVersion)
+  }
+}
+
diff --git a/x-pack/plugin/inference/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/application/AzureOpenAiServiceUpgradeIT.java b/x-pack/plugin/inference/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/application/AzureOpenAiServiceUpgradeIT.java
new file mode 100644
index 0000000000000..db5e62a367ab3
--- /dev/null
+++ b/x-pack/plugin/inference/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/application/AzureOpenAiServiceUpgradeIT.java
@@ -0,0 +1,112 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.application;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.inference.TaskType;
+import org.elasticsearch.test.http.MockResponse;
+import org.elasticsearch.test.http.MockWebServer;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.empty;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.not;
+
+public class AzureOpenAiServiceUpgradeIT extends InferenceUpgradeTestCase {
+
+    private static final String OPEN_AI_AZURE_EMBEDDINGS_ADDED = "8.14.0";
+
+    private static MockWebServer openAiEmbeddingsServer;
+
+    public AzureOpenAiServiceUpgradeIT(@Name("upgradedNodes") int upgradedNodes) {
+        super(upgradedNodes);
+    }
+
+    @BeforeClass
+    public static void startWebServer() throws IOException {
+        openAiEmbeddingsServer = new MockWebServer();
+        openAiEmbeddingsServer.start();
+    }
+
+    @AfterClass
+    public static void shutdown() {
+        openAiEmbeddingsServer.close();
+    }
+
+    @SuppressWarnings("unchecked")
+    @AwaitsFix(bugUrl = "Cannot set the URL in the tests")
+    public void testOpenAiEmbeddings() throws IOException {
+        var openAiEmbeddingsSupported = getOldClusterTestVersion().onOrAfter(OPEN_AI_AZURE_EMBEDDINGS_ADDED);
+        assumeTrue("Azure OpenAI embedding service added in " + OPEN_AI_AZURE_EMBEDDINGS_ADDED, openAiEmbeddingsSupported);
+
+        final String oldClusterId = "old-cluster-embeddings";
+        final String upgradedClusterId = "upgraded-cluster-embeddings";
+
+        if (isOldCluster()) {
+            // queue a response as PUT will call the service
+            openAiEmbeddingsServer.enqueue(new MockResponse().setResponseCode(200).setBody(OpenAiServiceUpgradeIT.embeddingResponse()));
+            put(oldClusterId, embeddingConfig(getUrl(openAiEmbeddingsServer)), TaskType.TEXT_EMBEDDING);
+
+            var configs = (List<Map<String, Object>>) get(TaskType.TEXT_EMBEDDING, oldClusterId).get("models");
+            assertThat(configs, hasSize(1));
+        } else if (isMixedCluster()) {
+            var configs = (List<Map<String, Object>>) get(TaskType.TEXT_EMBEDDING, oldClusterId).get("models");
+            assertEquals("azureopenai", configs.get(0).get("service"));
+
+            assertEmbeddingInference(oldClusterId);
+        } else if (isUpgradedCluster()) {
+            // check old cluster model
+            var configs = (List<Map<String, Object>>) get(TaskType.TEXT_EMBEDDING, oldClusterId).get("models");
+            var serviceSettings = (Map<String, Object>) configs.get(0).get("service_settings");
+
+            // Inference on old cluster model
+            assertEmbeddingInference(oldClusterId);
+
+            openAiEmbeddingsServer.enqueue(new MockResponse().setResponseCode(200).setBody(OpenAiServiceUpgradeIT.embeddingResponse()));
+            put(upgradedClusterId, embeddingConfig(getUrl(openAiEmbeddingsServer)), TaskType.TEXT_EMBEDDING);
+
+            configs = (List<Map<String, Object>>) get(TaskType.TEXT_EMBEDDING, upgradedClusterId).get("models");
+            assertThat(configs, hasSize(1));
+
+            // Inference on the new config
+            assertEmbeddingInference(upgradedClusterId);
+
+            delete(oldClusterId);
+            delete(upgradedClusterId);
+        }
+    }
+
+    void assertEmbeddingInference(String inferenceId) throws IOException {
+        openAiEmbeddingsServer.enqueue(new MockResponse().setResponseCode(200).setBody(OpenAiServiceUpgradeIT.embeddingResponse()));
+        var inferenceMap = inference(inferenceId, TaskType.TEXT_EMBEDDING, "some text");
+        assertThat(inferenceMap.entrySet(), not(empty()));
+    }
+
+    private String embeddingConfig(String url) {
+        return Strings.format("""
+            {
+                "service": "azureopenai",
+                "service_settings": {
+                    "api_key": "XXXX",
+                    "url": "%s",
+                    "resource_name": "resource_name",
+                    "deployment_id": "deployment_id",
+                    "api_version": "2024-02-01"
+                }
+            }
+            """, url);
+    }
+
+}
diff --git a/x-pack/plugin/inference/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/application/CohereServiceUpgradeIT.java b/x-pack/plugin/inference/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/application/CohereServiceUpgradeIT.java
new file mode 100644
index 0000000000000..9f009ef32f3aa
--- /dev/null
+++ b/x-pack/plugin/inference/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/application/CohereServiceUpgradeIT.java
@@ -0,0 +1,354 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.application;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.inference.TaskType;
+import org.elasticsearch.test.http.MockResponse;
+import org.elasticsearch.test.http.MockWebServer;
+import org.elasticsearch.xpack.inference.services.cohere.embeddings.CohereEmbeddingType;
+import org.hamcrest.Matchers;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.empty;
+import static org.hamcrest.Matchers.hasEntry;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.not;
+import static org.hamcrest.Matchers.oneOf;
+
+public class CohereServiceUpgradeIT extends InferenceUpgradeTestCase {
+
+    private static final String COHERE_EMBEDDINGS_ADDED = "8.13.0";
+    private static final String COHERE_RERANK_ADDED = "8.14.0";
+    private static final String BYTE_ALIAS_FOR_INT8_ADDED = "8.14.0";
+
+    private static MockWebServer cohereEmbeddingsServer;
+    private static MockWebServer cohereRerankServer;
+
+    public CohereServiceUpgradeIT(@Name("upgradedNodes") int upgradedNodes) {
+        super(upgradedNodes);
+    }
+
+    @BeforeClass
+    public static void startWebServer() throws IOException {
+        cohereEmbeddingsServer = new MockWebServer();
+        cohereEmbeddingsServer.start();
+
+        cohereRerankServer = new MockWebServer();
+        cohereRerankServer.start();
+    }
+
+    @AfterClass
+    public static void shutdown() {
+        cohereEmbeddingsServer.close();
+        cohereRerankServer.close();
+    }
+
+    @SuppressWarnings("unchecked")
+    public void testCohereEmbeddings() throws IOException {
+        var embeddingsSupported = getOldClusterTestVersion().onOrAfter(COHERE_EMBEDDINGS_ADDED);
+        assumeTrue("Cohere embedding service added in " + COHERE_EMBEDDINGS_ADDED, embeddingsSupported);
+
+        final String oldClusterIdInt8 = "old-cluster-embeddings-int8";
+        final String oldClusterIdFloat = "old-cluster-embeddings-float";
+
+        if (isOldCluster()) {
+            // queue a response as PUT will call the service
+            cohereEmbeddingsServer.enqueue(new MockResponse().setResponseCode(200).setBody(embeddingResponseByte()));
+            put(oldClusterIdInt8, embeddingConfigInt8(getUrl(cohereEmbeddingsServer)), TaskType.TEXT_EMBEDDING);
+            // float model
+            cohereEmbeddingsServer.enqueue(new MockResponse().setResponseCode(200).setBody(embeddingResponseFloat()));
+            put(oldClusterIdFloat, embeddingConfigFloat(getUrl(cohereEmbeddingsServer)), TaskType.TEXT_EMBEDDING);
+
+            var configs = (List<Map<String, Object>>) get(TaskType.TEXT_EMBEDDING, oldClusterIdInt8).get("models");
+            assertThat(configs, hasSize(1));
+            assertEquals("cohere", configs.get(0).get("service"));
+            var serviceSettings = (Map<String, Object>) configs.get(0).get("service_settings");
+            assertThat(serviceSettings, hasEntry("model_id", "embed-english-light-v3.0"));
+            var embeddingType = serviceSettings.get("embedding_type");
+            // An upgraded node will report the embedding type as byte, the old node int8
+            assertThat(embeddingType, Matchers.is(oneOf("int8", "byte")));
+
+            assertEmbeddingInference(oldClusterIdInt8, CohereEmbeddingType.BYTE);
+            assertEmbeddingInference(oldClusterIdFloat, CohereEmbeddingType.FLOAT);
+        } else if (isMixedCluster()) {
+            var configs = (List<Map<String, Object>>) get(TaskType.TEXT_EMBEDDING, oldClusterIdInt8).get("models");
+            assertEquals("cohere", configs.get(0).get("service"));
+            var serviceSettings = (Map<String, Object>) configs.get(0).get("service_settings");
+            assertThat(serviceSettings, hasEntry("model_id", "embed-english-light-v3.0"));
+            var embeddingType = serviceSettings.get("embedding_type");
+            // An upgraded node will report the embedding type as byte, an old node int8
+            assertThat(embeddingType, Matchers.is(oneOf("int8", "byte")));
+
+            configs = (List<Map<String, Object>>) get(TaskType.TEXT_EMBEDDING, oldClusterIdFloat).get("models");
+            serviceSettings = (Map<String, Object>) configs.get(0).get("service_settings");
+            assertThat(serviceSettings, hasEntry("embedding_type", "float"));
+
+            assertEmbeddingInference(oldClusterIdInt8, CohereEmbeddingType.BYTE);
+            assertEmbeddingInference(oldClusterIdFloat, CohereEmbeddingType.FLOAT);
+        } else if (isUpgradedCluster()) {
+            // check old cluster model
+            var configs = (List<Map<String, Object>>) get(TaskType.TEXT_EMBEDDING, oldClusterIdInt8).get("models");
+            var serviceSettings = (Map<String, Object>) configs.get(0).get("service_settings");
+            assertThat(serviceSettings, hasEntry("model_id", "embed-english-light-v3.0"));
+            assertThat(serviceSettings, hasEntry("embedding_type", "byte"));
+            var taskSettings = (Map<String, Object>) configs.get(0).get("task_settings");
+            assertThat(taskSettings.keySet(), empty());
+
+            // Inference on old cluster models
+            assertEmbeddingInference(oldClusterIdInt8, CohereEmbeddingType.BYTE);
+            assertEmbeddingInference(oldClusterIdFloat, CohereEmbeddingType.FLOAT);
+
+            {
+                final String upgradedClusterIdByte = "upgraded-cluster-embeddings-byte";
+
+                cohereEmbeddingsServer.enqueue(new MockResponse().setResponseCode(200).setBody(embeddingResponseByte()));
+                put(upgradedClusterIdByte, embeddingConfigByte(getUrl(cohereEmbeddingsServer)), TaskType.TEXT_EMBEDDING);
+
+                configs = (List<Map<String, Object>>) get(TaskType.TEXT_EMBEDDING, upgradedClusterIdByte).get("models");
+                serviceSettings = (Map<String, Object>) configs.get(0).get("service_settings");
+                assertThat(serviceSettings, hasEntry("embedding_type", "byte"));
+
+                assertEmbeddingInference(upgradedClusterIdByte, CohereEmbeddingType.BYTE);
+                delete(upgradedClusterIdByte);
+            }
+            {
+                final String upgradedClusterIdInt8 = "upgraded-cluster-embeddings-int8";
+
+                cohereEmbeddingsServer.enqueue(new MockResponse().setResponseCode(200).setBody(embeddingResponseByte()));
+                put(upgradedClusterIdInt8, embeddingConfigInt8(getUrl(cohereEmbeddingsServer)), TaskType.TEXT_EMBEDDING);
+
+                configs = (List<Map<String, Object>>) get(TaskType.TEXT_EMBEDDING, upgradedClusterIdInt8).get("models");
+                serviceSettings = (Map<String, Object>) configs.get(0).get("service_settings");
+                assertThat(serviceSettings, hasEntry("embedding_type", "byte")); // int8 rewritten to byte
+
+                assertEmbeddingInference(upgradedClusterIdInt8, CohereEmbeddingType.INT8);
+                delete(upgradedClusterIdInt8);
+            }
+            {
+                final String upgradedClusterIdFloat = "upgraded-cluster-embeddings-float";
+                cohereEmbeddingsServer.enqueue(new MockResponse().setResponseCode(200).setBody(embeddingResponseFloat()));
+                put(upgradedClusterIdFloat, embeddingConfigFloat(getUrl(cohereEmbeddingsServer)), TaskType.TEXT_EMBEDDING);
+
+                configs = (List<Map<String, Object>>) get(TaskType.TEXT_EMBEDDING, upgradedClusterIdFloat).get("models");
+                serviceSettings = (Map<String, Object>) configs.get(0).get("service_settings");
+                assertThat(serviceSettings, hasEntry("embedding_type", "float"));
+
+                assertEmbeddingInference(upgradedClusterIdFloat, CohereEmbeddingType.FLOAT);
+                delete(upgradedClusterIdFloat);
+            }
+
+            delete(oldClusterIdFloat);
+            delete(oldClusterIdInt8);
+        }
+    }
+
+    void assertEmbeddingInference(String inferenceId, CohereEmbeddingType type) throws IOException {
+        switch (type) {
+            case INT8:
+            case BYTE:
+                cohereEmbeddingsServer.enqueue(new MockResponse().setResponseCode(200).setBody(embeddingResponseByte()));
+                break;
+            case FLOAT:
+                cohereEmbeddingsServer.enqueue(new MockResponse().setResponseCode(200).setBody(embeddingResponseFloat()));
+        }
+
+        var inferenceMap = inference(inferenceId, TaskType.TEXT_EMBEDDING, "some text");
+        assertThat(inferenceMap.entrySet(), not(empty()));
+    }
+
+    @SuppressWarnings("unchecked")
+    public void testRerank() throws IOException {
+        var rerankSupported = getOldClusterTestVersion().onOrAfter(COHERE_RERANK_ADDED);
+        assumeTrue("Cohere rerank service added in " + COHERE_RERANK_ADDED, rerankSupported);
+
+        final String oldClusterId = "old-cluster-rerank";
+        final String upgradedClusterId = "upgraded-cluster-rerank";
+
+        if (isOldCluster()) {
+            put(oldClusterId, rerankConfig(getUrl(cohereRerankServer)), TaskType.RERANK);
+            var configs = (List<Map<String, Object>>) get(TaskType.RERANK, oldClusterId).get("models");
+            assertThat(configs, hasSize(1));
+
+            assertRerank(oldClusterId);
+        } else if (isMixedCluster()) {
+            var configs = (List<Map<String, Object>>) get(TaskType.RERANK, oldClusterId).get("models");
+            assertEquals("cohere", configs.get(0).get("service"));
+            var serviceSettings = (Map<String, Object>) configs.get(0).get("service_settings");
+            assertThat(serviceSettings, hasEntry("model_id", "rerank-english-v3.0"));
+            var taskSettings = (Map<String, Object>) configs.get(0).get("task_settings");
+            assertThat(taskSettings, hasEntry("top_n", 3));
+
+            assertRerank(oldClusterId);
+
+        } else if (isUpgradedCluster()) {
+            // check old cluster model
+            var configs = (List<Map<String, Object>>) get(TaskType.RERANK, oldClusterId).get("models");
+            assertEquals("cohere", configs.get(0).get("service"));
+            var serviceSettings = (Map<String, Object>) configs.get(0).get("service_settings");
+            assertThat(serviceSettings, hasEntry("model_id", "rerank-english-v3.0"));
+            var taskSettings = (Map<String, Object>) configs.get(0).get("task_settings");
+            assertThat(taskSettings, hasEntry("top_n", 3));
+
+            assertRerank(oldClusterId);
+
+            // New endpoint
+            put(upgradedClusterId, rerankConfig(getUrl(cohereRerankServer)), TaskType.RERANK);
+            configs = (List<Map<String, Object>>) get(upgradedClusterId).get("models");
+            assertThat(configs, hasSize(1));
+
+            assertRerank(upgradedClusterId);
+
+            delete(oldClusterId);
+            delete(upgradedClusterId);
+        }
+    }
+
+    private void assertRerank(String inferenceId) throws IOException {
+        cohereRerankServer.enqueue(new MockResponse().setResponseCode(200).setBody(rerankResponse()));
+        var inferenceMap = rerank(
+            inferenceId,
+            List.of("luke", "like", "leia", "chewy", "r2d2", "star", "wars"),
+            "star wars main character"
+        );
+        assertThat(inferenceMap.entrySet(), not(empty()));
+    }
+
+    private String embeddingConfigByte(String url) {
+        return embeddingConfigTemplate(url, "byte");
+    }
+
+    private String embeddingConfigInt8(String url) {
+        return embeddingConfigTemplate(url, "int8");
+    }
+
+    private String embeddingConfigFloat(String url) {
+        return embeddingConfigTemplate(url, "float");
+    }
+
+    private String embeddingConfigTemplate(String url, String embeddingType) {
+        return Strings.format("""
+            {
+                "service": "cohere",
+                "service_settings": {
+                    "url": "%s",
+                    "api_key": "XXXX",
+                    "model_id": "embed-english-light-v3.0",
+                    "embedding_type": "%s"
+                }
+            }
+            """, url, embeddingType);
+    }
+
+    private String embeddingResponseByte() {
+        return """
+            {
+                "id": "3198467e-399f-4d4a-aa2c-58af93bd6dc4",
+                "texts": [
+                    "hello"
+                ],
+                "embeddings": [
+                    [
+                        12,
+                        56
+                    ]
+                ],
+                "meta": {
+                    "api_version": {
+                        "version": "1"
+                    },
+                    "billed_units": {
+                        "input_tokens": 1
+                    }
+                },
+                "response_type": "embeddings_bytes"
+            }
+            """;
+    }
+
+    private String embeddingResponseFloat() {
+        return """
+            {
+                "id": "3198467e-399f-4d4a-aa2c-58af93bd6dc4",
+                "texts": [
+                    "hello"
+                ],
+                "embeddings": [
+                    [
+                        -0.0018434525,
+                        0.01777649
+                    ]
+                ],
+                "meta": {
+                    "api_version": {
+                        "version": "1"
+                    },
+                    "billed_units": {
+                        "input_tokens": 1
+                    }
+                },
+                "response_type": "embeddings_floats"
+            }
+            """;
+    }
+
+    private String rerankConfig(String url) {
+        return Strings.format("""
+            {
+                "service": "cohere",
+                "service_settings": {
+                    "api_key": "XXXX",
+                    "model_id": "rerank-english-v3.0",
+                    "url": "%s"
+                },
+                "task_settings": {
+                    "return_documents": false,
+                    "top_n": 3
+                }
+            }
+            """, url);
+    }
+
+    private String rerankResponse() {
+        return """
+            {
+                "index": "d0760819-5a73-4d58-b163-3956d3648b62",
+                "results": [
+                    {
+                        "index": 2,
+                        "relevance_score": 0.98005307
+                    },
+                    {
+                        "index": 3,
+                        "relevance_score": 0.27904198
+                    },
+                    {
+                        "index": 0,
+                        "relevance_score": 0.10194652
+                    }
+                ],
+                "meta": {
+                    "api_version": {
+                        "version": "1"
+                    },
+                    "billed_units": {
+                        "search_units": 1
+                    }
+                }
+            }
+            """;
+    }
+
+}
diff --git a/x-pack/plugin/inference/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/application/HuggingFaceServiceUpgradeIT.java b/x-pack/plugin/inference/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/application/HuggingFaceServiceUpgradeIT.java
new file mode 100644
index 0000000000000..28ecbc2847ad4
--- /dev/null
+++ b/x-pack/plugin/inference/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/application/HuggingFaceServiceUpgradeIT.java
@@ -0,0 +1,194 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.application;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.inference.TaskType;
+import org.elasticsearch.test.http.MockResponse;
+import org.elasticsearch.test.http.MockWebServer;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.empty;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.not;
+
+public class HuggingFaceServiceUpgradeIT extends InferenceUpgradeTestCase {
+
+    private static final String HF_EMBEDDINGS_ADDED = "8.12.0";
+    private static final String HF_ELSER_ADDED = "8.12.0";
+
+    private static MockWebServer embeddingsServer;
+    private static MockWebServer elserServer;
+
+    public HuggingFaceServiceUpgradeIT(@Name("upgradedNodes") int upgradedNodes) {
+        super(upgradedNodes);
+    }
+
+    @BeforeClass
+    public static void startWebServer() throws IOException {
+        embeddingsServer = new MockWebServer();
+        embeddingsServer.start();
+
+        elserServer = new MockWebServer();
+        elserServer.start();
+    }
+
+    @AfterClass
+    public static void shutdown() {
+        embeddingsServer.close();
+        elserServer.close();
+    }
+
+    @SuppressWarnings("unchecked")
+    public void testHFEmbeddings() throws IOException {
+        var embeddingsSupported = getOldClusterTestVersion().onOrAfter(HF_EMBEDDINGS_ADDED);
+        assumeTrue("Hugging Face embedding service added in " + HF_EMBEDDINGS_ADDED, embeddingsSupported);
+
+        final String oldClusterId = "old-cluster-embeddings";
+        final String upgradedClusterId = "upgraded-cluster-embeddings";
+
+        if (isOldCluster()) {
+            // queue a response as PUT will call the service
+            embeddingsServer.enqueue(new MockResponse().setResponseCode(200).setBody(embeddingResponse()));
+            put(oldClusterId, embeddingConfig(getUrl(embeddingsServer)), TaskType.TEXT_EMBEDDING);
+
+            var configs = (List<Map<String, Object>>) get(TaskType.TEXT_EMBEDDING, oldClusterId).get("models");
+            assertThat(configs, hasSize(1));
+
+            assertEmbeddingInference(oldClusterId);
+        } else if (isMixedCluster()) {
+            var configs = (List<Map<String, Object>>) get(TaskType.TEXT_EMBEDDING, oldClusterId).get("models");
+            assertEquals("hugging_face", configs.get(0).get("service"));
+
+            assertEmbeddingInference(oldClusterId);
+        } else if (isUpgradedCluster()) {
+            // check old cluster model
+            var configs = (List<Map<String, Object>>) get(TaskType.TEXT_EMBEDDING, oldClusterId).get("models");
+            assertEquals("hugging_face", configs.get(0).get("service"));
+
+            // Inference on old cluster model
+            assertEmbeddingInference(oldClusterId);
+
+            embeddingsServer.enqueue(new MockResponse().setResponseCode(200).setBody(embeddingResponse()));
+            put(upgradedClusterId, embeddingConfig(getUrl(embeddingsServer)), TaskType.TEXT_EMBEDDING);
+
+            configs = (List<Map<String, Object>>) get(TaskType.TEXT_EMBEDDING, upgradedClusterId).get("models");
+            assertThat(configs, hasSize(1));
+
+            assertEmbeddingInference(upgradedClusterId);
+
+            delete(oldClusterId);
+            delete(upgradedClusterId);
+        }
+    }
+
+    void assertEmbeddingInference(String inferenceId) throws IOException {
+        embeddingsServer.enqueue(new MockResponse().setResponseCode(200).setBody(embeddingResponse()));
+        var inferenceMap = inference(inferenceId, TaskType.TEXT_EMBEDDING, "some text");
+        assertThat(inferenceMap.entrySet(), not(empty()));
+    }
+
+    @SuppressWarnings("unchecked")
+    public void testElser() throws IOException {
+        var supported = getOldClusterTestVersion().onOrAfter(HF_ELSER_ADDED);
+        assumeTrue("HF elser service added in " + HF_ELSER_ADDED, supported);
+
+        final String oldClusterId = "old-cluster-elser";
+        final String upgradedClusterId = "upgraded-cluster-elser";
+
+        if (isOldCluster()) {
+            put(oldClusterId, elserConfig(getUrl(elserServer)), TaskType.SPARSE_EMBEDDING);
+            var configs = (List<Map<String, Object>>) get(TaskType.SPARSE_EMBEDDING, oldClusterId).get("models");
+            assertThat(configs, hasSize(1));
+
+            assertElser(oldClusterId);
+        } else if (isMixedCluster()) {
+            var configs = (List<Map<String, Object>>) get(TaskType.SPARSE_EMBEDDING, oldClusterId).get("models");
+            assertEquals("hugging_face", configs.get(0).get("service"));
+            assertElser(oldClusterId);
+        } else if (isUpgradedCluster()) {
+            // check old cluster model
+            var configs = (List<Map<String, Object>>) get(TaskType.SPARSE_EMBEDDING, oldClusterId).get("models");
+            assertEquals("hugging_face", configs.get(0).get("service"));
+            var taskSettings = (Map<String, Object>) configs.get(0).get("task_settings");
+            assertThat(taskSettings.keySet(), empty());
+
+            assertElser(oldClusterId);
+
+            // New endpoint
+            put(upgradedClusterId, elserConfig(getUrl(elserServer)), TaskType.SPARSE_EMBEDDING);
+            configs = (List<Map<String, Object>>) get(upgradedClusterId).get("models");
+            assertThat(configs, hasSize(1));
+
+            assertElser(upgradedClusterId);
+
+            delete(oldClusterId);
+            delete(upgradedClusterId);
+        }
+    }
+
+    private void assertElser(String inferenceId) throws IOException {
+        elserServer.enqueue(new MockResponse().setResponseCode(200).setBody(elserResponse()));
+        var inferenceMap = inference(inferenceId, TaskType.SPARSE_EMBEDDING, "some text");
+        assertThat(inferenceMap.entrySet(), not(empty()));
+    }
+
+    private String embeddingConfig(String url) {
+        return Strings.format("""
+            {
+                "service": "hugging_face",
+                "service_settings": {
+                    "url": "%s",
+                    "api_key": "XXXX"
+                }
+            }
+            """, url);
+    }
+
+    private String embeddingResponse() {
+        return """
+            [
+                  [
+                      0.014539449,
+                      -0.015288644
+                  ]
+            ]
+            """;
+    }
+
+    private String elserConfig(String url) {
+        return Strings.format("""
+            {
+                "service": "hugging_face",
+                "service_settings": {
+                    "api_key": "XXXX",
+                    "url": "%s"
+                }
+            }
+            """, url);
+    }
+
+    private String elserResponse() {
+        return """
+            [
+                {
+                    ".": 0.133155956864357,
+                    "the": 0.6747211217880249
+                }
+            ]
+            """;
+    }
+
+}
diff --git a/x-pack/plugin/inference/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/application/InferenceUpgradeTestCase.java b/x-pack/plugin/inference/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/application/InferenceUpgradeTestCase.java
new file mode 100644
index 0000000000000..fe08db9b94b89
--- /dev/null
+++ b/x-pack/plugin/inference/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/application/InferenceUpgradeTestCase.java
@@ -0,0 +1,107 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.application;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+
+import org.elasticsearch.client.Request;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.inference.TaskType;
+import org.elasticsearch.test.http.MockWebServer;
+import org.elasticsearch.upgrades.ParameterizedRollingUpgradeTestCase;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+import static org.elasticsearch.core.Strings.format;
+
+public class InferenceUpgradeTestCase extends ParameterizedRollingUpgradeTestCase {
+
+    public InferenceUpgradeTestCase(@Name("upgradedNodes") int upgradedNodes) {
+        super(upgradedNodes);
+    }
+
+    protected static String getUrl(MockWebServer webServer) {
+        return format("http://%s:%s", webServer.getHostName(), webServer.getPort());
+    }
+
+    protected void delete(String inferenceId, TaskType taskType) throws IOException {
+        var request = new Request("DELETE", Strings.format("_inference/%s/%s", taskType, inferenceId));
+        var response = client().performRequest(request);
+        assertOK(response);
+    }
+
+    protected void delete(String inferenceId) throws IOException {
+        var request = new Request("DELETE", Strings.format("_inference/%s", inferenceId));
+        var response = client().performRequest(request);
+        assertOK(response);
+    }
+
+    protected Map<String, Object> getAll() throws IOException {
+        var request = new Request("GET", "_inference/_all");
+        var response = client().performRequest(request);
+        assertOK(response);
+        return entityAsMap(response);
+    }
+
+    protected Map<String, Object> get(String inferenceId) throws IOException {
+        var endpoint = Strings.format("_inference/%s", inferenceId);
+        var request = new Request("GET", endpoint);
+        var response = client().performRequest(request);
+        assertOK(response);
+        return entityAsMap(response);
+    }
+
+    protected Map<String, Object> get(TaskType taskType, String inferenceId) throws IOException {
+        var endpoint = Strings.format("_inference/%s/%s", taskType, inferenceId);
+        var request = new Request("GET", endpoint);
+        var response = client().performRequest(request);
+        assertOK(response);
+        return entityAsMap(response);
+    }
+
+    protected Map<String, Object> inference(String inferenceId, TaskType taskType, String input) throws IOException {
+        var endpoint = Strings.format("_inference/%s/%s", taskType, inferenceId);
+        var request = new Request("POST", endpoint);
+        request.setJsonEntity("{\"input\": [" + '"' + input + '"' + "]}");
+
+        var response = client().performRequest(request);
+        assertOK(response);
+        return entityAsMap(response);
+    }
+
+    protected Map<String, Object> rerank(String inferenceId, List<String> inputs, String query) throws IOException {
+        var endpoint = Strings.format("_inference/rerank/%s", inferenceId);
+        var request = new Request("POST", endpoint);
+
+        StringBuilder body = new StringBuilder("{").append("\"query\":\"").append(query).append("\",").append("\"input\":[");
+
+        for (int i = 0; i < inputs.size(); i++) {
+            body.append("\"").append(inputs.get(i)).append("\"");
+            if (i < inputs.size() - 1) {
+                body.append(",");
+            }
+        }
+
+        body.append("]}");
+        request.setJsonEntity(body.toString());
+
+        var response = client().performRequest(request);
+        assertOK(response);
+        return entityAsMap(response);
+    }
+
+    protected void put(String inferenceId, String modelConfig, TaskType taskType) throws IOException {
+        String endpoint = Strings.format("_inference/%s/%s?error_trace", taskType, inferenceId);
+        var request = new Request("PUT", endpoint);
+        request.setJsonEntity(modelConfig);
+        var response = client().performRequest(request);
+        assertOKAndConsume(response);
+    }
+}
diff --git a/x-pack/plugin/inference/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/application/OpenAiServiceUpgradeIT.java b/x-pack/plugin/inference/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/application/OpenAiServiceUpgradeIT.java
new file mode 100644
index 0000000000000..97d6176f18eed
--- /dev/null
+++ b/x-pack/plugin/inference/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/application/OpenAiServiceUpgradeIT.java
@@ -0,0 +1,265 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.application;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.inference.TaskType;
+import org.elasticsearch.test.http.MockResponse;
+import org.elasticsearch.test.http.MockWebServer;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.empty;
+import static org.hamcrest.Matchers.hasEntry;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.not;
+
+public class OpenAiServiceUpgradeIT extends InferenceUpgradeTestCase {
+
+    private static final String OPEN_AI_EMBEDDINGS_ADDED = "8.12.0";
+    private static final String OPEN_AI_EMBEDDINGS_MODEL_SETTING_MOVED = "8.13.0";
+    private static final String OPEN_AI_COMPLETIONS_ADDED = "8.14.0";
+
+    private static MockWebServer openAiEmbeddingsServer;
+    private static MockWebServer openAiChatCompletionsServer;
+
+    public OpenAiServiceUpgradeIT(@Name("upgradedNodes") int upgradedNodes) {
+        super(upgradedNodes);
+    }
+
+    @BeforeClass
+    public static void startWebServer() throws IOException {
+        openAiEmbeddingsServer = new MockWebServer();
+        openAiEmbeddingsServer.start();
+
+        openAiChatCompletionsServer = new MockWebServer();
+        openAiChatCompletionsServer.start();
+    }
+
+    @AfterClass
+    public static void shutdown() {
+        openAiEmbeddingsServer.close();
+        openAiChatCompletionsServer.close();
+    }
+
+    @SuppressWarnings("unchecked")
+    public void testOpenAiEmbeddings() throws IOException {
+        var openAiEmbeddingsSupported = getOldClusterTestVersion().onOrAfter(OPEN_AI_EMBEDDINGS_ADDED);
+        assumeTrue("OpenAI embedding service added in " + OPEN_AI_EMBEDDINGS_ADDED, openAiEmbeddingsSupported);
+
+        final String oldClusterId = "old-cluster-embeddings";
+        final String upgradedClusterId = "upgraded-cluster-embeddings";
+
+        if (isOldCluster()) {
+            String inferenceConfig = oldClusterVersionCompatibleEmbeddingConfig();
+            // queue a response as PUT will call the service
+            openAiEmbeddingsServer.enqueue(new MockResponse().setResponseCode(200).setBody(embeddingResponse()));
+            put(oldClusterId, inferenceConfig, TaskType.TEXT_EMBEDDING);
+
+            var configs = (List<Map<String, Object>>) get(TaskType.TEXT_EMBEDDING, oldClusterId).get("models");
+            assertThat(configs, hasSize(1));
+
+            assertEmbeddingInference(oldClusterId);
+        } else if (isMixedCluster()) {
+            var configs = (List<Map<String, Object>>) get(TaskType.TEXT_EMBEDDING, oldClusterId).get("models");
+            assertEquals("openai", configs.get(0).get("service"));
+            var serviceSettings = (Map<String, Object>) configs.get(0).get("service_settings");
+            var taskSettings = (Map<String, Object>) configs.get(0).get("task_settings");
+            var modelIdFound = serviceSettings.containsKey("model_id") || taskSettings.containsKey("model_id");
+            assertTrue("model_id not found in config: " + configs.toString(), modelIdFound);
+
+            assertEmbeddingInference(oldClusterId);
+        } else if (isUpgradedCluster()) {
+            // check old cluster model
+            var configs = (List<Map<String, Object>>) get(TaskType.TEXT_EMBEDDING, oldClusterId).get("models");
+            var serviceSettings = (Map<String, Object>) configs.get(0).get("service_settings");
+            // model id is moved to service settings
+            assertThat(serviceSettings, hasEntry("model_id", "text-embedding-ada-002"));
+            var taskSettings = (Map<String, Object>) configs.get(0).get("task_settings");
+            assertThat(taskSettings.keySet(), empty());
+
+            // Inference on old cluster model
+            assertEmbeddingInference(oldClusterId);
+
+            String inferenceConfig = embeddingConfigWithModelInServiceSettings(getUrl(openAiEmbeddingsServer));
+            openAiEmbeddingsServer.enqueue(new MockResponse().setResponseCode(200).setBody(embeddingResponse()));
+            put(upgradedClusterId, inferenceConfig, TaskType.TEXT_EMBEDDING);
+
+            configs = (List<Map<String, Object>>) get(TaskType.TEXT_EMBEDDING, upgradedClusterId).get("models");
+            assertThat(configs, hasSize(1));
+
+            assertEmbeddingInference(upgradedClusterId);
+
+            delete(oldClusterId);
+            delete(upgradedClusterId);
+        }
+    }
+
+    void assertEmbeddingInference(String inferenceId) throws IOException {
+        openAiEmbeddingsServer.enqueue(new MockResponse().setResponseCode(200).setBody(embeddingResponse()));
+        var inferenceMap = inference(inferenceId, TaskType.TEXT_EMBEDDING, "some text");
+        assertThat(inferenceMap.entrySet(), not(empty()));
+    }
+
+    @SuppressWarnings("unchecked")
+    public void testOpenAiCompletions() throws IOException {
+        var openAiEmbeddingsSupported = getOldClusterTestVersion().onOrAfter(OPEN_AI_COMPLETIONS_ADDED);
+        assumeTrue("OpenAI completions service added in " + OPEN_AI_COMPLETIONS_ADDED, openAiEmbeddingsSupported);
+
+        final String oldClusterId = "old-cluster-completions";
+        final String upgradedClusterId = "upgraded-cluster-completions";
+
+        if (isOldCluster()) {
+            put(oldClusterId, chatCompletionsConfig(getUrl(openAiChatCompletionsServer)), TaskType.COMPLETION);
+
+            var configs = (List<Map<String, Object>>) get(TaskType.COMPLETION, oldClusterId).get("models");
+            assertThat(configs, hasSize(1));
+
+            assertCompletionInference(oldClusterId);
+        } else if (isMixedCluster()) {
+            var configs = (List<Map<String, Object>>) get(TaskType.COMPLETION, oldClusterId).get("models");
+            assertEquals("openai", configs.get(0).get("service"));
+            var serviceSettings = (Map<String, Object>) configs.get(0).get("service_settings");
+            assertThat(serviceSettings, hasEntry("model_id", "gpt-4"));
+            var taskSettings = (Map<String, Object>) configs.get(0).get("task_settings");
+            assertThat(taskSettings.keySet(), empty());
+
+            assertCompletionInference(oldClusterId);
+        } else if (isUpgradedCluster()) {
+            // check old cluster model
+            var configs = (List<Map<String, Object>>) get(TaskType.COMPLETION, oldClusterId).get("models");
+            var serviceSettings = (Map<String, Object>) configs.get(0).get("service_settings");
+            assertThat(serviceSettings, hasEntry("model_id", "gpt-4"));
+            var taskSettings = (Map<String, Object>) configs.get(0).get("task_settings");
+            assertThat(taskSettings.keySet(), empty());
+
+            assertCompletionInference(oldClusterId);
+
+            put(upgradedClusterId, chatCompletionsConfig(getUrl(openAiChatCompletionsServer)), TaskType.COMPLETION);
+            configs = (List<Map<String, Object>>) get(TaskType.COMPLETION, upgradedClusterId).get("models");
+            assertThat(configs, hasSize(1));
+
+            // Inference on the new config
+            assertCompletionInference(upgradedClusterId);
+
+            delete(oldClusterId);
+            delete(upgradedClusterId);
+        }
+    }
+
+    void assertCompletionInference(String inferenceId) throws IOException {
+        openAiChatCompletionsServer.enqueue(new MockResponse().setResponseCode(200).setBody(chatCompletionsResponse()));
+        var inferenceMap = inference(inferenceId, TaskType.COMPLETION, "some text");
+        assertThat(inferenceMap.entrySet(), not(empty()));
+    }
+
+    private String oldClusterVersionCompatibleEmbeddingConfig() {
+        if (getOldClusterTestVersion().before(OPEN_AI_EMBEDDINGS_MODEL_SETTING_MOVED)) {
+            return embeddingConfigWithModelInTaskSettings(getUrl(openAiEmbeddingsServer));
+        } else {
+            return embeddingConfigWithModelInServiceSettings(getUrl(openAiEmbeddingsServer));
+        }
+    }
+
+    private String embeddingConfigWithModelInTaskSettings(String url) {
+        return Strings.format("""
+            {
+                "service": "openai",
+                "service_settings": {
+                    "api_key": "XXXX",
+                    "url": "%s"
+                },
+                "task_settings": {
+                   "model": "text-embedding-ada-002"
+                }
+            }
+            """, url);
+    }
+
+    static String embeddingConfigWithModelInServiceSettings(String url) {
+        return Strings.format("""
+            {
+                "service": "openai",
+                "service_settings": {
+                    "api_key": "XXXX",
+                    "url": "%s",
+                    "model_id": "text-embedding-ada-002"
+                }
+            }
+            """, url);
+    }
+
+    private String chatCompletionsConfig(String url) {
+        return Strings.format("""
+            {
+                "service": "openai",
+                "service_settings": {
+                    "api_key": "XXXX",
+                    "url": "%s",
+                    "model_id": "gpt-4"
+                }
+            }
+            """, url);
+    }
+
+    static String embeddingResponse() {
+        return """
+            {
+              "object": "list",
+              "data": [
+                  {
+                      "object": "embedding",
+                      "index": 0,
+                      "embedding": [
+                          0.0123,
+                          -0.0123
+                      ]
+                  }
+              ],
+              "model": "text-embedding-ada-002",
+              "usage": {
+                  "prompt_tokens": 8,
+                  "total_tokens": 8
+              }
+            }
+            """;
+    }
+
+    private String chatCompletionsResponse() {
+        return """
+            {
+              "id": "some-id",
+              "object": "chat.completion",
+              "created": 1705397787,
+              "model": "gpt-3.5-turbo-0613",
+              "choices": [
+                {
+                  "index": 0,
+                  "message": {
+                    "role": "assistant",
+                    "content": "some content"
+                  },
+                  "logprobs": null,
+                  "finish_reason": "stop"
+                }
+              ],
+              "usage": {
+                "prompt_tokens": 46,
+                "completion_tokens": 39,
+                "total_tokens": 85
+              },
+              "system_fingerprint": null
+            }
+            """;
+    }
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ServiceUtils.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ServiceUtils.java
index 8648dfe5595cd..6f9e32e32f667 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ServiceUtils.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ServiceUtils.java
@@ -296,14 +296,13 @@ public static <E extends Enum<E>> E extractOptionalEnum(
             return null;
         }
 
-        var validValuesAsStrings = validValues.stream().map(value -> value.toString().toLowerCase(Locale.ROOT)).toArray(String[]::new);
-
         try {
             var createdEnum = constructor.apply(enumString);
             validateEnumValue(createdEnum, validValues);
 
             return createdEnum;
         } catch (IllegalArgumentException e) {
+            var validValuesAsStrings = validValues.stream().map(value -> value.toString().toLowerCase(Locale.ROOT)).toArray(String[]::new);
             validationException.addValidationError(invalidValue(settingName, scope, enumString, validValuesAsStrings));
         }
 
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/embeddings/CohereEmbeddingsServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/embeddings/CohereEmbeddingsServiceSettings.java
index 9f40ee8ae077f..7d78091a20106 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/embeddings/CohereEmbeddingsServiceSettings.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/embeddings/CohereEmbeddingsServiceSettings.java
@@ -19,10 +19,12 @@
 import org.elasticsearch.xcontent.ToXContentObject;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xpack.inference.services.ConfigurationParseContext;
+import org.elasticsearch.xpack.inference.services.ServiceUtils;
 import org.elasticsearch.xpack.inference.services.cohere.CohereServiceSettings;
 
 import java.io.IOException;
 import java.util.EnumSet;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
 
@@ -46,13 +48,13 @@ public static CohereEmbeddingsServiceSettings fromMap(Map<String, Object> map, C
         return new CohereEmbeddingsServiceSettings(commonServiceSettings, embeddingTypes);
     }
 
-    private static CohereEmbeddingType parseEmbeddingType(
+    static CohereEmbeddingType parseEmbeddingType(
         Map<String, Object> map,
         ConfigurationParseContext context,
         ValidationException validationException
     ) {
-        if (context == ConfigurationParseContext.REQUEST) {
-            return Objects.requireNonNullElse(
+        return switch (context) {
+            case REQUEST -> Objects.requireNonNullElse(
                 extractOptionalEnum(
                     map,
                     EMBEDDING_TYPE,
@@ -63,21 +65,46 @@ private static CohereEmbeddingType parseEmbeddingType(
                 ),
                 CohereEmbeddingType.FLOAT
             );
+            case PERSISTENT -> {
+                var embeddingType = ServiceUtils.extractOptionalString(
+                    map,
+                    EMBEDDING_TYPE,
+                    ModelConfigurations.SERVICE_SETTINGS,
+                    validationException
+                );
+                yield fromCohereOrDenseVectorEnumValues(embeddingType, validationException);
+            }
+
+        };
+    }
+
+    /**
+     * Before TransportVersions::ML_INFERENCE_COHERE_EMBEDDINGS_ADDED element
+     * type was persisted as a CohereEmbeddingType enum. After
+     * DenseVectorFieldMapper.ElementType was used.
+     *
+     * Parse either and convert to a CohereEmbeddingType
+     */
+    static CohereEmbeddingType fromCohereOrDenseVectorEnumValues(String enumString, ValidationException validationException) {
+        if (enumString == null) {
+            return CohereEmbeddingType.FLOAT;
         }
 
-        DenseVectorFieldMapper.ElementType elementType = Objects.requireNonNullElse(
-            extractOptionalEnum(
-                map,
-                EMBEDDING_TYPE,
-                ModelConfigurations.SERVICE_SETTINGS,
-                DenseVectorFieldMapper.ElementType::fromString,
-                CohereEmbeddingType.SUPPORTED_ELEMENT_TYPES,
-                validationException
-            ),
-            DenseVectorFieldMapper.ElementType.FLOAT
-        );
-
-        return CohereEmbeddingType.fromElementType(elementType);
+        try {
+            return CohereEmbeddingType.fromString(enumString);
+        } catch (IllegalArgumentException ae) {
+            try {
+                return CohereEmbeddingType.fromElementType(DenseVectorFieldMapper.ElementType.fromString(enumString));
+            } catch (IllegalArgumentException iae) {
+                var validValuesAsStrings = CohereEmbeddingType.SUPPORTED_ELEMENT_TYPES.stream()
+                    .map(value -> value.toString().toLowerCase(Locale.ROOT))
+                    .toArray(String[]::new);
+                validationException.addValidationError(
+                    ServiceUtils.invalidValue(EMBEDDING_TYPE, ModelConfigurations.SERVICE_SETTINGS, enumString, validValuesAsStrings)
+                );
+                return null;
+            }
+        }
     }
 
     private final CohereServiceSettings commonSettings;
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/embeddings/OpenAiEmbeddingsServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/embeddings/OpenAiEmbeddingsServiceSettings.java
index 87159bb576445..169861381028c 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/embeddings/OpenAiEmbeddingsServiceSettings.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/embeddings/OpenAiEmbeddingsServiceSettings.java
@@ -20,7 +20,6 @@
 import org.elasticsearch.xcontent.ToXContentObject;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xpack.inference.services.ConfigurationParseContext;
-import org.elasticsearch.xpack.inference.services.ServiceUtils;
 import org.elasticsearch.xpack.inference.services.openai.OpenAiRateLimitServiceSettings;
 import org.elasticsearch.xpack.inference.services.settings.RateLimitSettings;
 
@@ -63,22 +62,19 @@ public static OpenAiEmbeddingsServiceSettings fromMap(Map<String, Object> map, C
     }
 
     private static OpenAiEmbeddingsServiceSettings fromPersistentMap(Map<String, Object> map) {
+        // Reading previously persisted config, assume the validation
+        // passed at that time and never throw.
         ValidationException validationException = new ValidationException();
 
         var commonFields = fromMap(map, validationException);
 
         Boolean dimensionsSetByUser = removeAsType(map, DIMENSIONS_SET_BY_USER, Boolean.class);
         if (dimensionsSetByUser == null) {
-            validationException.addValidationError(
-                ServiceUtils.missingSettingErrorMsg(DIMENSIONS_SET_BY_USER, ModelConfigurations.SERVICE_SETTINGS)
-            );
+            // Setting added in 8.13, default to false for configs created prior
+            dimensionsSetByUser = Boolean.FALSE;
         }
 
-        if (validationException.validationErrors().isEmpty() == false) {
-            throw validationException;
-        }
-
-        return new OpenAiEmbeddingsServiceSettings(commonFields, Boolean.TRUE.equals(dimensionsSetByUser));
+        return new OpenAiEmbeddingsServiceSettings(commonFields, dimensionsSetByUser);
     }
 
     private static OpenAiEmbeddingsServiceSettings fromRequestMap(Map<String, Object> map) {
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/embeddings/CohereEmbeddingsServiceSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/embeddings/CohereEmbeddingsServiceSettingsTests.java
index 5d88988292c87..24edb9bfe87f0 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/embeddings/CohereEmbeddingsServiceSettingsTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/embeddings/CohereEmbeddingsServiceSettingsTests.java
@@ -296,6 +296,24 @@ public void testFromMap_PreservesEmbeddingTypeFloat() {
         );
     }
 
+    public void testFromMap_PersistentReadsInt8() {
+        assertThat(
+            CohereEmbeddingsServiceSettings.fromMap(
+                new HashMap<>(Map.of(CohereEmbeddingsServiceSettings.EMBEDDING_TYPE, "int8")),
+                ConfigurationParseContext.PERSISTENT
+            ),
+            is(new CohereEmbeddingsServiceSettings(new CohereServiceSettings(), CohereEmbeddingType.INT8))
+        );
+    }
+
+    public void testFromCohereOrDenseVectorEnumValues() {
+        var validation = new ValidationException();
+        assertEquals(CohereEmbeddingType.BYTE, CohereEmbeddingsServiceSettings.fromCohereOrDenseVectorEnumValues("byte", validation));
+        assertEquals(CohereEmbeddingType.INT8, CohereEmbeddingsServiceSettings.fromCohereOrDenseVectorEnumValues("int8", validation));
+        assertEquals(CohereEmbeddingType.FLOAT, CohereEmbeddingsServiceSettings.fromCohereOrDenseVectorEnumValues("float", validation));
+        assertTrue(validation.validationErrors().isEmpty());
+    }
+
     @Override
     protected Writeable.Reader<CohereEmbeddingsServiceSettings> instanceReader() {
         return CohereEmbeddingsServiceSettings::new;
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/embeddings/OpenAiEmbeddingsServiceSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/embeddings/OpenAiEmbeddingsServiceSettingsTests.java
index b3d9de4a90713..92fb00a4061e2 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/embeddings/OpenAiEmbeddingsServiceSettingsTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/embeddings/OpenAiEmbeddingsServiceSettingsTests.java
@@ -258,18 +258,10 @@ public void testFromMap_PersistentContext_DoesNotThrowException_WhenDimensionsIs
         assertThat(settings, is(new OpenAiEmbeddingsServiceSettings("m", (URI) null, null, null, null, null, true, null)));
     }
 
-    public void testFromMap_PersistentContext_ThrowsException_WhenDimensionsSetByUserIsNull() {
-        var exception = expectThrows(
-            ValidationException.class,
-            () -> OpenAiEmbeddingsServiceSettings.fromMap(
-                new HashMap<>(Map.of(ServiceFields.DIMENSIONS, 1, ServiceFields.MODEL_ID, "m")),
-                ConfigurationParseContext.PERSISTENT
-            )
-        );
-
-        assertThat(
-            exception.getMessage(),
-            containsString("Validation Failed: 1: [service_settings] does not contain the required setting [dimensions_set_by_user];")
+    public void testFromMap_PersistentContext_DoesNotThrowException_WhenDimensionsSetByUserIsNull() {
+        OpenAiEmbeddingsServiceSettings.fromMap(
+            new HashMap<>(Map.of(ServiceFields.DIMENSIONS, 1, ServiceFields.MODEL_ID, "m")),
+            ConfigurationParseContext.PERSISTENT
         );
     }
 

From ee566e4f130a37310f6dd2edfe6ea6653969cb7b Mon Sep 17 00:00:00 2001
From: Oleksandr Kolomiiets <oleksandr.kolomiiets@elastic.co>
Date: Wed, 24 Apr 2024 14:08:21 -0700
Subject: [PATCH 09/58] [TEST] Use GET API instead of search in range field
 synthetic source tests (#107874)

---
 .../test/range/20_synthetic_source.yml        | 296 ++++++++++++++----
 1 file changed, 240 insertions(+), 56 deletions(-)

diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/range/20_synthetic_source.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/range/20_synthetic_source.yml
index eac0fb9a52aa2..60c61ddbb698e 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/range/20_synthetic_source.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/range/20_synthetic_source.yml
@@ -73,28 +73,58 @@ setup:
       indices.refresh: {}
 
   - do:
-      search:
+      get:
         index: synthetic_source_test
-  - match: { hits.total.value: 7 }
+        id: "1"
   - match:
-      hits.hits.0._source:
+      _source:
         integer_range:  { "gte": 1, "lte": 5 }
+
+  - do:
+      get:
+        index: synthetic_source_test
+        id: "2"
   - match:
-      hits.hits.1._source:
+      _source:
         integer_range: { "gte": 2, "lte": 3 }
+
+  - do:
+      get:
+        index: synthetic_source_test
+        id: "3"
   - match:
-      hits.hits.2._source:
+      _source:
         integer_range: { "gte": 4, "lte": 4 }
+
+  - do:
+      get:
+        index: synthetic_source_test
+        id: "4"
   - match:
-      hits.hits.3._source:
+      _source:
         integer_range: [ { "gte": 5, "lte": 6 }, { "gte": 5, "lte": 7 } ]
+
+  - do:
+      get:
+        index: synthetic_source_test
+        id: "5"
   - match:
-      hits.hits.4._source: {}
+      _source: {}
+
+  - do:
+      get:
+        index: synthetic_source_test
+        id: "6"
   - match:
-      hits.hits.5._source:
+      _source:
         integer_range: { "gte": -2147483648, "lte": 10 }
+
+  - do:
+      get:
+        index: synthetic_source_test
+        id: "7"
   - match:
-      hits.hits.6._source:
+      _source:
         integer_range: { "gte": 1, "lte": 2147483647 }
 
 ---
@@ -146,28 +176,58 @@ setup:
       indices.refresh: {}
 
   - do:
-      search:
+      get:
         index: synthetic_source_test
-  - match: { hits.total.value: 7 }
+        id: "1"
   - match:
-      hits.hits.0._source:
+      _source:
         long_range:  { "gte": 1, "lte": 5 }
+
+  - do:
+      get:
+        index: synthetic_source_test
+        id: "2"
   - match:
-      hits.hits.1._source:
+      _source:
         long_range: { "gte": 2, "lte": 3 }
+
+  - do:
+      get:
+        index: synthetic_source_test
+        id: "3"
   - match:
-      hits.hits.2._source:
+      _source:
         long_range: { "gte": 4, "lte": 4 }
+
+  - do:
+      get:
+        index: synthetic_source_test
+        id: "4"
   - match:
-      hits.hits.3._source:
+      _source:
         long_range: [ { "gte": 5, "lte": 6 }, { "gte": 5, "lte": 7 } ]
+
+  - do:
+      get:
+        index: synthetic_source_test
+        id: "5"
   - match:
-      hits.hits.4._source: {}
+      _source: {}
+
+  - do:
+      get:
+        index: synthetic_source_test
+        id: "6"
   - match:
-      hits.hits.5._source:
+      _source:
         long_range: { "gte": -9223372036854775808, "lte": 10 }
+
+  - do:
+      get:
+        index: synthetic_source_test
+        id: "7"
   - match:
-      hits.hits.6._source:
+      _source:
         long_range: { "gte": 1, "lte": 9223372036854775807 }
 
 ---
@@ -213,25 +273,50 @@ setup:
       indices.refresh: {}
 
   - do:
-      search:
+      get:
         index: synthetic_source_test
-  - match: { hits.total.value: 6 }
+        id: "1"
   - match:
-      hits.hits.0._source:
+      _source:
         float_range:  { "gte": 1.0, "lte": 5.0 }
+
+  - do:
+      get:
+        index: synthetic_source_test
+        id: "2"
   - match:
-      hits.hits.1._source:
+      _source:
         float_range: { "gte": 4.0, "lte": 5.0 }
+
+  - do:
+      get:
+        index: synthetic_source_test
+        id: "3"
   - match:
-      hits.hits.2._source:
+      _source:
         float_range: [ { "gte": 4.0, "lte": 7.0 }, { "gte": 4.0, "lte": 8.0 } ]
+
+  - do:
+      get:
+        index: synthetic_source_test
+        id: "4"
   - match:
-      hits.hits.3._source: {}
+      _source: {}
+
+  - do:
+      get:
+        index: synthetic_source_test
+        id: "5"
   - match:
-      hits.hits.4._source:
+      _source:
         float_range: { "gte": "-Infinity", "lte": 10.0 }
+
+  - do:
+      get:
+        index: synthetic_source_test
+        id: "6"
   - match:
-      hits.hits.5._source:
+      _source:
         float_range: { "gte": 1.0, "lte": "Infinity" }
 
 ---
@@ -277,25 +362,50 @@ setup:
       indices.refresh: {}
 
   - do:
-      search:
+      get:
         index: synthetic_source_test
-  - match: { hits.total.value: 6 }
+        id: "1"
   - match:
-      hits.hits.0._source:
+      _source:
         double_range:  { "gte": 1.0, "lte": 5.0 }
+
+  - do:
+      get:
+        index: synthetic_source_test
+        id: "2"
   - match:
-      hits.hits.1._source:
+      _source:
         double_range: { "gte": 4.0, "lte": 5.0 }
+
+  - do:
+      get:
+        index: synthetic_source_test
+        id: "3"
   - match:
-      hits.hits.2._source:
+      _source:
         double_range: [ { "gte": 4.0, "lte": 7.0 }, { "gte": 4.0, "lte": 8.0 } ]
+
+  - do:
+      get:
+        index: synthetic_source_test
+        id: "4"
   - match:
-      hits.hits.3._source: {}
+      _source: {}
+
+  - do:
+      get:
+        index: synthetic_source_test
+        id: "5"
   - match:
-      hits.hits.4._source:
+      _source:
         double_range: { "gte": "-Infinity", "lte": 10.0 }
+
+  - do:
+      get:
+        index: synthetic_source_test
+        id: "6"
   - match:
-      hits.hits.5._source:
+      _source:
         double_range: { "gte": 1.0, "lte": "Infinity" }
 
 ---
@@ -353,31 +463,66 @@ setup:
       indices.refresh: {}
 
   - do:
-      search:
+      get:
         index: synthetic_source_test
-  - match: { hits.total.value: 8 }
+        id: "1"
   - match:
-      hits.hits.0._source:
+      _source:
         ip_range:  { "gte": "192.168.0.1", "lte": "192.168.0.5" }
+
+  - do:
+      get:
+        index: synthetic_source_test
+        id: "2"
   - match:
-      hits.hits.1._source:
+      _source:
         ip_range: { "gte": "192.168.0.2", "lte": "192.168.0.3" }
+
+  - do:
+      get:
+        index: synthetic_source_test
+        id: "3"
   - match:
-      hits.hits.2._source:
+      _source:
         ip_range: { "gte": "192.168.0.4", "lte": "192.168.0.4" }
+
+  - do:
+      get:
+        index: synthetic_source_test
+        id: "4"
   - match:
-      hits.hits.3._source:
+      _source:
         ip_range: { "gte": "2001:db8::1", "lte": "200a:ff:ffff:ffff:ffff:ffff:ffff:ffff" }
+
+  - do:
+      get:
+        index: synthetic_source_test
+        id: "5"
   - match:
-      hits.hits.4._source:
+      _source:
         ip_range: { "gte": "74.125.227.0", "lte": "74.125.227.127" }
+
+  - do:
+      get:
+        index: synthetic_source_test
+        id: "6"
   - match:
-      hits.hits.5._source: {}
+      _source: {}
+
+  - do:
+      get:
+        index: synthetic_source_test
+        id: "7"
   - match:
-      hits.hits.6._source:
+      _source:
         ip_range: { "gte": "0.0.0.0", "lte": "10.10.10.10" }
+
+  - do:
+      get:
+        index: synthetic_source_test
+        id: "8"
   - match:
-      hits.hits.7._source:
+      _source:
         ip_range: { "gte": "2001:db8::", "lte": "ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff" }
 
 ---
@@ -441,33 +586,72 @@ setup:
       indices.refresh: {}
 
   - do:
-      search:
+      get:
         index: synthetic_source_test
-  - match: { hits.total.value: 9 }
+        id: "1"
   - match:
-      hits.hits.0._source:
+      _source:
         date_range:  { "gte": "2017-09-01T00:00:00.000Z", "lte": "2017-09-05T00:00:00.000Z" }
+
+  - do:
+      get:
+        index: synthetic_source_test
+        id: "2"
   - match:
-      hits.hits.1._source:
+      _source:
         date_range: { "gte": "2017-09-01T00:00:00.001Z", "lte": "2017-09-03T00:00:00.000Z" }
+
+  - do:
+      get:
+        index: synthetic_source_test
+        id: "3"
   - match:
-      hits.hits.2._source:
+      _source:
         date_range: { "gte": "2017-09-04T00:00:00.000Z", "lte": "2017-09-04T23:59:59.999Z" }
+
+  - do:
+      get:
+        index: synthetic_source_test
+        id: "4"
   - match:
-      hits.hits.3._source:
+      _source:
         date_range: [ { "gte": "2017-09-04T00:00:00.001Z", "lte": "2017-09-06T23:59:59.999Z" }, { "gte": "2017-09-04T00:00:00.001Z", "lte": "2017-09-07T23:59:59.999Z" } ]
+
+  - do:
+      get:
+        index: synthetic_source_test
+        id: "5"
   - match:
-      hits.hits.4._source:
+      _source:
         date_range: { "gte": "2017-09-01T00:00:00.000Z", "lte": "2017-09-05T00:00:00.000Z" }
+
+  - do:
+      get:
+        index: synthetic_source_test
+        id: "6"
   - match:
-      hits.hits.5._source:
+      _source:
         date_range: { "gte": "2017-09-01T10:20:30.123Z", "lte": "2017-09-05T03:04:05.789Z" }
+
+  - do:
+      get:
+        index: synthetic_source_test
+        id: "7"
   - match:
-      hits.hits.6._source: {}
+      _source: {}
+
+  - do:
+      get:
+        index: synthetic_source_test
+        id: "8"
   - match:
-      hits.hits.7._source:
+      _source:
         date_range: { "gte": "-292275055-05-16T16:47:04.192Z", "lte": "2017-09-05T00:00:00.000Z" }
+
+  - do:
+      get:
+        index: synthetic_source_test
+        id: "9"
   - match:
-      hits.hits.8._source:
+      _source:
         date_range: { "gte": "2017-09-05T00:00:00.000Z", "lte": "+292278994-08-17T07:12:55.807Z" }
-

From 0719c906be0f66bb0b74e2b549b2e30d0e21dfc4 Mon Sep 17 00:00:00 2001
From: Lee Hinman <dakrone@users.noreply.github.com>
Date: Wed, 24 Apr 2024 17:39:24 -0600
Subject: [PATCH 10/58] Gather unassigned replicas corresponding to
 newly-created primary uniquely (#107794)

Related to the work in #101638 this changes the way we calculate whether
all replicas are unassigned when corresponding to newly created
primaries. While this doesn't affect anything in Stateful ES on its own,
it's a building-block used for object-store-based ES (Serverless).

Semi-related to #99951, though it does not solve (and does not strive to
solve) that issue.
---
 ...rdsAvailabilityHealthIndicatorService.java |  29 ++-
 ...ailabilityHealthIndicatorServiceTests.java | 182 +++++++++++++++++-
 2 files changed, 208 insertions(+), 3 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/shards/ShardsAvailabilityHealthIndicatorService.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/shards/ShardsAvailabilityHealthIndicatorService.java
index 74da033fd8811..7c176f65599a9 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/shards/ShardsAvailabilityHealthIndicatorService.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/shards/ShardsAvailabilityHealthIndicatorService.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.cluster.ClusterInfo;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
+import org.elasticsearch.cluster.health.ClusterShardHealth;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.metadata.NodesShutdownMetadata;
@@ -440,7 +441,8 @@ public class ShardAllocationCounts {
         public void increment(ShardRouting routing, ClusterState state, NodesShutdownMetadata shutdowns, boolean verbose) {
             boolean isNew = isUnassignedDueToNewInitialization(routing, state);
             boolean isRestarting = isUnassignedDueToTimelyRestart(routing, shutdowns);
-            boolean allUnavailable = areAllShardsOfThisTypeUnavailable(routing, state);
+            boolean allUnavailable = areAllShardsOfThisTypeUnavailable(routing, state)
+                && isNewlyCreatedAndInitializingReplica(routing, state) == false;
             if (allUnavailable) {
                 indicesWithAllShardsUnavailable.add(routing.getIndexName());
             }
@@ -498,7 +500,7 @@ private void addDefinition(Diagnosis.Definition diagnosisDefinition, String inde
      * example: if a replica is passed then this will return true if ALL replicas are unassigned,
      * but if at least one is assigned, it will return false.
      */
-    private boolean areAllShardsOfThisTypeUnavailable(ShardRouting routing, ClusterState state) {
+    boolean areAllShardsOfThisTypeUnavailable(ShardRouting routing, ClusterState state) {
         return StreamSupport.stream(
             state.routingTable().allActiveShardsGrouped(new String[] { routing.getIndexName() }, true).spliterator(),
             false
@@ -509,6 +511,29 @@ private boolean areAllShardsOfThisTypeUnavailable(ShardRouting routing, ClusterS
             .allMatch(ShardRouting::unassigned);
     }
 
+    /**
+     * Returns true if the given shard is a replica that is only unassigned due to its primary being
+     * newly created. See {@link ClusterShardHealth#getInactivePrimaryHealth(ShardRouting)} for more
+     * information.
+     *
+     * We use this information when considering whether a cluster should turn red. For some cases
+     * (a newly created index having unassigned replicas for example), we don't want the cluster
+     * to turn "unhealthy" for the tiny amount of time before the shards are allocated.
+     */
+    static boolean isNewlyCreatedAndInitializingReplica(ShardRouting routing, ClusterState state) {
+        if (routing.active()) {
+            return false;
+        }
+        if (routing.primary()) {
+            return false;
+        }
+        ShardRouting primary = state.routingTable().shardRoutingTable(routing.shardId()).primaryShard();
+        if (primary.active()) {
+            return false;
+        }
+        return ClusterShardHealth.getInactivePrimaryHealth(primary) == ClusterHealthStatus.YELLOW;
+    }
+
     private static boolean isUnassignedDueToTimelyRestart(ShardRouting routing, NodesShutdownMetadata shutdowns) {
         var info = routing.unassignedInfo();
         if (info == null || info.getReason() != UnassignedInfo.Reason.NODE_RESTARTING) {
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/shards/ShardsAvailabilityHealthIndicatorServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/shards/ShardsAvailabilityHealthIndicatorServiceTests.java
index bb7523661a0fa..77b1fd8988d63 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/shards/ShardsAvailabilityHealthIndicatorServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/shards/ShardsAvailabilityHealthIndicatorServiceTests.java
@@ -23,6 +23,7 @@
 import org.elasticsearch.cluster.routing.RecoverySource;
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.routing.ShardRoutingState;
 import org.elasticsearch.cluster.routing.UnassignedInfo;
 import org.elasticsearch.cluster.routing.allocation.AllocateUnassignedDecision;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
@@ -407,6 +408,95 @@ public void testAllReplicasUnassigned() {
             );
             assertTrue(status.replicas.doAnyIndicesHaveAllUnavailable());
         }
+        {
+            ClusterState clusterState = createClusterStateWith(
+                List.of(
+                    indexNewlyCreated(
+                        "myindex",
+                        new ShardAllocation(
+                            randomNodeId(),
+                            CREATING,
+                            new UnassignedInfo(
+                                UnassignedInfo.Reason.NODE_LEFT,
+                                "message",
+                                null,
+                                0,
+                                0,
+                                0,
+                                false,
+                                UnassignedInfo.AllocationStatus.NO_ATTEMPT,
+                                Set.of(),
+                                null
+                            )
+                        ), // Primary 1
+                        new ShardAllocation(randomNodeId(), UNAVAILABLE) // Replica 1
+                    )
+                ),
+                List.of()
+            );
+            var service = createShardsAvailabilityIndicatorService(clusterState);
+            ShardAllocationStatus status = service.createNewStatus(clusterState.metadata());
+            ShardsAvailabilityHealthIndicatorService.updateShardAllocationStatus(
+                status,
+                clusterState,
+                NodesShutdownMetadata.EMPTY,
+                randomBoolean()
+            );
+            // Here because the replica is unassigned due to the primary being created, it's treated as though the replica can be ignored.
+            assertFalse(
+                "an unassigned replica from a newly created and initializing primary "
+                    + "should not be treated as an index with all replicas unavailable",
+                status.replicas.doAnyIndicesHaveAllUnavailable()
+            );
+        }
+
+        /*
+          A couple of tests for
+          {@link ShardsAvailabilityHealthIndicatorService#areAllShardsOfThisTypeUnavailable(ShardRouting, ClusterState)}
+         */
+        {
+            IndexRoutingTable routingTable = indexWithTwoPrimaryOneReplicaShard(
+                "myindex",
+                new ShardAllocation(randomNodeId(), AVAILABLE), // Primary 1
+                new ShardAllocation(randomNodeId(), AVAILABLE), // Replica 1
+                new ShardAllocation(randomNodeId(), AVAILABLE), // Primary 2
+                new ShardAllocation(randomNodeId(), UNAVAILABLE) // Replica 2
+            );
+            ClusterState clusterState = createClusterStateWith(List.of(routingTable), List.of());
+            var service = createShardsAvailabilityIndicatorService(clusterState);
+            ShardAllocationStatus status = service.createNewStatus(clusterState.metadata());
+            ShardsAvailabilityHealthIndicatorService.updateShardAllocationStatus(
+                status,
+                clusterState,
+                NodesShutdownMetadata.EMPTY,
+                randomBoolean()
+            );
+            var shardRouting = routingTable.shardsWithState(ShardRoutingState.UNASSIGNED).get(0);
+            assertTrue(service.areAllShardsOfThisTypeUnavailable(shardRouting, clusterState));
+        }
+        {
+            ClusterState clusterState = createClusterStateWith(
+                List.of(
+                    index(
+                        "myindex",
+                        new ShardAllocation(randomNodeId(), AVAILABLE),
+                        new ShardAllocation(randomNodeId(), AVAILABLE),
+                        new ShardAllocation(randomNodeId(), UNAVAILABLE)
+                    )
+                ),
+                List.of()
+            );
+            var service = createShardsAvailabilityIndicatorService(clusterState);
+            ShardAllocationStatus status = service.createNewStatus(clusterState.metadata());
+            ShardsAvailabilityHealthIndicatorService.updateShardAllocationStatus(
+                status,
+                clusterState,
+                NodesShutdownMetadata.EMPTY,
+                randomBoolean()
+            );
+            var shardRouting = clusterState.routingTable().index("myindex").shardsWithState(ShardRoutingState.UNASSIGNED).get(0);
+            assertFalse(service.areAllShardsOfThisTypeUnavailable(shardRouting, clusterState));
+        }
     }
 
     public void testShouldBeRedWhenThereAreUnassignedPrimariesAndNoReplicas() {
@@ -1913,6 +2003,72 @@ public void testMappedFieldsForTelemetry() {
         }
     }
 
+    public void testIsNewlyCreatedAndInitializingReplica() {
+        ShardId id = new ShardId("index", "uuid", 0);
+        IndexMetadata idxMeta = IndexMetadata.builder("index")
+            .numberOfShards(1)
+            .numberOfReplicas(1)
+            .settings(
+                Settings.builder()
+                    .put("index.number_of_shards", 1)
+                    .put("index.number_of_replicas", 1)
+                    .put("index.version.created", IndexVersion.current())
+                    .put("index.uuid", "uuid")
+                    .build()
+            )
+            .build();
+        ShardRouting primary = createShardRouting(id, true, new ShardAllocation("node", AVAILABLE));
+        var state = createClusterStateWith(List.of(index("index", new ShardAllocation("node", AVAILABLE))), List.of());
+        assertFalse(ShardsAvailabilityHealthIndicatorService.isNewlyCreatedAndInitializingReplica(primary, state));
+
+        ShardRouting replica = createShardRouting(id, false, new ShardAllocation("node", AVAILABLE));
+        state = createClusterStateWith(List.of(index("index", new ShardAllocation("node", AVAILABLE))), List.of());
+        assertFalse(ShardsAvailabilityHealthIndicatorService.isNewlyCreatedAndInitializingReplica(replica, state));
+
+        ShardRouting unassignedReplica = createShardRouting(id, false, new ShardAllocation("node", UNAVAILABLE));
+        state = createClusterStateWith(
+            List.of(idxMeta),
+            List.of(index("index", "uuid", new ShardAllocation("node", UNAVAILABLE))),
+            List.of(),
+            List.of()
+        );
+        assertFalse(ShardsAvailabilityHealthIndicatorService.isNewlyCreatedAndInitializingReplica(unassignedReplica, state));
+
+        UnassignedInfo.Reason reason = randomFrom(UnassignedInfo.Reason.NODE_LEFT, UnassignedInfo.Reason.NODE_RESTARTING);
+        ShardAllocation allocation = new ShardAllocation(
+            "node",
+            UNAVAILABLE,
+            new UnassignedInfo(
+                reason,
+                "message",
+                null,
+                0,
+                0,
+                0,
+                randomBoolean(),
+                randomFrom(UnassignedInfo.AllocationStatus.values()),
+                Set.of(),
+                reason == UnassignedInfo.Reason.NODE_LEFT ? null : randomAlphaOfLength(20)
+            )
+        );
+        ShardRouting unallocatedReplica = createShardRouting(id, false, allocation);
+        state = createClusterStateWith(
+            List.of(idxMeta),
+            List.of(index(idxMeta, new ShardAllocation("node", UNAVAILABLE), allocation)),
+            List.of(),
+            List.of()
+        );
+        assertFalse(ShardsAvailabilityHealthIndicatorService.isNewlyCreatedAndInitializingReplica(unallocatedReplica, state));
+
+        state = createClusterStateWith(
+            List.of(idxMeta),
+            List.of(index(idxMeta, new ShardAllocation("node", CREATING), allocation)),
+            List.of(),
+            List.of()
+        );
+        assertTrue(ShardsAvailabilityHealthIndicatorService.isNewlyCreatedAndInitializingReplica(unallocatedReplica, state));
+    }
+
     private HealthIndicatorResult createExpectedResult(
         HealthStatus status,
         String symptom,
@@ -2038,9 +2194,18 @@ private static Map<String, Object> addDefaults(Map<String, Object> override) {
     }
 
     private static IndexRoutingTable index(String name, ShardAllocation primaryState, ShardAllocation... replicaStates) {
+        return index(name, "_na_", primaryState, replicaStates);
+    }
+
+    private static IndexRoutingTable index(String name, String uuid, ShardAllocation primaryState, ShardAllocation... replicaStates) {
         return index(
             IndexMetadata.builder(name)
-                .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()).build())
+                .settings(
+                    Settings.builder()
+                        .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())
+                        .put(IndexMetadata.SETTING_INDEX_UUID, uuid)
+                        .build()
+                )
                 .numberOfShards(1)
                 .numberOfReplicas(replicaStates.length)
                 .build(),
@@ -2049,6 +2214,21 @@ private static IndexRoutingTable index(String name, ShardAllocation primaryState
         );
     }
 
+    private static IndexRoutingTable indexNewlyCreated(String name, ShardAllocation primary1State, ShardAllocation replica1State) {
+        var indexMetadata = IndexMetadata.builder(name)
+            .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()).build())
+            .numberOfShards(1)
+            .numberOfReplicas(1)
+            .build();
+        var index = indexMetadata.getIndex();
+        var shard1Id = new ShardId(index, 0);
+
+        var builder = IndexRoutingTable.builder(index);
+        builder.addShard(createShardRouting(shard1Id, true, primary1State));
+        builder.addShard(createShardRouting(shard1Id, false, replica1State));
+        return builder.build();
+    }
+
     private static IndexRoutingTable indexWithTwoPrimaryOneReplicaShard(
         String name,
         ShardAllocation primary1State,

From c0b023c113d6bf0ac8b27b4e4e471c0b5ddd3bab Mon Sep 17 00:00:00 2001
From: Ignacio Vera <ignacio.vera@elastic.co>
Date: Thu, 25 Apr 2024 07:54:29 +0200
Subject: [PATCH 11/58] Optimise few metric aggregations for single value
 fields (#107832)

This commit adds two new abstractions for NumericMetricsAggregator which expects implementation for
LeafCollectors for single and multi-value fields.
---
 docs/changelog/107832.yaml                    |  5 ++
 .../AbstractHDRPercentilesAggregator.java     | 35 ++++----
 .../AbstractTDigestPercentilesAggregator.java | 34 ++++----
 .../aggregations/metrics/AvgAggregator.java   | 62 +++++++------
 .../metrics/ExtendedStatsAggregator.java      | 87 +++++++++++--------
 .../MedianAbsoluteDeviationAggregator.java    | 57 ++++++------
 .../metrics/NumericMetricsAggregator.java     | 74 ++++++++++++++++
 .../aggregations/metrics/StatsAggregator.java | 73 +++++++++-------
 .../aggregations/metrics/SumAggregator.java   | 57 ++++++------
 9 files changed, 306 insertions(+), 178 deletions(-)
 create mode 100644 docs/changelog/107832.yaml

diff --git a/docs/changelog/107832.yaml b/docs/changelog/107832.yaml
new file mode 100644
index 0000000000000..491c491736005
--- /dev/null
+++ b/docs/changelog/107832.yaml
@@ -0,0 +1,5 @@
+pr: 107832
+summary: Optimise few metric aggregations for single value fields
+area: Aggregations
+type: enhancement
+issues: []
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractHDRPercentilesAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractHDRPercentilesAggregator.java
index 670cf08038e03..a1cb547ec0bdd 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractHDRPercentilesAggregator.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractHDRPercentilesAggregator.java
@@ -9,27 +9,24 @@
 package org.elasticsearch.search.aggregations.metrics;
 
 import org.HdrHistogram.DoubleHistogram;
-import org.apache.lucene.search.ScoreMode;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.ObjectArray;
 import org.elasticsearch.core.Releasables;
+import org.elasticsearch.index.fielddata.NumericDoubleValues;
 import org.elasticsearch.index.fielddata.SortedNumericDoubleValues;
 import org.elasticsearch.search.DocValueFormat;
-import org.elasticsearch.search.aggregations.AggregationExecutionContext;
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.LeafBucketCollector;
 import org.elasticsearch.search.aggregations.LeafBucketCollectorBase;
 import org.elasticsearch.search.aggregations.support.AggregationContext;
-import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
 
 import java.io.IOException;
 import java.util.Map;
 
-abstract class AbstractHDRPercentilesAggregator extends NumericMetricsAggregator.MultiValue {
+abstract class AbstractHDRPercentilesAggregator extends NumericMetricsAggregator.MultiDoubleValue {
 
     protected final double[] keys;
-    protected final ValuesSource valuesSource;
     protected final DocValueFormat format;
     protected ObjectArray<DoubleHistogram> states;
     protected final int numberOfSignificantValueDigits;
@@ -46,9 +43,8 @@ abstract class AbstractHDRPercentilesAggregator extends NumericMetricsAggregator
         DocValueFormat formatter,
         Map<String, Object> metadata
     ) throws IOException {
-        super(name, context, parent, metadata);
+        super(name, config, context, parent, metadata);
         assert config.hasValues();
-        this.valuesSource = config.getValuesSource();
         this.keyed = keyed;
         this.format = formatter;
         this.states = context.bigArrays().newObjectArray(1);
@@ -57,26 +53,31 @@ abstract class AbstractHDRPercentilesAggregator extends NumericMetricsAggregator
     }
 
     @Override
-    public ScoreMode scoreMode() {
-        return valuesSource.needsScores() ? ScoreMode.COMPLETE : ScoreMode.COMPLETE_NO_SCORES;
-    }
-
-    @Override
-    public LeafBucketCollector getLeafCollector(AggregationExecutionContext aggCtx, final LeafBucketCollector sub) throws IOException {
-        final SortedNumericDoubleValues values = ((ValuesSource.Numeric) valuesSource).doubleValues(aggCtx.getLeafReaderContext());
+    protected LeafBucketCollector getLeafCollector(SortedNumericDoubleValues values, LeafBucketCollector sub) {
         return new LeafBucketCollectorBase(sub, values) {
             @Override
             public void collect(int doc, long bucket) throws IOException {
                 if (values.advanceExact(doc)) {
-                    DoubleHistogram state = getExistingOrNewHistogram(bigArrays(), bucket);
-                    final int valueCount = values.docValueCount();
-                    for (int i = 0; i < valueCount; i++) {
+                    final DoubleHistogram state = getExistingOrNewHistogram(bigArrays(), bucket);
+                    for (int i = 0; i < values.docValueCount(); i++) {
                         state.recordValue(values.nextValue());
                     }
                 }
             }
         };
+    }
 
+    @Override
+    protected LeafBucketCollector getLeafCollector(NumericDoubleValues values, LeafBucketCollector sub) {
+        return new LeafBucketCollectorBase(sub, values) {
+            @Override
+            public void collect(int doc, long bucket) throws IOException {
+                if (values.advanceExact(doc)) {
+                    final DoubleHistogram state = getExistingOrNewHistogram(bigArrays(), bucket);
+                    state.recordValue(values.doubleValue());
+                }
+            }
+        };
     }
 
     private DoubleHistogram getExistingOrNewHistogram(final BigArrays bigArrays, long bucket) {
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractTDigestPercentilesAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractTDigestPercentilesAggregator.java
index 5b58d2e26abfb..9d86f6800c47e 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractTDigestPercentilesAggregator.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractTDigestPercentilesAggregator.java
@@ -8,27 +8,24 @@
 
 package org.elasticsearch.search.aggregations.metrics;
 
-import org.apache.lucene.search.ScoreMode;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.ObjectArray;
 import org.elasticsearch.core.Releasables;
+import org.elasticsearch.index.fielddata.NumericDoubleValues;
 import org.elasticsearch.index.fielddata.SortedNumericDoubleValues;
 import org.elasticsearch.search.DocValueFormat;
-import org.elasticsearch.search.aggregations.AggregationExecutionContext;
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.LeafBucketCollector;
 import org.elasticsearch.search.aggregations.LeafBucketCollectorBase;
 import org.elasticsearch.search.aggregations.support.AggregationContext;
-import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
 
 import java.io.IOException;
 import java.util.Map;
 
-abstract class AbstractTDigestPercentilesAggregator extends NumericMetricsAggregator.MultiValue {
+abstract class AbstractTDigestPercentilesAggregator extends NumericMetricsAggregator.MultiDoubleValue {
 
     protected final double[] keys;
-    protected final ValuesSource valuesSource;
     protected final DocValueFormat formatter;
     protected ObjectArray<TDigestState> states;
     protected final double compression;
@@ -47,9 +44,8 @@ abstract class AbstractTDigestPercentilesAggregator extends NumericMetricsAggreg
         DocValueFormat formatter,
         Map<String, Object> metadata
     ) throws IOException {
-        super(name, context, parent, metadata);
+        super(name, config, context, parent, metadata);
         assert config.hasValues();
-        this.valuesSource = config.getValuesSource();
         this.keyed = keyed;
         this.formatter = formatter;
         this.states = context.bigArrays().newObjectArray(1);
@@ -59,22 +55,28 @@ abstract class AbstractTDigestPercentilesAggregator extends NumericMetricsAggreg
     }
 
     @Override
-    public ScoreMode scoreMode() {
-        return valuesSource.needsScores() ? ScoreMode.COMPLETE : ScoreMode.COMPLETE_NO_SCORES;
+    protected LeafBucketCollector getLeafCollector(SortedNumericDoubleValues values, final LeafBucketCollector sub) {
+        return new LeafBucketCollectorBase(sub, values) {
+            @Override
+            public void collect(int doc, long bucket) throws IOException {
+                if (values.advanceExact(doc)) {
+                    final TDigestState state = getExistingOrNewHistogram(bigArrays(), bucket);
+                    for (int i = 0; i < values.docValueCount(); i++) {
+                        state.add(values.nextValue());
+                    }
+                }
+            }
+        };
     }
 
     @Override
-    public LeafBucketCollector getLeafCollector(AggregationExecutionContext aggCtx, final LeafBucketCollector sub) throws IOException {
-        final SortedNumericDoubleValues values = ((ValuesSource.Numeric) valuesSource).doubleValues(aggCtx.getLeafReaderContext());
+    protected LeafBucketCollector getLeafCollector(NumericDoubleValues values, final LeafBucketCollector sub) {
         return new LeafBucketCollectorBase(sub, values) {
             @Override
             public void collect(int doc, long bucket) throws IOException {
                 if (values.advanceExact(doc)) {
-                    TDigestState state = getExistingOrNewHistogram(bigArrays(), bucket);
-                    final int valueCount = values.docValueCount();
-                    for (int i = 0; i < valueCount; i++) {
-                        state.add(values.nextValue());
-                    }
+                    final TDigestState state = getExistingOrNewHistogram(bigArrays(), bucket);
+                    state.add(values.doubleValue());
                 }
             }
         };
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AvgAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AvgAggregator.java
index 575108951b899..6588592820547 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AvgAggregator.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AvgAggregator.java
@@ -7,28 +7,24 @@
  */
 package org.elasticsearch.search.aggregations.metrics;
 
-import org.apache.lucene.search.ScoreMode;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.DoubleArray;
 import org.elasticsearch.common.util.LongArray;
 import org.elasticsearch.core.Releasables;
+import org.elasticsearch.index.fielddata.NumericDoubleValues;
 import org.elasticsearch.index.fielddata.SortedNumericDoubleValues;
 import org.elasticsearch.search.DocValueFormat;
-import org.elasticsearch.search.aggregations.AggregationExecutionContext;
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.LeafBucketCollector;
 import org.elasticsearch.search.aggregations.LeafBucketCollectorBase;
 import org.elasticsearch.search.aggregations.support.AggregationContext;
-import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
 
 import java.io.IOException;
 import java.util.Map;
 
-class AvgAggregator extends NumericMetricsAggregator.SingleValue {
-
-    final ValuesSource.Numeric valuesSource;
+class AvgAggregator extends NumericMetricsAggregator.SingleDoubleValue {
 
     LongArray counts;
     DoubleArray sums;
@@ -42,9 +38,8 @@ class AvgAggregator extends NumericMetricsAggregator.SingleValue {
         Aggregator parent,
         Map<String, Object> metadata
     ) throws IOException {
-        super(name, context, parent, metadata);
+        super(name, valuesSourceConfig, context, parent, metadata);
         assert valuesSourceConfig.hasValues();
-        this.valuesSource = (ValuesSource.Numeric) valuesSourceConfig.getValuesSource();
         this.format = valuesSourceConfig.format();
         final BigArrays bigArrays = context.bigArrays();
         counts = bigArrays.newLongArray(1, true);
@@ -53,38 +48,41 @@ class AvgAggregator extends NumericMetricsAggregator.SingleValue {
     }
 
     @Override
-    public ScoreMode scoreMode() {
-        return valuesSource.needsScores() ? ScoreMode.COMPLETE : ScoreMode.COMPLETE_NO_SCORES;
-    }
-
-    @Override
-    public LeafBucketCollector getLeafCollector(AggregationExecutionContext aggCtx, final LeafBucketCollector sub) throws IOException {
-        final SortedNumericDoubleValues values = valuesSource.doubleValues(aggCtx.getLeafReaderContext());
+    protected LeafBucketCollector getLeafCollector(SortedNumericDoubleValues values, final LeafBucketCollector sub) {
         final CompensatedSum kahanSummation = new CompensatedSum(0, 0);
-
         return new LeafBucketCollectorBase(sub, values) {
             @Override
             public void collect(int doc, long bucket) throws IOException {
                 if (values.advanceExact(doc)) {
-                    if (bucket >= counts.size()) {
-                        counts = bigArrays().grow(counts, bucket + 1);
-                        sums = bigArrays().grow(sums, bucket + 1);
-                        compensations = bigArrays().grow(compensations, bucket + 1);
-                    }
+                    maybeGrow(bucket);
                     final int valueCount = values.docValueCount();
                     counts.increment(bucket, valueCount);
                     // Compute the sum of double values with Kahan summation algorithm which is more
                     // accurate than naive summation.
-                    double sum = sums.get(bucket);
-                    double compensation = compensations.get(bucket);
-
-                    kahanSummation.reset(sum, compensation);
-
+                    kahanSummation.reset(sums.get(bucket), compensations.get(bucket));
                     for (int i = 0; i < valueCount; i++) {
-                        double value = values.nextValue();
-                        kahanSummation.add(value);
+                        kahanSummation.add(values.nextValue());
                     }
+                    sums.set(bucket, kahanSummation.value());
+                    compensations.set(bucket, kahanSummation.delta());
+                }
+            }
+        };
+    }
 
+    @Override
+    protected LeafBucketCollector getLeafCollector(NumericDoubleValues values, final LeafBucketCollector sub) {
+        final CompensatedSum kahanSummation = new CompensatedSum(0, 0);
+        return new LeafBucketCollectorBase(sub, values) {
+            @Override
+            public void collect(int doc, long bucket) throws IOException {
+                if (values.advanceExact(doc)) {
+                    maybeGrow(bucket);
+                    counts.increment(bucket, 1L);
+                    // Compute the sum of double values with Kahan summation algorithm which is more
+                    // accurate than naive summation.
+                    kahanSummation.reset(sums.get(bucket), compensations.get(bucket));
+                    kahanSummation.add(values.doubleValue());
                     sums.set(bucket, kahanSummation.value());
                     compensations.set(bucket, kahanSummation.delta());
                 }
@@ -92,6 +90,14 @@ public void collect(int doc, long bucket) throws IOException {
         };
     }
 
+    private void maybeGrow(long bucket) {
+        if (bucket >= counts.size()) {
+            counts = bigArrays().grow(counts, bucket + 1);
+            sums = bigArrays().grow(sums, bucket + 1);
+            compensations = bigArrays().grow(compensations, bucket + 1);
+        }
+    }
+
     @Override
     public double metric(long owningBucketOrd) {
         if (owningBucketOrd >= sums.size()) {
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsAggregator.java
index 194ec2b641757..3645766f47bdf 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsAggregator.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsAggregator.java
@@ -7,31 +7,28 @@
  */
 package org.elasticsearch.search.aggregations.metrics;
 
-import org.apache.lucene.search.ScoreMode;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.DoubleArray;
 import org.elasticsearch.common.util.LongArray;
 import org.elasticsearch.core.Releasables;
+import org.elasticsearch.index.fielddata.NumericDoubleValues;
 import org.elasticsearch.index.fielddata.SortedNumericDoubleValues;
 import org.elasticsearch.search.DocValueFormat;
-import org.elasticsearch.search.aggregations.AggregationExecutionContext;
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.LeafBucketCollector;
 import org.elasticsearch.search.aggregations.LeafBucketCollectorBase;
 import org.elasticsearch.search.aggregations.support.AggregationContext;
-import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
 import org.elasticsearch.xcontent.ParseField;
 
 import java.io.IOException;
 import java.util.Map;
 
-class ExtendedStatsAggregator extends NumericMetricsAggregator.MultiValue {
+class ExtendedStatsAggregator extends NumericMetricsAggregator.MultiDoubleValue {
 
     static final ParseField SIGMA_FIELD = new ParseField("sigma");
 
-    final ValuesSource.Numeric valuesSource;
     final DocValueFormat format;
     final double sigma;
 
@@ -51,9 +48,8 @@ class ExtendedStatsAggregator extends NumericMetricsAggregator.MultiValue {
         double sigma,
         Map<String, Object> metadata
     ) throws IOException {
-        super(name, context, parent, metadata);
+        super(name, config, context, parent, metadata);
         assert config.hasValues();
-        this.valuesSource = (ValuesSource.Numeric) config.getValuesSource();
         this.format = config.format();
         this.sigma = sigma;
         final BigArrays bigArrays = context.bigArrays();
@@ -69,13 +65,7 @@ class ExtendedStatsAggregator extends NumericMetricsAggregator.MultiValue {
     }
 
     @Override
-    public ScoreMode scoreMode() {
-        return valuesSource.needsScores() ? ScoreMode.COMPLETE : ScoreMode.COMPLETE_NO_SCORES;
-    }
-
-    @Override
-    public LeafBucketCollector getLeafCollector(AggregationExecutionContext aggCtx, final LeafBucketCollector sub) throws IOException {
-        final SortedNumericDoubleValues values = valuesSource.doubleValues(aggCtx.getLeafReaderContext());
+    protected LeafBucketCollector getLeafCollector(SortedNumericDoubleValues values, final LeafBucketCollector sub) {
         final CompensatedSum compensatedSum = new CompensatedSum(0, 0);
         final CompensatedSum compensatedSumOfSqr = new CompensatedSum(0, 0);
         return new LeafBucketCollectorBase(sub, values) {
@@ -83,32 +73,15 @@ public LeafBucketCollector getLeafCollector(AggregationExecutionContext aggCtx,
             @Override
             public void collect(int doc, long bucket) throws IOException {
                 if (values.advanceExact(doc)) {
-                    if (bucket >= counts.size()) {
-                        final long from = counts.size();
-                        final long overSize = BigArrays.overSize(bucket + 1);
-                        counts = bigArrays().resize(counts, overSize);
-                        sums = bigArrays().resize(sums, overSize);
-                        compensations = bigArrays().resize(compensations, overSize);
-                        mins = bigArrays().resize(mins, overSize);
-                        maxes = bigArrays().resize(maxes, overSize);
-                        sumOfSqrs = bigArrays().resize(sumOfSqrs, overSize);
-                        compensationOfSqrs = bigArrays().resize(compensationOfSqrs, overSize);
-                        mins.fill(from, overSize, Double.POSITIVE_INFINITY);
-                        maxes.fill(from, overSize, Double.NEGATIVE_INFINITY);
-                    }
+                    maybeGrow(bucket);
                     final int valuesCount = values.docValueCount();
                     counts.increment(bucket, valuesCount);
                     double min = mins.get(bucket);
                     double max = maxes.get(bucket);
                     // Compute the sum and sum of squires for double values with Kahan summation algorithm
                     // which is more accurate than naive summation.
-                    double sum = sums.get(bucket);
-                    double compensation = compensations.get(bucket);
-                    compensatedSum.reset(sum, compensation);
-
-                    double sumOfSqr = sumOfSqrs.get(bucket);
-                    double compensationOfSqr = compensationOfSqrs.get(bucket);
-                    compensatedSumOfSqr.reset(sumOfSqr, compensationOfSqr);
+                    compensatedSum.reset(sums.get(bucket), compensations.get(bucket));
+                    compensatedSumOfSqr.reset(sumOfSqrs.get(bucket), compensationOfSqrs.get(bucket));
 
                     for (int i = 0; i < valuesCount; i++) {
                         double value = values.nextValue();
@@ -126,10 +99,56 @@ public void collect(int doc, long bucket) throws IOException {
                     maxes.set(bucket, max);
                 }
             }
+        };
+    }
+
+    @Override
+    protected LeafBucketCollector getLeafCollector(NumericDoubleValues values, final LeafBucketCollector sub) {
+        final CompensatedSum compensatedSum = new CompensatedSum(0, 0);
+        return new LeafBucketCollectorBase(sub, values) {
+
+            @Override
+            public void collect(int doc, long bucket) throws IOException {
+                if (values.advanceExact(doc)) {
+                    maybeGrow(bucket);
+                    final double value = values.doubleValue();
+                    counts.increment(bucket, 1L);
+                    // Compute the sum and sum of squires for double values with Kahan summation algorithm
+                    // which is more accurate than naive summation.
+                    compensatedSum.reset(sums.get(bucket), compensations.get(bucket));
+                    compensatedSum.add(value);
+                    sums.set(bucket, compensatedSum.value());
+                    compensations.set(bucket, compensatedSum.delta());
+
+                    compensatedSum.reset(sumOfSqrs.get(bucket), compensationOfSqrs.get(bucket));
+                    compensatedSum.add(value * value);
+                    sumOfSqrs.set(bucket, compensatedSum.value());
+                    compensationOfSqrs.set(bucket, compensatedSum.delta());
+
+                    mins.set(bucket, Math.min(mins.get(bucket), value));
+                    maxes.set(bucket, Math.max(maxes.get(bucket), value));
+                }
+            }
 
         };
     }
 
+    private void maybeGrow(long bucket) {
+        if (bucket >= counts.size()) {
+            final long from = counts.size();
+            final long overSize = BigArrays.overSize(bucket + 1);
+            counts = bigArrays().resize(counts, overSize);
+            sums = bigArrays().resize(sums, overSize);
+            compensations = bigArrays().resize(compensations, overSize);
+            mins = bigArrays().resize(mins, overSize);
+            maxes = bigArrays().resize(maxes, overSize);
+            sumOfSqrs = bigArrays().resize(sumOfSqrs, overSize);
+            compensationOfSqrs = bigArrays().resize(compensationOfSqrs, overSize);
+            mins.fill(from, overSize, Double.POSITIVE_INFINITY);
+            maxes.fill(from, overSize, Double.NEGATIVE_INFINITY);
+        }
+    }
+
     @Override
     public boolean hasMetric(String name) {
         return InternalExtendedStats.Metrics.hasMetric(name);
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MedianAbsoluteDeviationAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MedianAbsoluteDeviationAggregator.java
index 61c2c75a49d7c..4382b07ad5460 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MedianAbsoluteDeviationAggregator.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MedianAbsoluteDeviationAggregator.java
@@ -8,18 +8,17 @@
 
 package org.elasticsearch.search.aggregations.metrics;
 
-import org.apache.lucene.search.ScoreMode;
+import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.ObjectArray;
 import org.elasticsearch.core.Releasables;
+import org.elasticsearch.index.fielddata.NumericDoubleValues;
 import org.elasticsearch.index.fielddata.SortedNumericDoubleValues;
 import org.elasticsearch.search.DocValueFormat;
-import org.elasticsearch.search.aggregations.AggregationExecutionContext;
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.LeafBucketCollector;
 import org.elasticsearch.search.aggregations.LeafBucketCollectorBase;
 import org.elasticsearch.search.aggregations.support.AggregationContext;
-import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
 
 import java.io.IOException;
@@ -28,9 +27,8 @@
 
 import static org.elasticsearch.search.aggregations.metrics.InternalMedianAbsoluteDeviation.computeMedianAbsoluteDeviation;
 
-public class MedianAbsoluteDeviationAggregator extends NumericMetricsAggregator.SingleValue {
+public class MedianAbsoluteDeviationAggregator extends NumericMetricsAggregator.SingleDoubleValue {
 
-    private final ValuesSource.Numeric valuesSource;
     private final DocValueFormat format;
 
     private final double compression;
@@ -49,9 +47,8 @@ public class MedianAbsoluteDeviationAggregator extends NumericMetricsAggregator.
         double compression,
         TDigestExecutionHint executionHint
     ) throws IOException {
-        super(name, context, parent, metadata);
+        super(name, config, context, parent, metadata);
         assert config.hasValues();
-        this.valuesSource = (ValuesSource.Numeric) config.getValuesSource();
         this.format = Objects.requireNonNull(format);
         this.compression = compression;
         this.executionHint = executionHint;
@@ -72,39 +69,43 @@ public double metric(long owningBucketOrd) {
     }
 
     @Override
-    public ScoreMode scoreMode() {
-        if (valuesSource.needsScores()) {
-            return ScoreMode.COMPLETE;
-        } else {
-            return ScoreMode.COMPLETE_NO_SCORES;
-        }
+    protected LeafBucketCollector getLeafCollector(SortedNumericDoubleValues values, LeafBucketCollector sub) {
+        return new LeafBucketCollectorBase(sub, values) {
+            @Override
+            public void collect(int doc, long bucket) throws IOException {
+                if (values.advanceExact(doc)) {
+                    final TDigestState valueSketch = getExistingOrNewHistogram(bigArrays(), bucket);
+                    for (int i = 0; i < values.docValueCount(); i++) {
+                        valueSketch.add(values.nextValue());
+                    }
+                }
+            }
+        };
     }
 
     @Override
-    protected LeafBucketCollector getLeafCollector(AggregationExecutionContext aggCtx, LeafBucketCollector sub) throws IOException {
-        final SortedNumericDoubleValues values = valuesSource.doubleValues(aggCtx.getLeafReaderContext());
-
+    protected LeafBucketCollector getLeafCollector(NumericDoubleValues values, LeafBucketCollector sub) {
         return new LeafBucketCollectorBase(sub, values) {
             @Override
             public void collect(int doc, long bucket) throws IOException {
                 if (values.advanceExact(doc)) {
-                    valueSketches = bigArrays().grow(valueSketches, bucket + 1);
-
-                    TDigestState valueSketch = valueSketches.get(bucket);
-                    if (valueSketch == null) {
-                        valueSketch = TDigestState.create(compression, executionHint);
-                        valueSketches.set(bucket, valueSketch);
-                    }
-                    final int valueCount = values.docValueCount();
-                    for (int i = 0; i < valueCount; i++) {
-                        final double value = values.nextValue();
-                        valueSketch.add(value);
-                    }
+                    final TDigestState valueSketch = getExistingOrNewHistogram(bigArrays(), bucket);
+                    valueSketch.add(values.doubleValue());
                 }
             }
         };
     }
 
+    private TDigestState getExistingOrNewHistogram(final BigArrays bigArrays, long bucket) {
+        valueSketches = bigArrays.grow(valueSketches, bucket + 1);
+        TDigestState state = valueSketches.get(bucket);
+        if (state == null) {
+            state = TDigestState.create(compression, executionHint);
+            valueSketches.set(bucket, state);
+        }
+        return state;
+    }
+
     @Override
     public InternalAggregation buildAggregation(long bucket) throws IOException {
         if (hasDataForBucket(bucket)) {
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/NumericMetricsAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/NumericMetricsAggregator.java
index 7853422daac2c..02c7eb3ceba5d 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/NumericMetricsAggregator.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/NumericMetricsAggregator.java
@@ -7,9 +7,17 @@
  */
 package org.elasticsearch.search.aggregations.metrics;
 
+import org.apache.lucene.search.ScoreMode;
 import org.elasticsearch.common.util.Comparators;
+import org.elasticsearch.index.fielddata.FieldData;
+import org.elasticsearch.index.fielddata.NumericDoubleValues;
+import org.elasticsearch.index.fielddata.SortedNumericDoubleValues;
+import org.elasticsearch.search.aggregations.AggregationExecutionContext;
 import org.elasticsearch.search.aggregations.Aggregator;
+import org.elasticsearch.search.aggregations.LeafBucketCollector;
 import org.elasticsearch.search.aggregations.support.AggregationContext;
+import org.elasticsearch.search.aggregations.support.ValuesSource;
+import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
 import org.elasticsearch.search.sort.SortOrder;
 
 import java.io.IOException;
@@ -42,6 +50,39 @@ public BucketComparator bucketComparator(String key, SortOrder order) {
         }
     }
 
+    public abstract static class SingleDoubleValue extends SingleValue {
+
+        private final ValuesSource.Numeric valuesSource;
+
+        protected SingleDoubleValue(
+            String name,
+            ValuesSourceConfig valuesSourceConfig,
+            AggregationContext context,
+            Aggregator parent,
+            Map<String, Object> metadata
+        ) throws IOException {
+            super(name, context, parent, metadata);
+            this.valuesSource = (ValuesSource.Numeric) valuesSourceConfig.getValuesSource();
+        }
+
+        @Override
+        public ScoreMode scoreMode() {
+            return valuesSource.needsScores() ? ScoreMode.COMPLETE : ScoreMode.COMPLETE_NO_SCORES;
+        }
+
+        @Override
+        public final LeafBucketCollector getLeafCollector(AggregationExecutionContext aggCtx, final LeafBucketCollector sub)
+            throws IOException {
+            final SortedNumericDoubleValues values = valuesSource.doubleValues(aggCtx.getLeafReaderContext());
+            final NumericDoubleValues singleton = FieldData.unwrapSingleton(values);
+            return singleton != null ? getLeafCollector(singleton, sub) : getLeafCollector(values, sub);
+        }
+
+        protected abstract LeafBucketCollector getLeafCollector(SortedNumericDoubleValues values, LeafBucketCollector sub);
+
+        protected abstract LeafBucketCollector getLeafCollector(NumericDoubleValues values, LeafBucketCollector sub);
+    }
+
     public abstract static class MultiValue extends NumericMetricsAggregator {
 
         protected MultiValue(String name, AggregationContext context, Aggregator parent, Map<String, Object> metadata) throws IOException {
@@ -64,4 +105,37 @@ public BucketComparator bucketComparator(String key, SortOrder order) {
             return (lhs, rhs) -> Comparators.compareDiscardNaN(metric(key, lhs), metric(key, rhs), order == SortOrder.ASC);
         }
     }
+
+    public abstract static class MultiDoubleValue extends MultiValue {
+
+        private final ValuesSource.Numeric valuesSource;
+
+        protected MultiDoubleValue(
+            String name,
+            ValuesSourceConfig valuesSourceConfig,
+            AggregationContext context,
+            Aggregator parent,
+            Map<String, Object> metadata
+        ) throws IOException {
+            super(name, context, parent, metadata);
+            this.valuesSource = (ValuesSource.Numeric) valuesSourceConfig.getValuesSource();
+        }
+
+        @Override
+        public ScoreMode scoreMode() {
+            return valuesSource.needsScores() ? ScoreMode.COMPLETE : ScoreMode.COMPLETE_NO_SCORES;
+        }
+
+        @Override
+        public final LeafBucketCollector getLeafCollector(AggregationExecutionContext aggCtx, final LeafBucketCollector sub)
+            throws IOException {
+            final SortedNumericDoubleValues values = valuesSource.doubleValues(aggCtx.getLeafReaderContext());
+            final NumericDoubleValues singleton = FieldData.unwrapSingleton(values);
+            return singleton != null ? getLeafCollector(singleton, sub) : getLeafCollector(values, sub);
+        }
+
+        protected abstract LeafBucketCollector getLeafCollector(SortedNumericDoubleValues values, LeafBucketCollector sub);
+
+        protected abstract LeafBucketCollector getLeafCollector(NumericDoubleValues values, LeafBucketCollector sub);
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/StatsAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/StatsAggregator.java
index 8f571a95a145f..61e735901cd2f 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/StatsAggregator.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/StatsAggregator.java
@@ -7,28 +7,25 @@
  */
 package org.elasticsearch.search.aggregations.metrics;
 
-import org.apache.lucene.search.ScoreMode;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.DoubleArray;
 import org.elasticsearch.common.util.LongArray;
 import org.elasticsearch.core.Releasables;
+import org.elasticsearch.index.fielddata.NumericDoubleValues;
 import org.elasticsearch.index.fielddata.SortedNumericDoubleValues;
 import org.elasticsearch.search.DocValueFormat;
-import org.elasticsearch.search.aggregations.AggregationExecutionContext;
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.LeafBucketCollector;
 import org.elasticsearch.search.aggregations.LeafBucketCollectorBase;
 import org.elasticsearch.search.aggregations.support.AggregationContext;
-import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
 
 import java.io.IOException;
 import java.util.Map;
 
-class StatsAggregator extends NumericMetricsAggregator.MultiValue {
+class StatsAggregator extends NumericMetricsAggregator.MultiDoubleValue {
 
-    final ValuesSource.Numeric valuesSource;
     final DocValueFormat format;
 
     LongArray counts;
@@ -39,9 +36,8 @@ class StatsAggregator extends NumericMetricsAggregator.MultiValue {
 
     StatsAggregator(String name, ValuesSourceConfig config, AggregationContext context, Aggregator parent, Map<String, Object> metadata)
         throws IOException {
-        super(name, context, parent, metadata);
+        super(name, config, context, parent, metadata);
         assert config.hasValues();
-        this.valuesSource = (ValuesSource.Numeric) config.getValuesSource();
         counts = bigArrays().newLongArray(1, true);
         sums = bigArrays().newDoubleArray(1, true);
         compensations = bigArrays().newDoubleArray(1, true);
@@ -53,40 +49,20 @@ class StatsAggregator extends NumericMetricsAggregator.MultiValue {
     }
 
     @Override
-    public ScoreMode scoreMode() {
-        return valuesSource.needsScores() ? ScoreMode.COMPLETE : ScoreMode.COMPLETE_NO_SCORES;
-    }
-
-    @Override
-    public LeafBucketCollector getLeafCollector(AggregationExecutionContext aggCtx, final LeafBucketCollector sub) throws IOException {
-        final SortedNumericDoubleValues values = valuesSource.doubleValues(aggCtx.getLeafReaderContext());
+    public LeafBucketCollector getLeafCollector(SortedNumericDoubleValues values, LeafBucketCollector sub) {
         final CompensatedSum kahanSummation = new CompensatedSum(0, 0);
-
         return new LeafBucketCollectorBase(sub, values) {
             @Override
             public void collect(int doc, long bucket) throws IOException {
                 if (values.advanceExact(doc)) {
-                    if (bucket >= counts.size()) {
-                        final long from = counts.size();
-                        final long overSize = BigArrays.overSize(bucket + 1);
-                        counts = bigArrays().resize(counts, overSize);
-                        sums = bigArrays().resize(sums, overSize);
-                        compensations = bigArrays().resize(compensations, overSize);
-                        mins = bigArrays().resize(mins, overSize);
-                        maxes = bigArrays().resize(maxes, overSize);
-                        mins.fill(from, overSize, Double.POSITIVE_INFINITY);
-                        maxes.fill(from, overSize, Double.NEGATIVE_INFINITY);
-                    }
+                    maybeGrow(bucket);
                     final int valuesCount = values.docValueCount();
                     counts.increment(bucket, valuesCount);
                     double min = mins.get(bucket);
                     double max = maxes.get(bucket);
                     // Compute the sum of double values with Kahan summation algorithm which is more
                     // accurate than naive summation.
-                    double sum = sums.get(bucket);
-                    double compensation = compensations.get(bucket);
-                    kahanSummation.reset(sum, compensation);
-
+                    kahanSummation.reset(sums.get(bucket), compensations.get(bucket));
                     for (int i = 0; i < valuesCount; i++) {
                         double value = values.nextValue();
                         kahanSummation.add(value);
@@ -102,6 +78,43 @@ public void collect(int doc, long bucket) throws IOException {
         };
     }
 
+    @Override
+    public LeafBucketCollector getLeafCollector(NumericDoubleValues values, LeafBucketCollector sub) {
+        final CompensatedSum kahanSummation = new CompensatedSum(0, 0);
+        return new LeafBucketCollectorBase(sub, values) {
+            @Override
+            public void collect(int doc, long bucket) throws IOException {
+                if (values.advanceExact(doc)) {
+                    maybeGrow(bucket);
+                    counts.increment(bucket, 1L);
+                    // Compute the sum of double values with Kahan summation algorithm which is more
+                    // accurate than naive summation.
+                    kahanSummation.reset(sums.get(bucket), compensations.get(bucket));
+                    double value = values.doubleValue();
+                    kahanSummation.add(value);
+                    sums.set(bucket, kahanSummation.value());
+                    compensations.set(bucket, kahanSummation.delta());
+                    mins.set(bucket, Math.min(mins.get(bucket), value));
+                    maxes.set(bucket, Math.max(maxes.get(bucket), value));
+                }
+            }
+        };
+    }
+
+    private void maybeGrow(long bucket) {
+        if (bucket >= counts.size()) {
+            final long from = counts.size();
+            final long overSize = BigArrays.overSize(bucket + 1);
+            counts = bigArrays().resize(counts, overSize);
+            sums = bigArrays().resize(sums, overSize);
+            compensations = bigArrays().resize(compensations, overSize);
+            mins = bigArrays().resize(mins, overSize);
+            maxes = bigArrays().resize(maxes, overSize);
+            mins.fill(from, overSize, Double.POSITIVE_INFINITY);
+            maxes.fill(from, overSize, Double.NEGATIVE_INFINITY);
+        }
+    }
+
     @Override
     public boolean hasMetric(String name) {
         return InternalStats.Metrics.hasMetric(name);
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/SumAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/SumAggregator.java
index 94d9f311db621..105e52dbc91c7 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/SumAggregator.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/SumAggregator.java
@@ -7,26 +7,23 @@
  */
 package org.elasticsearch.search.aggregations.metrics;
 
-import org.apache.lucene.search.ScoreMode;
 import org.elasticsearch.common.util.DoubleArray;
 import org.elasticsearch.core.Releasables;
+import org.elasticsearch.index.fielddata.NumericDoubleValues;
 import org.elasticsearch.index.fielddata.SortedNumericDoubleValues;
 import org.elasticsearch.search.DocValueFormat;
-import org.elasticsearch.search.aggregations.AggregationExecutionContext;
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.LeafBucketCollector;
 import org.elasticsearch.search.aggregations.LeafBucketCollectorBase;
 import org.elasticsearch.search.aggregations.support.AggregationContext;
-import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
 
 import java.io.IOException;
 import java.util.Map;
 
-public class SumAggregator extends NumericMetricsAggregator.SingleValue {
+public class SumAggregator extends NumericMetricsAggregator.SingleDoubleValue {
 
-    private final ValuesSource.Numeric valuesSource;
     private final DocValueFormat format;
 
     private DoubleArray sums;
@@ -39,43 +36,46 @@ public class SumAggregator extends NumericMetricsAggregator.SingleValue {
         Aggregator parent,
         Map<String, Object> metadata
     ) throws IOException {
-        super(name, context, parent, metadata);
+        super(name, valuesSourceConfig, context, parent, metadata);
         assert valuesSourceConfig.hasValues();
-        this.valuesSource = (ValuesSource.Numeric) valuesSourceConfig.getValuesSource();
         this.format = valuesSourceConfig.format();
         sums = bigArrays().newDoubleArray(1, true);
         compensations = bigArrays().newDoubleArray(1, true);
     }
 
     @Override
-    public ScoreMode scoreMode() {
-        return valuesSource.needsScores() ? ScoreMode.COMPLETE : ScoreMode.COMPLETE_NO_SCORES;
+    protected LeafBucketCollector getLeafCollector(SortedNumericDoubleValues values, final LeafBucketCollector sub) {
+        final CompensatedSum kahanSummation = new CompensatedSum(0, 0);
+        return new LeafBucketCollectorBase(sub, values) {
+            @Override
+            public void collect(int doc, long bucket) throws IOException {
+                if (values.advanceExact(doc)) {
+                    maybeGrow(bucket);
+                    // Compute the sum of double values with Kahan summation algorithm which is more
+                    // accurate than naive summation.
+                    kahanSummation.reset(sums.get(bucket), compensations.get(bucket));
+                    for (int i = 0; i < values.docValueCount(); i++) {
+                        kahanSummation.add(values.nextValue());
+                    }
+                    compensations.set(bucket, kahanSummation.delta());
+                    sums.set(bucket, kahanSummation.value());
+                }
+            }
+        };
     }
 
     @Override
-    public LeafBucketCollector getLeafCollector(AggregationExecutionContext aggCtx, final LeafBucketCollector sub) throws IOException {
-        final SortedNumericDoubleValues values = valuesSource.doubleValues(aggCtx.getLeafReaderContext());
+    protected LeafBucketCollector getLeafCollector(NumericDoubleValues values, final LeafBucketCollector sub) {
         final CompensatedSum kahanSummation = new CompensatedSum(0, 0);
         return new LeafBucketCollectorBase(sub, values) {
             @Override
             public void collect(int doc, long bucket) throws IOException {
                 if (values.advanceExact(doc)) {
-                    if (bucket >= sums.size()) {
-                        sums = bigArrays().grow(sums, bucket + 1);
-                        compensations = bigArrays().grow(compensations, bucket + 1);
-                    }
-                    final int valuesCount = values.docValueCount();
+                    maybeGrow(bucket);
                     // Compute the sum of double values with Kahan summation algorithm which is more
                     // accurate than naive summation.
-                    double sum = sums.get(bucket);
-                    double compensation = compensations.get(bucket);
-                    kahanSummation.reset(sum, compensation);
-
-                    for (int i = 0; i < valuesCount; i++) {
-                        double value = values.nextValue();
-                        kahanSummation.add(value);
-                    }
-
+                    kahanSummation.reset(sums.get(bucket), compensations.get(bucket));
+                    kahanSummation.add(values.doubleValue());
                     compensations.set(bucket, kahanSummation.delta());
                     sums.set(bucket, kahanSummation.value());
                 }
@@ -83,6 +83,13 @@ public void collect(int doc, long bucket) throws IOException {
         };
     }
 
+    private void maybeGrow(long bucket) {
+        if (bucket >= sums.size()) {
+            sums = bigArrays().grow(sums, bucket + 1);
+            compensations = bigArrays().grow(compensations, bucket + 1);
+        }
+    }
+
     @Override
     public double metric(long owningBucketOrd) {
         if (owningBucketOrd >= sums.size()) {

From b687ef702dc5a07876897026050df5da3eb5b7b9 Mon Sep 17 00:00:00 2001
From: Armin Braun <me@obrown.io>
Date: Thu, 25 Apr 2024 09:06:41 +0200
Subject: [PATCH 12/58] Remove unused reader field from TransportTasksAction
 (#107458)

Removing the unused reader field as ewll as some upstream code
that becomes unused as a result.
---
 .../elasticsearch/reindex/TransportRethrottleAction.java    | 1 -
 .../node/tasks/cancel/TransportCancelTasksAction.java       | 1 -
 .../admin/cluster/node/tasks/list/ListTasksResponse.java    | 6 ------
 .../cluster/node/tasks/list/TransportListTasksAction.java   | 1 -
 .../action/support/tasks/TransportTasksAction.java          | 3 ---
 .../action/admin/cluster/node/tasks/TestTaskPlugin.java     | 1 -
 .../admin/cluster/node/tasks/TransportTasksActionTests.java | 1 -
 .../elasticsearch/persistent/TestPersistentTasksPlugin.java | 1 -
 .../xpack/ccr/action/TransportFollowStatsAction.java        | 1 -
 .../ml/action/TransportClearDeploymentCacheAction.java      | 1 -
 .../xpack/ml/action/TransportCloseJobAction.java            | 1 -
 .../action/TransportGetDataFrameAnalyticsStatsAction.java   | 1 -
 .../ml/action/TransportGetDatafeedRunningStateAction.java   | 1 -
 .../xpack/ml/action/TransportGetDeploymentStatsAction.java  | 1 -
 .../xpack/ml/action/TransportGetJobsStatsAction.java        | 1 -
 .../action/TransportInferTrainedModelDeploymentAction.java  | 1 -
 .../xpack/ml/action/TransportIsolateDatafeedAction.java     | 1 -
 .../xpack/ml/action/TransportJobTaskAction.java             | 2 +-
 .../xpack/ml/action/TransportKillProcessAction.java         | 1 -
 .../ml/action/TransportStopDataFrameAnalyticsAction.java    | 1 -
 .../xpack/ml/action/TransportStopDatafeedAction.java        | 1 -
 .../action/TransportStopTrainedModelDeploymentAction.java   | 1 -
 .../xpack/rollup/action/TransportDeleteRollupJobAction.java | 1 -
 .../xpack/rollup/action/TransportGetRollupJobAction.java    | 1 -
 .../xpack/rollup/action/TransportStartRollupAction.java     | 1 -
 .../xpack/rollup/action/TransportStopRollupAction.java      | 1 -
 .../transform/action/TransportGetTransformStatsAction.java  | 1 -
 .../action/TransportScheduleNowTransformAction.java         | 1 -
 .../transform/action/TransportStopTransformAction.java      | 1 -
 .../transform/action/TransportUpdateTransformAction.java    | 1 -
 30 files changed, 1 insertion(+), 37 deletions(-)

diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/TransportRethrottleAction.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/TransportRethrottleAction.java
index 68e7d14038b67..e5df42a484172 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/reindex/TransportRethrottleAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/TransportRethrottleAction.java
@@ -44,7 +44,6 @@ public TransportRethrottleAction(
             transportService,
             actionFilters,
             RethrottleRequest::new,
-            ListTasksResponse::new,
             TaskInfo::from,
             transportService.getThreadPool().executor(ThreadPool.Names.MANAGEMENT)
         );
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/cancel/TransportCancelTasksAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/cancel/TransportCancelTasksAction.java
index d2e79bc63daf8..4582a1cb26f82 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/cancel/TransportCancelTasksAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/cancel/TransportCancelTasksAction.java
@@ -47,7 +47,6 @@ public TransportCancelTasksAction(ClusterService clusterService, TransportServic
             transportService,
             actionFilters,
             CancelTasksRequest::new,
-            ListTasksResponse::new,
             TaskInfo::from,
             // Cancellation is usually lightweight, and runs on the transport thread if the task didn't even start yet, but some
             // implementations of CancellableTask#onCancelled() are nontrivial so we use GENERIC here. TODO could it be SAME?
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/ListTasksResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/ListTasksResponse.java
index 6d052c242c55c..6116a96faf60b 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/ListTasksResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/ListTasksResponse.java
@@ -16,7 +16,6 @@
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.collect.Iterators;
-import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.ChunkedToXContent;
 import org.elasticsearch.common.xcontent.ChunkedToXContentObject;
@@ -52,11 +51,6 @@ public ListTasksResponse(
         this.tasks = tasks == null ? List.of() : List.copyOf(tasks);
     }
 
-    public ListTasksResponse(StreamInput in) throws IOException {
-        super(in);
-        tasks = in.readCollectionAsImmutableList(TaskInfo::from);
-    }
-
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TransportListTasksAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TransportListTasksAction.java
index 4f8a6b6db2980..c4888b9900428 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TransportListTasksAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TransportListTasksAction.java
@@ -60,7 +60,6 @@ public TransportListTasksAction(ClusterService clusterService, TransportService
             transportService,
             actionFilters,
             ListTasksRequest::new,
-            ListTasksResponse::new,
             TaskInfo::from,
             transportService.getThreadPool().executor(ThreadPool.Names.MANAGEMENT)
         );
diff --git a/server/src/main/java/org/elasticsearch/action/support/tasks/TransportTasksAction.java b/server/src/main/java/org/elasticsearch/action/support/tasks/TransportTasksAction.java
index 0ee6af717a1cb..b367d33adb908 100644
--- a/server/src/main/java/org/elasticsearch/action/support/tasks/TransportTasksAction.java
+++ b/server/src/main/java/org/elasticsearch/action/support/tasks/TransportTasksAction.java
@@ -56,7 +56,6 @@ public abstract class TransportTasksAction<
     protected final ClusterService clusterService;
     protected final TransportService transportService;
     protected final Writeable.Reader<TasksRequest> requestReader;
-    protected final Writeable.Reader<TasksResponse> responsesReader;
     protected final Writeable.Reader<TaskResponse> responseReader;
 
     protected final String transportNodeAction;
@@ -67,7 +66,6 @@ protected TransportTasksAction(
         TransportService transportService,
         ActionFilters actionFilters,
         Writeable.Reader<TasksRequest> requestReader,
-        Writeable.Reader<TasksResponse> responsesReader,
         Writeable.Reader<TaskResponse> responseReader,
         Executor nodeExecutor
     ) {
@@ -77,7 +75,6 @@ protected TransportTasksAction(
         this.transportService = transportService;
         this.transportNodeAction = actionName + "[n]";
         this.requestReader = requestReader;
-        this.responsesReader = responsesReader;
         this.responseReader = responseReader;
 
         transportService.registerRequestHandler(transportNodeAction, nodeExecutor, NodeTaskRequest::new, new NodeTransportHandler());
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TestTaskPlugin.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TestTaskPlugin.java
index 8b8d4e52d33d4..63629e16974d5 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TestTaskPlugin.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TestTaskPlugin.java
@@ -391,7 +391,6 @@ public TransportUnblockTestTasksAction(ClusterService clusterService, TransportS
                 transportService,
                 new ActionFilters(new HashSet<>()),
                 UnblockTestTasksRequest::new,
-                UnblockTestTasksResponse::new,
                 UnblockTestTaskResponse::new,
                 transportService.getThreadPool().executor(ThreadPool.Names.MANAGEMENT)
             );
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java
index 67cba13661e34..6f4da1fe1ebe0 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java
@@ -261,7 +261,6 @@ protected TestTasksAction(String actionName, ClusterService clusterService, Tran
                 transportService,
                 new ActionFilters(new HashSet<>()),
                 TestTasksRequest::new,
-                TestTasksResponse::new,
                 TestTaskResponse::new,
                 transportService.getThreadPool().executor(ThreadPool.Names.MANAGEMENT)
             );
diff --git a/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java b/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java
index 8545ddc067a8d..bf436b235d93b 100644
--- a/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java
+++ b/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java
@@ -531,7 +531,6 @@ public TransportTestTaskAction(ClusterService clusterService, TransportService t
                 transportService,
                 actionFilters,
                 TestTasksRequest::new,
-                TestTasksResponse::new,
                 TestTaskResponse::new,
                 transportService.getThreadPool().executor(ThreadPool.Names.MANAGEMENT)
             );
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportFollowStatsAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportFollowStatsAction.java
index ecca422282736..44710372058a3 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportFollowStatsAction.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportFollowStatsAction.java
@@ -59,7 +59,6 @@ public TransportFollowStatsAction(
             transportService,
             actionFilters,
             FollowStatsAction.StatsRequest::new,
-            FollowStatsAction.StatsResponses::new,
             FollowStatsAction.StatsResponse::new,
             transportService.getThreadPool().executor(Ccr.CCR_THREAD_POOL_NAME)
         );
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportClearDeploymentCacheAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportClearDeploymentCacheAction.java
index 5ecd0322674e1..c3a09902de100 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportClearDeploymentCacheAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportClearDeploymentCacheAction.java
@@ -45,7 +45,6 @@ public TransportClearDeploymentCacheAction(
             actionFilters,
             Request::new,
             Response::new,
-            Response::new,
             EsExecutors.DIRECT_EXECUTOR_SERVICE
         );
     }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportCloseJobAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportCloseJobAction.java
index 1ddb7d84208d0..af18ba6170cda 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportCloseJobAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportCloseJobAction.java
@@ -100,7 +100,6 @@ public TransportCloseJobAction(
             actionFilters,
             CloseJobAction.Request::new,
             CloseJobAction.Response::new,
-            CloseJobAction.Response::new,
             EsExecutors.DIRECT_EXECUTOR_SERVICE
         );
         this.threadPool = threadPool;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDataFrameAnalyticsStatsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDataFrameAnalyticsStatsAction.java
index 38da82124e77f..07c0e3d7ea618 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDataFrameAnalyticsStatsAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDataFrameAnalyticsStatsAction.java
@@ -96,7 +96,6 @@ public TransportGetDataFrameAnalyticsStatsAction(
             transportService,
             actionFilters,
             GetDataFrameAnalyticsStatsAction.Request::new,
-            GetDataFrameAnalyticsStatsAction.Response::new,
             in -> new QueryPage<>(in, GetDataFrameAnalyticsStatsAction.Response.Stats::new),
             transportService.getThreadPool().executor(ThreadPool.Names.MANAGEMENT)
         );
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDatafeedRunningStateAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDatafeedRunningStateAction.java
index c9b85915a9fd6..a36fc2cdc1ee2 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDatafeedRunningStateAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDatafeedRunningStateAction.java
@@ -56,7 +56,6 @@ public TransportGetDatafeedRunningStateAction(
             actionFilters,
             Request::new,
             Response::new,
-            Response::new,
             transportService.getThreadPool().executor(ThreadPool.Names.MANAGEMENT)
         );
     }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDeploymentStatsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDeploymentStatsAction.java
index 14afd6999b0c0..04b597292dad6 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDeploymentStatsAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDeploymentStatsAction.java
@@ -64,7 +64,6 @@ public TransportGetDeploymentStatsAction(
             transportService,
             actionFilters,
             GetDeploymentStatsAction.Request::new,
-            GetDeploymentStatsAction.Response::new,
             AssignmentStats::new,
             transportService.getThreadPool().executor(ThreadPool.Names.MANAGEMENT)
         );
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetJobsStatsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetJobsStatsAction.java
index 46aa219db9fbe..c5061b77e2c6a 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetJobsStatsAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetJobsStatsAction.java
@@ -81,7 +81,6 @@ public TransportGetJobsStatsAction(
             transportService,
             actionFilters,
             GetJobsStatsAction.Request::new,
-            GetJobsStatsAction.Response::new,
             in -> new QueryPage<>(in, JobStats::new),
             threadPool.executor(ThreadPool.Names.MANAGEMENT)
         );
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportInferTrainedModelDeploymentAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportInferTrainedModelDeploymentAction.java
index 2760c2990fadf..c492ed1d804d8 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportInferTrainedModelDeploymentAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportInferTrainedModelDeploymentAction.java
@@ -50,7 +50,6 @@ public TransportInferTrainedModelDeploymentAction(
             actionFilters,
             InferTrainedModelDeploymentAction.Request::new,
             InferTrainedModelDeploymentAction.Response::new,
-            InferTrainedModelDeploymentAction.Response::new,
             EsExecutors.DIRECT_EXECUTOR_SERVICE
         );
     }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportIsolateDatafeedAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportIsolateDatafeedAction.java
index 4ceba52a7bd83..57ec0194bb918 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportIsolateDatafeedAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportIsolateDatafeedAction.java
@@ -40,7 +40,6 @@ public TransportIsolateDatafeedAction(TransportService transportService, ActionF
             actionFilters,
             IsolateDatafeedAction.Request::new,
             IsolateDatafeedAction.Response::new,
-            IsolateDatafeedAction.Response::new,
             transportService.getThreadPool().executor(MachineLearning.UTILITY_THREAD_POOL_NAME)
         );
     }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportJobTaskAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportJobTaskAction.java
index b9e908dc724b3..9678f007e9397 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportJobTaskAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportJobTaskAction.java
@@ -46,7 +46,7 @@ public abstract class TransportJobTaskAction<Request extends JobTaskRequest<Requ
         Executor nodeExecutor,
         AutodetectProcessManager processManager
     ) {
-        super(actionName, clusterService, transportService, actionFilters, requestReader, responseReader, responseReader, nodeExecutor);
+        super(actionName, clusterService, transportService, actionFilters, requestReader, responseReader, nodeExecutor);
         this.processManager = processManager;
     }
 
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportKillProcessAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportKillProcessAction.java
index b79ba6a3cc46c..dd575ce6c9a22 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportKillProcessAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportKillProcessAction.java
@@ -57,7 +57,6 @@ public TransportKillProcessAction(
             actionFilters,
             KillProcessAction.Request::new,
             KillProcessAction.Response::new,
-            KillProcessAction.Response::new,
             transportService.getThreadPool().executor(MachineLearning.UTILITY_THREAD_POOL_NAME)
         );
         this.auditor = auditor;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStopDataFrameAnalyticsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStopDataFrameAnalyticsAction.java
index 9ad0213bf7ee5..94cce17b4709f 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStopDataFrameAnalyticsAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStopDataFrameAnalyticsAction.java
@@ -92,7 +92,6 @@ public TransportStopDataFrameAnalyticsAction(
             actionFilters,
             StopDataFrameAnalyticsAction.Request::new,
             StopDataFrameAnalyticsAction.Response::new,
-            StopDataFrameAnalyticsAction.Response::new,
             EsExecutors.DIRECT_EXECUTOR_SERVICE
         );
         this.threadPool = threadPool;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStopDatafeedAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStopDatafeedAction.java
index 8ba8132ecafa2..b3178e7311447 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStopDatafeedAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStopDatafeedAction.java
@@ -94,7 +94,6 @@ public TransportStopDatafeedAction(
             actionFilters,
             StopDatafeedAction.Request::new,
             StopDatafeedAction.Response::new,
-            StopDatafeedAction.Response::new,
             threadPool.executor(MachineLearning.UTILITY_THREAD_POOL_NAME)
         );
         this.threadPool = Objects.requireNonNull(threadPool);
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStopTrainedModelDeploymentAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStopTrainedModelDeploymentAction.java
index a3eb15a372d2a..5f1ec76ae2de2 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStopTrainedModelDeploymentAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStopTrainedModelDeploymentAction.java
@@ -83,7 +83,6 @@ public TransportStopTrainedModelDeploymentAction(
             actionFilters,
             StopTrainedModelDeploymentAction.Request::new,
             StopTrainedModelDeploymentAction.Response::new,
-            StopTrainedModelDeploymentAction.Response::new,
             EsExecutors.DIRECT_EXECUTOR_SERVICE
         );
         this.ingestService = ingestService;
diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportDeleteRollupJobAction.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportDeleteRollupJobAction.java
index 86477f4f8c180..6923ab2a1f618 100644
--- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportDeleteRollupJobAction.java
+++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportDeleteRollupJobAction.java
@@ -46,7 +46,6 @@ public TransportDeleteRollupJobAction(TransportService transportService, ActionF
             actionFilters,
             DeleteRollupJobAction.Request::new,
             DeleteRollupJobAction.Response::new,
-            DeleteRollupJobAction.Response::new,
             EsExecutors.DIRECT_EXECUTOR_SERVICE
         );
     }
diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportGetRollupJobAction.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportGetRollupJobAction.java
index 82c2782fa469d..646efed572df5 100644
--- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportGetRollupJobAction.java
+++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportGetRollupJobAction.java
@@ -49,7 +49,6 @@ public TransportGetRollupJobAction(TransportService transportService, ActionFilt
             actionFilters,
             GetRollupJobsAction.Request::new,
             GetRollupJobsAction.Response::new,
-            GetRollupJobsAction.Response::new,
             EsExecutors.DIRECT_EXECUTOR_SERVICE
         );
     }
diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportStartRollupAction.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportStartRollupAction.java
index 81fa7f8199b61..acbe8a0ed7ce1 100644
--- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportStartRollupAction.java
+++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportStartRollupAction.java
@@ -37,7 +37,6 @@ public TransportStartRollupAction(TransportService transportService, ActionFilte
             actionFilters,
             StartRollupJobAction.Request::new,
             StartRollupJobAction.Response::new,
-            StartRollupJobAction.Response::new,
             EsExecutors.DIRECT_EXECUTOR_SERVICE
         );
     }
diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportStopRollupAction.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportStopRollupAction.java
index df87d6e46081b..6b518a68e4936 100644
--- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportStopRollupAction.java
+++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportStopRollupAction.java
@@ -51,7 +51,6 @@ public TransportStopRollupAction(
             actionFilters,
             StopRollupJobAction.Request::new,
             StopRollupJobAction.Response::new,
-            StopRollupJobAction.Response::new,
             EsExecutors.DIRECT_EXECUTOR_SERVICE
         );
         this.threadPool = threadPool;
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportGetTransformStatsAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportGetTransformStatsAction.java
index ce15098775d9a..e4a346833add8 100644
--- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportGetTransformStatsAction.java
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportGetTransformStatsAction.java
@@ -89,7 +89,6 @@ public TransportGetTransformStatsAction(
             actionFilters,
             Request::new,
             Response::new,
-            Response::new,
             EsExecutors.DIRECT_EXECUTOR_SERVICE
         );
         this.transformConfigManager = transformServices.getConfigManager();
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportScheduleNowTransformAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportScheduleNowTransformAction.java
index 8553297ab9cde..ad03632ec0975 100644
--- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportScheduleNowTransformAction.java
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportScheduleNowTransformAction.java
@@ -61,7 +61,6 @@ public TransportScheduleNowTransformAction(
             actionFilters,
             Request::new,
             Response::new,
-            Response::new,
             EsExecutors.DIRECT_EXECUTOR_SERVICE
         );
 
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportStopTransformAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportStopTransformAction.java
index 4f2f60328d738..5ae3ccdbb4354 100644
--- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportStopTransformAction.java
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportStopTransformAction.java
@@ -87,7 +87,6 @@ public TransportStopTransformAction(
             actionFilters,
             Request::new,
             Response::new,
-            Response::new,
             EsExecutors.DIRECT_EXECUTOR_SERVICE
         );
         this.threadPool = threadPool;
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportUpdateTransformAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportUpdateTransformAction.java
index 3fb271aeb1535..e8790407f65a6 100644
--- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportUpdateTransformAction.java
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportUpdateTransformAction.java
@@ -87,7 +87,6 @@ public TransportUpdateTransformAction(
             actionFilters,
             Request::new,
             Response::new,
-            Response::new,
             EsExecutors.DIRECT_EXECUTOR_SERVICE
         );
 

From b412ae630e2d2fdf2306c2611f129a259523e948 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Johannes=20Fred=C3=A9n?=
 <109296772+jfreden@users.noreply.github.com>
Date: Thu, 25 Apr 2024 09:35:56 +0200
Subject: [PATCH 13/58] Fix api key remover unit test (#107711)

---
 .../security/authc/ApiKeyIntegTests.java      | 48 +++++++++++--------
 1 file changed, 29 insertions(+), 19 deletions(-)

diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java
index d787ae45369ec..dc2d4ecc1dd74 100644
--- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java
+++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java
@@ -522,7 +522,6 @@ private void verifyInvalidateResponse(
         assertThat(invalidateResponse.getErrors().size(), equalTo(0));
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/107699")
     public void testApiKeyRemover() throws Exception {
         final String namePrefix = randomAlphaOfLength(10);
         try {
@@ -659,24 +658,6 @@ private void doTestInvalidKeysImmediatelyDeletedByRemover(String namePrefix) thr
         );
     }
 
-    private Client waitForInactiveApiKeysRemoverTriggerReadyAndGetClient() throws Exception {
-        String nodeWithMostRecentRun = null;
-        long apiKeyLastTrigger = -1L;
-        for (String nodeName : internalCluster().getNodeNames()) {
-            ApiKeyService apiKeyService = internalCluster().getInstance(ApiKeyService.class, nodeName);
-            if (apiKeyService != null) {
-                if (apiKeyService.lastTimeWhenApiKeysRemoverWasTriggered() > apiKeyLastTrigger) {
-                    nodeWithMostRecentRun = nodeName;
-                    apiKeyLastTrigger = apiKeyService.lastTimeWhenApiKeysRemoverWasTriggered();
-                }
-            }
-        }
-        final ThreadPool threadPool = internalCluster().getInstance(ThreadPool.class, nodeWithMostRecentRun);
-        final long lastRunTime = apiKeyLastTrigger;
-        assertBusy(() -> { assertThat(threadPool.relativeTimeInMillis() - lastRunTime, greaterThan(DELETE_INTERVAL_MILLIS)); });
-        return internalCluster().client(nodeWithMostRecentRun);
-    }
-
     private void doTestDeletionBehaviorWhenKeysBecomeInvalidBeforeAndAfterRetentionPeriod(String namePrefix) throws Exception {
         assertThat(deleteRetentionPeriodDays, greaterThan(0L));
         Client client = waitForInactiveApiKeysRemoverTriggerReadyAndGetClient().filterWithHeader(
@@ -821,6 +802,35 @@ private void doTestDeletionBehaviorWhenKeysBecomeInvalidBeforeAndAfterRetentionP
         assertThat(getApiKeyResponseListener.get().getApiKeyInfoList().size(), is(4));
     }
 
+    private Client waitForInactiveApiKeysRemoverTriggerReadyAndGetClient() throws Exception {
+        String[] nodeNames = internalCluster().getNodeNames();
+        // Default to first node in list of no remover run detected
+        String nodeWithMostRecentRun = nodeNames[0];
+        final long[] apiKeyRemoverLastTriggerTimestamp = new long[] { -1 };
+
+        for (String nodeName : nodeNames) {
+            ApiKeyService apiKeyService = internalCluster().getInstance(ApiKeyService.class, nodeName);
+            if (apiKeyService != null) {
+                if (apiKeyService.lastTimeWhenApiKeysRemoverWasTriggered() > apiKeyRemoverLastTriggerTimestamp[0]) {
+                    nodeWithMostRecentRun = nodeName;
+                    apiKeyRemoverLastTriggerTimestamp[0] = apiKeyService.lastTimeWhenApiKeysRemoverWasTriggered();
+                }
+            }
+        }
+        final ThreadPool threadPool = internalCluster().getInstance(ThreadPool.class, nodeWithMostRecentRun);
+
+        // If remover didn't run, no need to wait, just return a client
+        if (apiKeyRemoverLastTriggerTimestamp[0] == -1) {
+            return internalCluster().client(nodeWithMostRecentRun);
+        }
+
+        // If remover ran, wait until delete interval has passed to make sure next invalidate will trigger remover
+        assertBusy(
+            () -> assertThat(threadPool.relativeTimeInMillis() - apiKeyRemoverLastTriggerTimestamp[0], greaterThan(DELETE_INTERVAL_MILLIS))
+        );
+        return internalCluster().client(nodeWithMostRecentRun);
+    }
+
     private void refreshSecurityIndex() throws Exception {
         assertBusy(() -> {
             final BroadcastResponse refreshResponse = indicesAdmin().prepareRefresh(SECURITY_MAIN_ALIAS).get();

From a0caf336e6a0dbccf3b64b37f29b41bf02dea41f Mon Sep 17 00:00:00 2001
From: Rene Groeschke <rene@elastic.co>
Date: Thu, 25 Apr 2024 09:38:46 +0200
Subject: [PATCH 14/58] Support mdx file format for docs (#107428)

Adding support for MDX files in our :docs project. We parse those *.mdx files
like we do for asciidoc files for code snippets and generate yaml specs from them that
we test as part of our integration tests.

By default:
When searching for doc sources in the docs folder we fail the build if we detect multiple files of
the same name but different extension. E.g. having painless-field-context.mdx
and painless-field-context.asciidoc in the same source folder will fail the build.

Migration Mode:
To allow easier migration from asciidoc to mdx the build supports a kind of migration mode.
When running the build with -Dgradle.docs.migration=true (e.g. ./gradlew buildRestTests -Dgradle.docs.migration=true)

Duplicate doc source files (asciidoc and mdx) are allowed
The Generated yaml rest specs for duplicates will have the extension *.mdx.yml or *asciidoc.yml.
The generated yaml rest specs for duplicates are compared to each other to ensure they produce the same yml output.
---
 .../doc/DocsTestPluginFuncTest.groovy         |   2 +-
 .../internal/doc/AsciidocSnippetParser.java   | 310 +------
 .../gradle/internal/doc/DocSnippetTask.java   |  23 +-
 .../gradle/internal/doc/DocsTestPlugin.java   |   9 +-
 .../gradle/internal/doc/MdxSnippetParser.java |  80 ++
 .../gradle/internal/doc/ParsingUtils.java     |  14 +-
 .../doc/RestTestsFromDocSnippetTask.java      | 231 +++--
 .../gradle/internal/doc/Snippet.java          | 125 +--
 .../gradle/internal/doc/SnippetBuilder.java   | 273 ++++++
 .../gradle/internal/doc/SnippetParser.java    | 245 ++++-
 .../internal/doc/SnippetParserException.java  |  38 +
 .../doc/AbstractSnippetParserSpec.groovy      | 191 ++++
 .../internal/doc/AsciidocParserSpec.groovy    | 259 ++++--
 .../internal/doc/DocSnippetTaskSpec.groovy    | 565 +-----------
 .../gradle/internal/doc/DocTestUtils.groovy   | 745 +++++++++++++++
 .../internal/doc/MdxSnippetParserSpec.groovy  | 173 ++++
 .../RestTestsFromDocSnippetTaskSpec.groovy    | 869 +++---------------
 .../internal/doc/SnippetBuilderSpec.groovy    | 107 +++
 .../gradle/internal/test/TestUtils.java       |   4 +
 docs/build.gradle                             |  19 +-
 20 files changed, 2398 insertions(+), 1884 deletions(-)
 create mode 100644 build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/MdxSnippetParser.java
 create mode 100644 build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/SnippetBuilder.java
 create mode 100644 build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/SnippetParserException.java
 create mode 100644 build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/doc/AbstractSnippetParserSpec.groovy
 create mode 100644 build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/doc/DocTestUtils.groovy
 create mode 100644 build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/doc/MdxSnippetParserSpec.groovy
 create mode 100644 build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/doc/SnippetBuilderSpec.groovy

diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/doc/DocsTestPluginFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/doc/DocsTestPluginFuncTest.groovy
index 4c542d371c32c..934ff5233ec13 100644
--- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/doc/DocsTestPluginFuncTest.groovy
+++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/doc/DocsTestPluginFuncTest.groovy
@@ -45,7 +45,7 @@ mapper-annotated-text.asciidoc[51:69](console)// TEST[setup:seats]
 """)
     }
 
-    def "can console candidates"() {
+    def "can list console candidates"() {
         when:
         def result = gradleRunner("listConsoleCandidates").build()
         then:
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/AsciidocSnippetParser.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/AsciidocSnippetParser.java
index 7b35fd29fbd1a..f291566d526ff 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/AsciidocSnippetParser.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/AsciidocSnippetParser.java
@@ -8,296 +8,84 @@
 
 package org.elasticsearch.gradle.internal.doc;
 
-import org.gradle.api.InvalidUserDataException;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.util.ArrayList;
-import java.util.Collection;
 import java.util.List;
 import java.util.Map;
-import java.util.function.BiConsumer;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
 
-public class AsciidocSnippetParser implements SnippetParser {
+public class AsciidocSnippetParser extends SnippetParser {
     public static final Pattern SNIPPET_PATTERN = Pattern.compile("-{4,}\\s*");
+    public static final Pattern TEST_RESPONSE_PATTERN = Pattern.compile("\\/\\/\s*TESTRESPONSE(\\[(.+)\\])?\s*");
+    public static final Pattern SOURCE_PATTERN = Pattern.compile(
+        "\\[\"?source\"?(?:\\.[^,]+)?,\\s*\"?([-\\w]+)\"?(,((?!id=).)*(id=\"?([-\\w]+)\"?)?(.*))?].*"
+    );
 
-    private static final String CATCH = "catch:\\s*((?:\\/[^\\/]+\\/)|[^ \\]]+)";
-    private static final String SKIP_REGEX = "skip:([^\\]]+)";
-    private static final String SETUP = "setup:([^ \\]]+)";
-    private static final String TEARDOWN = "teardown:([^ \\]]+)";
-    private static final String WARNING = "warning:(.+)";
-    private static final String NON_JSON = "(non_json)";
-    private static final String SCHAR = "(?:\\\\\\/|[^\\/])";
-    private static final String SUBSTITUTION = "s\\/(" + SCHAR + "+)\\/(" + SCHAR + "*)\\/";
-    private static final String TEST_SYNTAX = "(?:"
-        + CATCH
-        + "|"
-        + SUBSTITUTION
-        + "|"
-        + SKIP_REGEX
-        + "|(continued)|"
-        + SETUP
-        + "|"
-        + TEARDOWN
-        + "|"
-        + WARNING
-        + "|(skip_shard_failures)) ?";
-
-    private final Map<String, String> defaultSubstitutions;
+    public static final String CONSOLE_REGEX = "\\/\\/\s*CONSOLE\s*";
+    public static final String NOTCONSOLE_REGEX = "\\/\\/\s*NOTCONSOLE\s*";
+    public static final String TESTSETUP_REGEX = "\\/\\/\s*TESTSETUP\s*";
+    public static final String TEARDOWN_REGEX = "\\/\\/\s*TEARDOWN\s*";
 
     public AsciidocSnippetParser(Map<String, String> defaultSubstitutions) {
-        this.defaultSubstitutions = defaultSubstitutions;
+        super(defaultSubstitutions);
     }
 
     @Override
-    public List<Snippet> parseDoc(File rootDir, File docFile, List<Map.Entry<String, String>> substitutions) {
-        String lastLanguage = null;
-        Snippet snippet = null;
-        String name = null;
-        int lastLanguageLine = 0;
-        StringBuilder contents = null;
-        List<Snippet> snippets = new ArrayList<>();
+    protected Pattern testResponsePattern() {
+        return TEST_RESPONSE_PATTERN;
+    }
 
-        try (Stream<String> lines = Files.lines(docFile.toPath(), StandardCharsets.UTF_8)) {
-            List<String> linesList = lines.collect(Collectors.toList());
-            for (int lineNumber = 0; lineNumber < linesList.size(); lineNumber++) {
-                String line = linesList.get(lineNumber);
-                if (SNIPPET_PATTERN.matcher(line).matches()) {
-                    if (snippet == null) {
-                        Path path = rootDir.toPath().relativize(docFile.toPath());
-                        snippet = new Snippet(path, lineNumber + 1, name);
-                        snippets.add(snippet);
-                        if (lastLanguageLine == lineNumber - 1) {
-                            snippet.language = lastLanguage;
-                        }
-                        name = null;
-                    } else {
-                        snippet.end = lineNumber + 1;
-                    }
-                    continue;
-                }
+    protected Pattern testPattern() {
+        return Pattern.compile("\\/\\/\s*TEST(\\[(.+)\\])?\s*");
+    }
 
-                Source source = matchSource(line);
-                if (source.matches) {
-                    lastLanguage = source.language;
-                    lastLanguageLine = lineNumber;
-                    name = source.name;
-                    continue;
-                }
-                if (consoleHandled(docFile.getName(), lineNumber, line, snippet)) {
-                    continue;
-                }
-                if (testHandled(docFile.getName(), lineNumber, line, snippet, substitutions)) {
-                    continue;
-                }
-                if (testResponseHandled(docFile.getName(), lineNumber, line, snippet, substitutions)) {
-                    continue;
+    private int lastLanguageLine = 0;
+    private String currentName = null;
+    private String lastLanguage = null;
+
+    protected void parseLine(List<Snippet> snippets, int lineNumber, String line) {
+        if (SNIPPET_PATTERN.matcher(line).matches()) {
+            if (snippetBuilder == null) {
+                snippetBuilder = newSnippetBuilder().withLineNumber(lineNumber + 1)
+                    .withName(currentName)
+                    .withSubstitutions(defaultSubstitutions);
+                if (lastLanguageLine == lineNumber - 1) {
+                    snippetBuilder.withLanguage(lastLanguage);
                 }
-                if (line.matches("\\/\\/\s*TESTSETUP\s*")) {
-                    snippet.testSetup = true;
-                    continue;
-                }
-                if (line.matches("\\/\\/\s*TEARDOWN\s*")) {
-                    snippet.testTearDown = true;
-                    continue;
-                }
-                if (snippet == null) {
-                    // Outside
-                    continue;
-                }
-                if (snippet.end == Snippet.NOT_FINISHED) {
-                    // Inside
-                    if (contents == null) {
-                        contents = new StringBuilder();
-                    }
-                    // We don't need the annotations
-                    line = line.replaceAll("<\\d+>", "");
-                    // Nor any trailing spaces
-                    line = line.replaceAll("\s+$", "");
-                    contents.append(line).append("\n");
-                    continue;
-                }
-                // Allow line continuations for console snippets within lists
-                if (snippet != null && line.trim().equals("+")) {
-                    continue;
-                }
-                finalizeSnippet(snippet, contents.toString(), defaultSubstitutions, substitutions);
-                substitutions = new ArrayList<>();
-                ;
-                snippet = null;
-                contents = null;
-            }
-            if (snippet != null) {
-                finalizeSnippet(snippet, contents.toString(), defaultSubstitutions, substitutions);
-                contents = null;
-                snippet = null;
-                substitutions = new ArrayList<>();
+                currentName = null;
+            } else {
+                snippetBuilder.withEnd(lineNumber + 1);
             }
-        } catch (IOException e) {
-            e.printStackTrace();
+            return;
         }
-        return snippets;
-    }
 
-    static Snippet finalizeSnippet(
-        final Snippet snippet,
-        String contents,
-        Map<String, String> defaultSubstitutions,
-        Collection<Map.Entry<String, String>> substitutions
-    ) {
-        snippet.contents = contents.toString();
-        snippet.validate();
-        escapeSubstitutions(snippet, defaultSubstitutions, substitutions);
-        return snippet;
+        Source source = matchSource(line);
+        if (source.matches) {
+            lastLanguage = source.language;
+            lastLanguageLine = lineNumber;
+            currentName = source.name;
+            return;
+        }
+        handleCommons(snippets, line);
     }
 
-    private static void escapeSubstitutions(
-        Snippet snippet,
-        Map<String, String> defaultSubstitutions,
-        Collection<Map.Entry<String, String>> substitutions
-    ) {
-        BiConsumer<String, String> doSubstitution = (pattern, subst) -> {
-            /*
-             * $body is really common but it looks like a
-             * backreference so we just escape it here to make the
-             * tests cleaner.
-             */
-            subst = subst.replace("$body", "\\$body");
-            subst = subst.replace("$_path", "\\$_path");
-            subst = subst.replace("\\n", "\n");
-            snippet.contents = snippet.contents.replaceAll(pattern, subst);
-        };
-        defaultSubstitutions.forEach(doSubstitution);
-
-        if (substitutions != null) {
-            substitutions.forEach(e -> doSubstitution.accept(e.getKey(), e.getValue()));
-        }
+    protected String getTestSetupRegex() {
+        return TESTSETUP_REGEX;
     }
 
-    private boolean testResponseHandled(
-        String name,
-        int lineNumber,
-        String line,
-        Snippet snippet,
-        final List<Map.Entry<String, String>> substitutions
-    ) {
-        Matcher matcher = Pattern.compile("\\/\\/\s*TESTRESPONSE(\\[(.+)\\])?\s*").matcher(line);
-        if (matcher.matches()) {
-            if (snippet == null) {
-                throw new InvalidUserDataException(name + ":" + lineNumber + ": TESTRESPONSE not paired with a snippet at ");
-            }
-            snippet.testResponse = true;
-            if (matcher.group(2) != null) {
-                String loc = name + ":" + lineNumber;
-                ParsingUtils.parse(
-                    loc,
-                    matcher.group(2),
-                    "(?:" + SUBSTITUTION + "|" + NON_JSON + "|" + SKIP_REGEX + ") ?",
-                    (Matcher m, Boolean last) -> {
-                        if (m.group(1) != null) {
-                            // TESTRESPONSE[s/adsf/jkl/]
-                            substitutions.add(Map.entry(m.group(1), m.group(2)));
-                        } else if (m.group(3) != null) {
-                            // TESTRESPONSE[non_json]
-                            substitutions.add(Map.entry("^", "/"));
-                            substitutions.add(Map.entry("\n$", "\\\\s*/"));
-                            substitutions.add(Map.entry("( +)", "$1\\\\s+"));
-                            substitutions.add(Map.entry("\n", "\\\\s*\n "));
-                        } else if (m.group(4) != null) {
-                            // TESTRESPONSE[skip:reason]
-                            snippet.skip = m.group(4);
-                        }
-                    }
-                );
-            }
-            return true;
-        }
-        return false;
+    protected String getTeardownRegex() {
+        return TEARDOWN_REGEX;
     }
 
-    private boolean testHandled(String name, int lineNumber, String line, Snippet snippet, List<Map.Entry<String, String>> substitutions) {
-        Matcher matcher = Pattern.compile("\\/\\/\s*TEST(\\[(.+)\\])?\s*").matcher(line);
-        if (matcher.matches()) {
-            if (snippet == null) {
-                throw new InvalidUserDataException(name + ":" + lineNumber + ": TEST not paired with a snippet at ");
-            }
-            snippet.test = true;
-            if (matcher.group(2) != null) {
-                String loc = name + ":" + lineNumber;
-                ParsingUtils.parse(loc, matcher.group(2), TEST_SYNTAX, (Matcher m, Boolean last) -> {
-                    if (m.group(1) != null) {
-                        snippet.catchPart = m.group(1);
-                        return;
-                    }
-                    if (m.group(2) != null) {
-                        substitutions.add(Map.entry(m.group(2), m.group(3)));
-                        return;
-                    }
-                    if (m.group(4) != null) {
-                        snippet.skip = m.group(4);
-                        return;
-                    }
-                    if (m.group(5) != null) {
-                        snippet.continued = true;
-                        return;
-                    }
-                    if (m.group(6) != null) {
-                        snippet.setup = m.group(6);
-                        return;
-                    }
-                    if (m.group(7) != null) {
-                        snippet.teardown = m.group(7);
-                        return;
-                    }
-                    if (m.group(8) != null) {
-                        snippet.warnings.add(m.group(8));
-                        return;
-                    }
-                    if (m.group(9) != null) {
-                        snippet.skipShardsFailures = true;
-                        return;
-                    }
-                    throw new InvalidUserDataException("Invalid test marker: " + line);
-                });
-            }
-            return true;
-        }
-        return false;
+    protected String getNotconsoleRegex() {
+        return NOTCONSOLE_REGEX;
     }
 
-    private boolean consoleHandled(String fileName, int lineNumber, String line, Snippet snippet) {
-        if (line.matches("\\/\\/\s*CONSOLE\s*")) {
-            if (snippet == null) {
-                throw new InvalidUserDataException(fileName + ":" + lineNumber + ": CONSOLE not paired with a snippet");
-            }
-            if (snippet.console != null) {
-                throw new InvalidUserDataException(fileName + ":" + lineNumber + ": Can't be both CONSOLE and NOTCONSOLE");
-            }
-            snippet.console = true;
-            return true;
-        } else if (line.matches("\\/\\/\s*NOTCONSOLE\s*")) {
-            if (snippet == null) {
-                throw new InvalidUserDataException(fileName + ":" + lineNumber + ": NOTCONSOLE not paired with a snippet");
-            }
-            if (snippet.console != null) {
-                throw new InvalidUserDataException(fileName + ":" + lineNumber + ": Can't be both CONSOLE and NOTCONSOLE");
-            }
-            snippet.console = false;
-            return true;
-        }
-        return false;
+    protected String getConsoleRegex() {
+        return CONSOLE_REGEX;
     }
 
     static Source matchSource(String line) {
-        Pattern pattern = Pattern.compile("\\[\"?source\"?(?:\\.[^,]+)?,\\s*\"?([-\\w]+)\"?(,((?!id=).)*(id=\"?([-\\w]+)\"?)?(.*))?].*");
-        Matcher matcher = pattern.matcher(line);
+        Matcher matcher = SOURCE_PATTERN.matcher(line);
         if (matcher.matches()) {
             return new Source(true, matcher.group(1), matcher.group(5));
         }
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/DocSnippetTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/DocSnippetTask.java
index 87f0621d53fba..07e3bc93bb6a1 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/DocSnippetTask.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/DocSnippetTask.java
@@ -8,19 +8,17 @@
 
 package org.elasticsearch.gradle.internal.doc;
 
-import org.apache.commons.collections.map.HashedMap;
 import org.gradle.api.Action;
 import org.gradle.api.DefaultTask;
 import org.gradle.api.InvalidUserDataException;
 import org.gradle.api.file.ConfigurableFileTree;
+import org.gradle.api.provider.MapProperty;
 import org.gradle.api.tasks.Input;
 import org.gradle.api.tasks.InputFiles;
 import org.gradle.api.tasks.TaskAction;
 
 import java.io.File;
-import java.util.ArrayList;
 import java.util.List;
-import java.util.Map;
 
 public abstract class DocSnippetTask extends DefaultTask {
 
@@ -36,7 +34,6 @@ public abstract class DocSnippetTask extends DefaultTask {
      * directory.
      */
     private ConfigurableFileTree docs;
-    private Map<String, String> defaultSubstitutions = new HashedMap();
 
     @InputFiles
     public ConfigurableFileTree getDocs() {
@@ -51,36 +48,32 @@ public void setDocs(ConfigurableFileTree docs) {
      * Substitutions done on every snippet's contents.
      */
     @Input
-    public Map<String, String> getDefaultSubstitutions() {
-        return defaultSubstitutions;
-    }
+    abstract MapProperty<String, String> getDefaultSubstitutions();
 
     @TaskAction
     void executeTask() {
         for (File file : docs) {
-            List<Snippet> snippets = parseDocFile(docs.getDir(), file, new ArrayList<>());
+            List<Snippet> snippets = parseDocFile(docs.getDir(), file);
             if (perSnippet != null) {
                 snippets.forEach(perSnippet::execute);
             }
         }
     }
 
-    List<Snippet> parseDocFile(File rootDir, File docFile, List<Map.Entry<String, String>> substitutions) {
+    List<Snippet> parseDocFile(File rootDir, File docFile) {
         SnippetParser parser = parserForFileType(docFile);
-        return parser.parseDoc(rootDir, docFile, substitutions);
+        return parser.parseDoc(rootDir, docFile);
     }
 
     private SnippetParser parserForFileType(File docFile) {
         if (docFile.getName().endsWith(".asciidoc")) {
-            return new AsciidocSnippetParser(defaultSubstitutions);
+            return new AsciidocSnippetParser(getDefaultSubstitutions().get());
+        } else if (docFile.getName().endsWith(".mdx")) {
+            return new MdxSnippetParser(getDefaultSubstitutions().get());
         }
         throw new InvalidUserDataException("Unsupported file type: " + docFile.getName());
     }
 
-    public void setDefaultSubstitutions(Map<String, String> defaultSubstitutions) {
-        this.defaultSubstitutions = defaultSubstitutions;
-    }
-
     public void setPerSnippet(Action<Snippet> perSnippet) {
         this.perSnippet = perSnippet;
     }
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/DocsTestPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/DocsTestPlugin.java
index bbb5102dd6699..2504ea1e74a36 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/DocsTestPlugin.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/DocsTestPlugin.java
@@ -75,14 +75,14 @@ public void apply(Project project) {
         project.getTasks().register("listSnippets", DocSnippetTask.class, task -> {
             task.setGroup("Docs");
             task.setDescription("List each snippet");
-            task.setDefaultSubstitutions(commonDefaultSubstitutions);
-            task.setPerSnippet(snippet -> System.out.println(snippet));
+            task.getDefaultSubstitutions().putAll(commonDefaultSubstitutions);
+            task.setPerSnippet(System.out::println);
         });
 
         project.getTasks().register("listConsoleCandidates", DocSnippetTask.class, task -> {
             task.setGroup("Docs");
             task.setDescription("List snippets that probably should be marked // CONSOLE");
-            task.setDefaultSubstitutions(commonDefaultSubstitutions);
+            task.getDefaultSubstitutions().putAll(commonDefaultSubstitutions);
             task.setPerSnippet(snippet -> {
                 if (snippet.isConsoleCandidate()) {
                     System.out.println(snippet);
@@ -93,8 +93,9 @@ public void apply(Project project) {
         Provider<Directory> restRootDir = projectLayout.getBuildDirectory().dir("rest");
         TaskProvider<RestTestsFromDocSnippetTask> buildRestTests = project.getTasks()
             .register("buildRestTests", RestTestsFromDocSnippetTask.class, task -> {
-                task.setDefaultSubstitutions(commonDefaultSubstitutions);
+                task.getDefaultSubstitutions().putAll(commonDefaultSubstitutions);
                 task.getTestRoot().convention(restRootDir);
+                task.getMigrationMode().set(Boolean.getBoolean("gradle.docs.migration"));
                 task.doFirst(task1 -> fileOperations.delete(restRootDir.get()));
             });
 
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/MdxSnippetParser.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/MdxSnippetParser.java
new file mode 100644
index 0000000000000..0a0bb6328491e
--- /dev/null
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/MdxSnippetParser.java
@@ -0,0 +1,80 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.gradle.internal.doc;
+
+import java.util.List;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class MdxSnippetParser extends SnippetParser {
+
+    public static final Pattern SNIPPET_PATTERN = Pattern.compile("```(.*)");
+
+    public static final Pattern TEST_RESPONSE_PATTERN = Pattern.compile("\\{\\/\\*\s*TESTRESPONSE(\\[(.*)\\])?\s\\*\\/\\}");
+    public static final Pattern TEST_PATTERN = Pattern.compile("\\{\\/\\*\s*TEST(\\[(.*)\\])?\s\\*\\/\\}");
+    public static final String CONSOLE_REGEX = "\\{\\/\\*\s*CONSOLE\s\\*\\/\\}";
+    public static final String NOTCONSOLE_REGEX = "\\{\\/\\*\s*NOTCONSOLE\s\\*\\/\\}";
+    public static final String TESTSETUP_REGEX = "\\{\\/\\*\s*TESTSETUP\s\\*\\/\\}";
+    public static final String TEARDOWN_REGEX = "\\{\\/\\*\s*TEARDOWN\s\\*\\/\\}";
+
+    public MdxSnippetParser(Map<String, String> defaultSubstitutions) {
+        super(defaultSubstitutions);
+    }
+
+    @Override
+    protected void parseLine(List<Snippet> snippets, int lineNumber, String line) {
+        Matcher snippetStartMatcher = SNIPPET_PATTERN.matcher(line);
+        if (snippetStartMatcher.matches()) {
+            if (snippetBuilder == null) {
+                if (snippetStartMatcher.groupCount() == 1) {
+                    String language = snippetStartMatcher.group(1);
+                    snippetBuilder = newSnippetBuilder().withLineNumber(lineNumber + 1)
+                        .withName(null)
+                        .withSubstitutions(defaultSubstitutions)
+                        .withLanguage(language);
+                }
+            } else {
+                snippetBuilder.withEnd(lineNumber + 1);
+            }
+            return;
+        }
+        handleCommons(snippets, line);
+    }
+
+    @Override
+    protected String getTestSetupRegex() {
+        return TESTSETUP_REGEX;
+    }
+
+    @Override
+    protected String getTeardownRegex() {
+        return TEARDOWN_REGEX;
+    }
+
+    @Override
+    protected String getNotconsoleRegex() {
+        return NOTCONSOLE_REGEX;
+    }
+
+    @Override
+    protected String getConsoleRegex() {
+        return CONSOLE_REGEX;
+    }
+
+    @Override
+    protected Pattern testResponsePattern() {
+        return TEST_RESPONSE_PATTERN;
+    }
+
+    @Override
+    protected Pattern testPattern() {
+        return TEST_PATTERN;
+    }
+}
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/ParsingUtils.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/ParsingUtils.java
index b17dd4c7e21d3..53009e1ce5978 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/ParsingUtils.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/ParsingUtils.java
@@ -16,15 +16,13 @@
 
 public class ParsingUtils {
 
-    static void extraContent(String message, String content, int offset, String location, String pattern) {
+    static void extraContent(String message, String content, int offset, String pattern) {
         StringBuilder cutOut = new StringBuilder();
         cutOut.append(content.substring(offset - 6, offset));
         cutOut.append('*');
         cutOut.append(content.substring(offset, Math.min(offset + 5, content.length())));
         String cutOutNoNl = cutOut.toString().replace("\n", "\\n");
-        throw new InvalidUserDataException(
-            location + ": Extra content " + message + " ('" + cutOutNoNl + "') matching [" + pattern + "]: " + content
-        );
+        throw new InvalidUserDataException("Extra content " + message + " ('" + cutOutNoNl + "') matching [" + pattern + "]: " + content);
     }
 
     /**
@@ -33,7 +31,7 @@ static void extraContent(String message, String content, int offset, String loca
      * match then blow up. If the closure takes two parameters then the second
      * one is "is this the last match?".
      */
-    static void parse(String location, String content, String pattern, BiConsumer<Matcher, Boolean> testHandler) {
+    static void parse(String content, String pattern, BiConsumer<Matcher, Boolean> testHandler) {
         if (content == null) {
             return; // Silly null, only real stuff gets to match!
         }
@@ -41,16 +39,16 @@ static void parse(String location, String content, String pattern, BiConsumer<Ma
         int offset = 0;
         while (m.find()) {
             if (m.start() != offset) {
-                extraContent("between [$offset] and [${m.start()}]", content, offset, location, pattern);
+                extraContent("between [$offset] and [${m.start()}]", content, offset, pattern);
             }
             offset = m.end();
             testHandler.accept(m, offset == content.length());
         }
         if (offset == 0) {
-            throw new InvalidUserDataException(location + ": Didn't match " + pattern + ": " + content);
+            throw new InvalidUserDataException("Didn't match " + pattern + ": " + content);
         }
         if (offset != content.length()) {
-            extraContent("after [" + offset + "]", content, offset, location, pattern);
+            extraContent("after [" + offset + "]", content, offset, pattern);
         }
     }
 
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/RestTestsFromDocSnippetTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/RestTestsFromDocSnippetTask.java
index c5b1d67627dd9..28733ff052777 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/RestTestsFromDocSnippetTask.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/RestTestsFromDocSnippetTask.java
@@ -10,10 +10,12 @@
 
 import groovy.transform.PackageScope;
 
+import org.gradle.api.GradleException;
 import org.gradle.api.InvalidUserDataException;
 import org.gradle.api.file.DirectoryProperty;
-import org.gradle.api.internal.file.FileOperations;
-import org.gradle.api.model.ObjectFactory;
+import org.gradle.api.provider.ListProperty;
+import org.gradle.api.provider.MapProperty;
+import org.gradle.api.provider.Property;
 import org.gradle.api.tasks.Input;
 import org.gradle.api.tasks.Internal;
 import org.gradle.api.tasks.OutputDirectory;
@@ -25,8 +27,8 @@
 import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.Collections;
-import java.util.HashMap;
 import java.util.HashSet;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -36,35 +38,32 @@
 
 public abstract class RestTestsFromDocSnippetTask extends DocSnippetTask {
 
-    private Map<String, String> setups = new HashMap<>();
-
-    private Map<String, String> teardowns = new HashMap();
+    /**
+     * For easier migration from asciidoc to mdx we support a migration mode that
+     * allows generation from the same file name but different extensions. The task
+     * will compare the generated tests from the asciidoc and mdx files and fail if
+     * they are not equal (ignoring the line numbers).
+     * */
+    @Input
+    public abstract Property<Boolean> getMigrationMode();
 
     /**
      * Test setups defined in the build instead of the docs so they can be
      * shared between many doc files.
      */
+    private Map<String, String> setups = new LinkedHashMap<>();
+
     @Input
     public Map<String, String> getSetups() {
         return setups;
     }
 
-    public void setSetups(Map<String, String> setups) {
-        this.setups = setups;
-    }
-
     /**
      * Test teardowns defined in the build instead of the docs so they can be
      * shared between many doc files.
      */
     @Input
-    public Map<String, String> getTeardowns() {
-        return teardowns;
-    }
-
-    public void setTeardowns(Map<String, String> teardowns) {
-        this.teardowns = teardowns;
-    }
+    public abstract MapProperty<String, String> getTeardowns();
 
     /**
      * A list of files that contain snippets that *probably* should be
@@ -73,36 +72,8 @@ public void setTeardowns(Map<String, String> teardowns) {
      * If there are unconverted snippets not in this list then this task will
      * fail. All files are paths relative to the docs dir.
      */
-    private List<String> expectedUnconvertedCandidates;
-
     @Input
-    public List<String> getExpectedUnconvertedCandidates() {
-        return expectedUnconvertedCandidates;
-    }
-
-    public void setExpectedUnconvertedCandidates(List<String> expectedUnconvertedCandidates) {
-        this.expectedUnconvertedCandidates = expectedUnconvertedCandidates;
-    }
-
-    /**
-     * Root directory of the tests being generated. To make rest tests happy
-     * we generate them in a testRoot which is contained in this directory.
-     */
-    private DirectoryProperty testRoot;
-
-    private Set<String> names = new HashSet<>();
-
-    @Internal
-    public Set<String> getNames() {
-        return names;
-    }
-
-    public void setNames(Set<String> names) {
-        this.names = names;
-    }
-
-    @Inject
-    public abstract FileOperations getFileOperations();
+    public abstract ListProperty<String> getExpectedUnconvertedCandidates();
 
     /**
      * Root directory containing all the files generated by this task. It is
@@ -110,23 +81,27 @@ public void setNames(Set<String> names) {
      */
     @OutputDirectory
     File getOutputRoot() {
-        return new File(testRoot.get().getAsFile(), "/rest-api-spec/test");
+        return new File(getTestRoot().get().getAsFile(), "/rest-api-spec/test");
     }
 
-    @OutputDirectory
-    DirectoryProperty getTestRoot() {
-        return testRoot;
-    }
+    /**
+     * Root directory of the tests being generated. To make rest tests happy
+     * we generate them in a testRoot which is contained in this directory.
+     */
+    @Internal
+    abstract DirectoryProperty getTestRoot();
 
     @Inject
-    public RestTestsFromDocSnippetTask(ObjectFactory objectFactory) {
-        testRoot = objectFactory.directoryProperty();
+    public RestTestsFromDocSnippetTask() {
         TestBuilder builder = new TestBuilder();
-
-        setPerSnippet(snippet -> builder.handleSnippet(snippet));
+        setPerSnippet(builder::handleSnippet);
+        getMigrationMode().convention(false);
         doLast(task -> {
             builder.finishLastTest();
             builder.checkUnconverted();
+            if (getMigrationMode().get()) {
+                assertEqualTestSnippetFromMigratedDocs();
+            }
         });
     }
 
@@ -223,38 +198,37 @@ private class TestBuilder {
          */
         public void handleSnippet(Snippet snippet) {
             if (snippet.isConsoleCandidate()) {
-                unconvertedCandidates.add(snippet.path.toString().replace('\\', '/'));
+                unconvertedCandidates.add(snippet.path().toString().replace('\\', '/'));
             }
-            if (BAD_LANGUAGES.contains(snippet.language)) {
-                throw new InvalidUserDataException(snippet + ": Use `js` instead of `" + snippet.language + "`.");
+            if (BAD_LANGUAGES.contains(snippet.language())) {
+                throw new InvalidUserDataException(snippet + ": Use `js` instead of `" + snippet.language() + "`.");
             }
-            if (snippet.testSetup) {
+            if (snippet.testSetup()) {
                 testSetup(snippet);
                 previousTest = snippet;
                 return;
             }
-            if (snippet.testTearDown) {
+            if (snippet.testTearDown()) {
                 testTearDown(snippet);
                 previousTest = snippet;
                 return;
             }
-            if (snippet.testResponse || snippet.language.equals("console-result")) {
+            if (snippet.testResponse() || snippet.language().equals("console-result")) {
                 if (previousTest == null) {
                     throw new InvalidUserDataException(snippet + ": No paired previous test");
                 }
-                if (previousTest.path.equals(snippet.path) == false) {
+                if (previousTest.path().equals(snippet.path()) == false) {
                     throw new InvalidUserDataException(snippet + ": Result can't be first in file");
                 }
                 response(snippet);
                 return;
             }
-            if (("js".equals(snippet.language)) && snippet.console != null && snippet.console) {
+            if (("js".equals(snippet.language())) && snippet.console() != null && snippet.console()) {
                 throw new InvalidUserDataException(snippet + ": Use `[source,console]` instead of `// CONSOLE`.");
             }
-            if (snippet.test || snippet.language.equals("console")) {
+            if (snippet.test() || snippet.language().equals("console")) {
                 test(snippet);
                 previousTest = snippet;
-                return;
             }
             // Must be an unmarked snippet....
         }
@@ -262,27 +236,27 @@ public void handleSnippet(Snippet snippet) {
         private void test(Snippet test) {
             setupCurrent(test);
 
-            if (test.continued) {
+            if (test.continued()) {
                 /* Catch some difficult to debug errors with // TEST[continued]
                  * and throw a helpful error message. */
-                if (previousTest == null || previousTest.path.equals(test.path) == false) {
+                if (previousTest == null || previousTest.path().equals(test.path()) == false) {
                     throw new InvalidUserDataException("// TEST[continued] " + "cannot be on first snippet in a file: " + test);
                 }
-                if (previousTest != null && previousTest.testSetup) {
+                if (previousTest != null && previousTest.testSetup()) {
                     throw new InvalidUserDataException("// TEST[continued] " + "cannot immediately follow // TESTSETUP: " + test);
                 }
-                if (previousTest != null && previousTest.testTearDown) {
+                if (previousTest != null && previousTest.testSetup()) {
                     throw new InvalidUserDataException("// TEST[continued] " + "cannot immediately follow // TEARDOWN: " + test);
                 }
             } else {
                 current.println("---");
-                if (test.name != null && test.name.isBlank() == false) {
-                    if (names.add(test.name) == false) {
-                        throw new InvalidUserDataException("Duplicated snippet name '" + test.name + "': " + test);
+                if (test.name() != null && test.name().isBlank() == false) {
+                    if (names.add(test.name()) == false) {
+                        throw new InvalidUserDataException("Duplicated snippet name '" + test.name() + "': " + test);
                     }
-                    current.println("\"" + test.name + "\":");
+                    current.println("\"" + test.name() + "\":");
                 } else {
-                    current.println("\"line_" + test.start + "\":");
+                    current.println("\"line_" + test.start() + "\":");
                 }
                 /* The Elasticsearch test runner doesn't support quite a few
                  * constructs unless we output this skip. We don't know if
@@ -296,36 +270,36 @@ private void test(Snippet test) {
                 current.println("        - stash_path_replace");
                 current.println("        - warnings");
             }
-            if (test.skip != null) {
-                if (test.continued) {
+            if (test.skip() != null) {
+                if (test.continued()) {
                     throw new InvalidUserDataException("Continued snippets " + "can't be skipped");
                 }
                 current.println("        - always_skip");
-                current.println("      reason: " + test.skip);
+                current.println("      reason: " + test.skip());
             }
-            if (test.setup != null) {
+            if (test.setup() != null) {
                 setup(test);
             }
 
             body(test, false);
 
-            if (test.teardown != null) {
+            if (test.teardown() != null) {
                 teardown(test);
             }
         }
 
         private void response(Snippet response) {
-            if (null == response.skip) {
+            if (null == response.skip()) {
                 current.println("  - match:");
                 current.println("      $body:");
-                replaceBlockQuote(response.contents).lines().forEach(line -> current.println("        " + line));
+                replaceBlockQuote(response.contents()).lines().forEach(line -> current.println("        " + line));
             }
         }
 
         private void teardown(final Snippet snippet) {
             // insert a teardown defined outside of the docs
-            for (final String name : snippet.teardown.split(",")) {
-                final String teardown = teardowns.get(name);
+            for (final String name : snippet.teardown().split(",")) {
+                final String teardown = getTeardowns().get().get(name);
                 if (teardown == null) {
                     throw new InvalidUserDataException("Couldn't find named teardown $name for " + snippet);
                 }
@@ -335,7 +309,7 @@ private void teardown(final Snippet snippet) {
         }
 
         private void testTearDown(Snippet snippet) {
-            if (previousTest != null && previousTest.testSetup == false && lastDocsPath == snippet.path) {
+            if (previousTest != null && previousTest.testSetup() == false && lastDocsPath.equals(snippet.path())) {
                 throw new InvalidUserDataException(snippet + " must follow test setup or be first");
             }
             setupCurrent(snippet);
@@ -411,7 +385,7 @@ void emitDo(
         }
 
         private void body(Snippet snippet, boolean inSetup) {
-            ParsingUtils.parse(snippet.getLocation(), snippet.contents, SYNTAX, (matcher, last) -> {
+            ParsingUtils.parse(snippet.contents(), SYNTAX, (matcher, last) -> {
                 if (matcher.group("comment") != null) {
                     // Comment
                     return;
@@ -424,30 +398,43 @@ private void body(Snippet snippet, boolean inSetup) {
                 String method = matcher.group("method");
                 String pathAndQuery = matcher.group("pathAndQuery");
                 String body = matcher.group("body");
-                String catchPart = last ? snippet.catchPart : null;
+                String catchPart = last ? snippet.catchPart() : null;
                 if (pathAndQuery.startsWith("/")) {
                     // Leading '/'s break the generated paths
                     pathAndQuery = pathAndQuery.substring(1);
                 }
-                emitDo(method, pathAndQuery, body, catchPart, snippet.warnings, inSetup, snippet.skipShardsFailures);
+                emitDo(method, pathAndQuery, body, catchPart, snippet.warnings(), inSetup, snippet.skipShardsFailures());
             });
-
         }
 
         private PrintWriter setupCurrent(Snippet test) {
-            if (test.path.equals(lastDocsPath)) {
+            if (test.path().equals(lastDocsPath)) {
                 return current;
             }
             names.clear();
             finishLastTest();
-            lastDocsPath = test.path;
+            lastDocsPath = test.path();
 
             // Make the destination file:
             // Shift the path into the destination directory tree
-            Path dest = getOutputRoot().toPath().resolve(test.path);
+            Path dest = getOutputRoot().toPath().resolve(test.path());
             // Replace the extension
             String fileName = dest.getName(dest.getNameCount() - 1).toString();
-            dest = dest.getParent().resolve(fileName.replace(".asciidoc", ".yml"));
+            if (hasMultipleDocImplementations(test.path())) {
+                String fileNameWithoutExt = dest.getName(dest.getNameCount() - 1).toString().replace(".asciidoc", "").replace(".mdx", "");
+
+                if (getMigrationMode().get() == false) {
+                    throw new InvalidUserDataException(
+                        "Found multiple files with the same name '" + fileNameWithoutExt + "' but different extensions: [asciidoc, mdx]"
+                    );
+                }
+                getLogger().warn("Found multiple doc file types for " + test.path() + ". Generating tests for all of them.");
+                dest = dest.getParent().resolve(fileName + ".yml");
+
+            } else {
+                dest = dest.getParent().resolve(fileName.replace(".asciidoc", ".yml").replace(".mdx", ".yml"));
+
+            }
 
             // Now setup the writer
             try {
@@ -460,7 +447,7 @@ private PrintWriter setupCurrent(Snippet test) {
         }
 
         private void testSetup(Snippet snippet) {
-            if (lastDocsPath == snippet.path) {
+            if (lastDocsPath == snippet.path()) {
                 throw new InvalidUserDataException(
                     snippet + ": wasn't first. TESTSETUP can only be used in the first snippet of a document."
                 );
@@ -468,7 +455,7 @@ private void testSetup(Snippet snippet) {
             setupCurrent(snippet);
             current.println("---");
             current.println("setup:");
-            if (snippet.setup != null) {
+            if (snippet.setup() != null) {
                 setup(snippet);
             }
             body(snippet, true);
@@ -476,8 +463,8 @@ private void testSetup(Snippet snippet) {
 
         private void setup(final Snippet snippet) {
             // insert a setup defined outside of the docs
-            for (final String name : snippet.setup.split(",")) {
-                final String setup = setups.get(name);
+            for (final String name : snippet.setup().split(",")) {
+                final String setup = getSetups().get(name);
                 if (setup == null) {
                     throw new InvalidUserDataException("Couldn't find named setup " + name + " for " + snippet);
                 }
@@ -488,7 +475,7 @@ private void setup(final Snippet snippet) {
 
         public void checkUnconverted() {
             List<String> listedButNotFound = new ArrayList<>();
-            for (String listed : expectedUnconvertedCandidates) {
+            for (String listed : getExpectedUnconvertedCandidates().get()) {
                 if (false == unconvertedCandidates.remove(listed)) {
                     listedButNotFound.add(listed);
                 }
@@ -523,4 +510,54 @@ public void finishLastTest() {
         }
     }
 
+    private void assertEqualTestSnippetFromMigratedDocs() {
+        getTestRoot().getAsFileTree().matching(patternSet -> { patternSet.include("**/*asciidoc.yml"); }).forEach(asciidocFile -> {
+            File mdxFile = new File(asciidocFile.getAbsolutePath().replace(".asciidoc.yml", ".mdx.yml"));
+            if (mdxFile.exists() == false) {
+                throw new InvalidUserDataException("Couldn't find the corresponding mdx file for " + asciidocFile.getAbsolutePath());
+            }
+            try {
+                List<String> asciidocLines = Files.readAllLines(asciidocFile.toPath());
+                List<String> mdxLines = Files.readAllLines(mdxFile.toPath());
+                if (asciidocLines.size() != mdxLines.size()) {
+                    throw new GradleException(
+                        "Yaml rest specs ("
+                            + asciidocFile.toPath()
+                            + " and "
+                            + mdxFile.getAbsolutePath()
+                            + ") are not equal, different line count"
+                    );
+
+                }
+                for (int i = 0; i < asciidocLines.size(); i++) {
+                    if (asciidocLines.get(i)
+                        .replaceAll("line_\\d+", "line_0")
+                        .equals(mdxLines.get(i).replaceAll("line_\\d+", "line_0")) == false) {
+                        throw new GradleException(
+                            "Yaml rest specs ("
+                                + asciidocFile.toPath()
+                                + " and "
+                                + mdxFile.getAbsolutePath()
+                                + ") are not equal, difference on line: "
+                                + (i + 1)
+                        );
+                    }
+                }
+            } catch (IOException e) {
+                throw new RuntimeException(e);
+            }
+        });
+    }
+
+    private boolean hasMultipleDocImplementations(Path path) {
+        File dir = getDocs().getDir();
+        String fileName = path.getName(path.getNameCount() - 1).toString();
+        if (fileName.endsWith("asciidoc")) {
+            return new File(dir, path.toString().replace(".asciidoc", ".mdx")).exists();
+        } else if (fileName.endsWith("mdx")) {
+            return new File(dir, path.toString().replace(".mdx", ".asciidoc")).exists();
+        }
+        return false;
+    }
+
 }
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/Snippet.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/Snippet.java
index b8aa864734f44..227ecbcbfd386 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/Snippet.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/Snippet.java
@@ -8,113 +8,30 @@
 
 package org.elasticsearch.gradle.internal.doc;
 
-import com.fasterxml.jackson.core.JsonFactory;
-import com.fasterxml.jackson.core.JsonParseException;
-import com.fasterxml.jackson.core.JsonParser;
-
-import org.gradle.api.InvalidUserDataException;
-
-import java.io.IOException;
 import java.nio.file.Path;
-import java.util.ArrayList;
 import java.util.List;
 
-public class Snippet {
-    static final int NOT_FINISHED = -1;
-
-    /**
-     * Path to the file containing this snippet. Relative to docs.dir of the
-     * SnippetsTask that created it.
-     */
-    Path path;
-    int start;
-    int end = NOT_FINISHED;
-    public String contents;
-
-    Boolean console = null;
-    boolean test = false;
-    boolean testResponse = false;
-    boolean testSetup = false;
-    boolean testTearDown = false;
-    String skip = null;
-    boolean continued = false;
-    String language = null;
-    String catchPart = null;
-    String setup = null;
-    String teardown = null;
-    boolean curl;
-    List<String> warnings = new ArrayList();
-    boolean skipShardsFailures = false;
-    String name;
-
-    public Snippet(Path path, int start, String name) {
-        this.path = path;
-        this.start = start;
-        this.name = name;
-    }
-
-    public void validate() {
-        if (language == null) {
-            throw new InvalidUserDataException(
-                name
-                    + ": "
-                    + "Snippet missing a language. This is required by "
-                    + "Elasticsearch's doc testing infrastructure so we "
-                    + "be sure we don't accidentally forget to test a "
-                    + "snippet."
-            );
-        }
-        assertValidCurlInput();
-        assertValidJsonInput();
-    }
-
-    String getLocation() {
-        return path + "[" + start + ":" + end + "]";
-    }
-
-    private void assertValidCurlInput() {
-        // Try to detect snippets that contain `curl`
-        if ("sh".equals(language) || "shell".equals(language)) {
-            curl = contents.contains("curl");
-            if (console == Boolean.FALSE && curl == false) {
-                throw new InvalidUserDataException(name + ": " + "No need for NOTCONSOLE if snippet doesn't " + "contain `curl`.");
-            }
-        }
-    }
-
-    private void assertValidJsonInput() {
-        if (testResponse && ("js" == language || "console-result" == language) && null == skip) {
-            String quoted = contents
-                // quote values starting with $
-                .replaceAll("([:,])\\s*(\\$[^ ,\\n}]+)", "$1 \"$2\"")
-                // quote fields starting with $
-                .replaceAll("(\\$[^ ,\\n}]+)\\s*:", "\"$1\":");
-
-            JsonFactory jf = new JsonFactory();
-            jf.configure(JsonParser.Feature.ALLOW_BACKSLASH_ESCAPING_ANY_CHARACTER, true);
-            JsonParser jsonParser;
-
-            try {
-                jsonParser = jf.createParser(quoted);
-                while (jsonParser.isClosed() == false) {
-                    jsonParser.nextToken();
-                }
-            } catch (JsonParseException e) {
-                throw new InvalidUserDataException(
-                    "Invalid json in "
-                        + name
-                        + ". The error is:\n"
-                        + e.getMessage()
-                        + ".\n"
-                        + "After substitutions and munging, the json looks like:\n"
-                        + quoted,
-                    e
-                );
-            } catch (IOException e) {
-                throw new RuntimeException(e);
-            }
-        }
-    }
+public record Snippet(
+    Path path,
+    int start,
+    int end,
+    String contents,
+    Boolean console,
+    boolean test,
+    boolean testResponse,
+    boolean testSetup,
+    boolean testTearDown,
+    String skip,
+    boolean continued,
+    String language,
+    String catchPart,
+    String setup,
+    String teardown,
+    boolean curl,
+    List<String> warnings,
+    boolean skipShardsFailures,
+    String name
+) {
 
     @Override
     public String toString() {
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/SnippetBuilder.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/SnippetBuilder.java
new file mode 100644
index 0000000000000..36d15b9eb33ca
--- /dev/null
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/SnippetBuilder.java
@@ -0,0 +1,273 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.gradle.internal.doc;
+
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonParseException;
+import com.fasterxml.jackson.core.JsonParser;
+
+import org.apache.commons.collections.map.MultiValueMap;
+import org.gradle.api.InvalidUserDataException;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+class SnippetBuilder {
+    static final int NOT_FINISHED = -1;
+
+    private Path path;
+    private int lineNumber;
+    private String name;
+    private String language;
+    private int end = NOT_FINISHED;
+    private boolean testSetup;
+    private boolean testTeardown;
+    // some tests rely on ugly regex substitutions using the same key multiple times
+    private MultiValueMap substitutions = MultiValueMap.decorate(new LinkedHashMap<String, String>());
+    private String catchPart;
+    private boolean test;
+    private String skip;
+    private boolean continued;
+    private String setup;
+    private String teardown;
+    private List<String> warnings = new ArrayList<>();
+    private boolean skipShardsFailures;
+    private boolean testResponse;
+    private boolean curl;
+
+    private StringBuilder contentBuilder = new StringBuilder();
+    private Boolean console = null;
+
+    public SnippetBuilder withPath(Path path) {
+        this.path = path;
+        return this;
+    }
+
+    public SnippetBuilder withLineNumber(int lineNumber) {
+        this.lineNumber = lineNumber;
+        return this;
+    }
+
+    public SnippetBuilder withName(String currentName) {
+        this.name = currentName;
+        return this;
+    }
+
+    public SnippetBuilder withLanguage(String language) {
+        this.language = language;
+        return this;
+    }
+
+    public SnippetBuilder withEnd(int end) {
+        this.end = end;
+        return this;
+    }
+
+    public SnippetBuilder withTestSetup(boolean testSetup) {
+        this.testSetup = testSetup;
+        return this;
+    }
+
+    public SnippetBuilder withTestTearDown(boolean testTeardown) {
+        this.testTeardown = testTeardown;
+        return this;
+    }
+
+    public boolean notFinished() {
+        return end == NOT_FINISHED;
+    }
+
+    public SnippetBuilder withSubstitutions(Map<String, String> substitutions) {
+        this.substitutions.putAll(substitutions);
+        return this;
+    }
+
+    public SnippetBuilder withSubstitution(String key, String value) {
+        this.substitutions.put(key, value);
+        return this;
+    }
+
+    public SnippetBuilder withTest(boolean test) {
+        this.test = test;
+        return this;
+    }
+
+    public SnippetBuilder withCatchPart(String catchPart) {
+        this.catchPart = catchPart;
+        return this;
+    }
+
+    public SnippetBuilder withSkip(String skip) {
+        this.skip = skip;
+        return this;
+    }
+
+    public SnippetBuilder withContinued(boolean continued) {
+        this.continued = continued;
+        return this;
+    }
+
+    public SnippetBuilder withSetup(String setup) {
+        this.setup = setup;
+        return this;
+    }
+
+    public SnippetBuilder withTeardown(String teardown) {
+        this.teardown = teardown;
+        return this;
+    }
+
+    public SnippetBuilder withWarning(String warning) {
+        this.warnings.add(warning);
+        return this;
+    }
+
+    public SnippetBuilder withSkipShardsFailures(boolean skipShardsFailures) {
+        this.skipShardsFailures = skipShardsFailures;
+        return this;
+    }
+
+    public SnippetBuilder withTestResponse(boolean testResponse) {
+        this.testResponse = testResponse;
+        return this;
+    }
+
+    public SnippetBuilder withContent(String content) {
+        return withContent(content, false);
+    }
+
+    public SnippetBuilder withContent(String content, boolean newLine) {
+        contentBuilder.append(content);
+        if (newLine) {
+            contentBuilder.append("\n");
+        }
+        return this;
+    }
+
+    private String escapeSubstitutions(String contents) {
+        Set<Map.Entry<String, List<String>>> set = substitutions.entrySet();
+        for (Map.Entry<String, List<String>> substitution : set) {
+            String pattern = substitution.getKey();
+            for (String subst : substitution.getValue()) {
+                /*
+                 * $body is really common, but it looks like a
+                 * backreference, so we just escape it here to make the
+                 * tests cleaner.
+                 */
+                subst = subst.replace("$body", "\\$body");
+                subst = subst.replace("$_path", "\\$_path");
+                subst = subst.replace("\\n", "\n");
+                contents = contents.replaceAll(pattern, subst);
+            }
+        }
+        return contents;
+    }
+
+    public Snippet build() {
+        String content = contentBuilder.toString();
+        validate(content);
+        String finalContent = escapeSubstitutions(content);
+        return new Snippet(
+            path,
+            lineNumber,
+            end,
+            finalContent,
+            console,
+            test,
+            testResponse,
+            testSetup,
+            testTeardown,
+            skip,
+            continued,
+            language,
+            catchPart,
+            setup,
+            teardown,
+            curl,
+            warnings,
+            skipShardsFailures,
+            name
+        );
+    }
+
+    public void validate(String content) {
+        if (language == null) {
+            throw new InvalidUserDataException(
+                name
+                    + ": "
+                    + "Snippet missing a language. This is required by "
+                    + "Elasticsearch's doc testing infrastructure so we "
+                    + "be sure we don't accidentally forget to test a "
+                    + "snippet."
+            );
+        }
+        assertValidCurlInput(content);
+        assertValidJsonInput(content);
+
+    }
+
+    private void assertValidCurlInput(String content) {
+        // Try to detect snippets that contain `curl`
+        if ("sh".equals(language) || "shell".equals(language)) {
+            curl = content.contains("curl");
+            if (console == Boolean.FALSE && curl == false) {
+                throw new InvalidUserDataException(name + ": " + "No need for NOTCONSOLE if snippet doesn't " + "contain `curl`.");
+            }
+        }
+    }
+
+    private void assertValidJsonInput(String content) {
+        if (testResponse && ("js" == language || "console-result" == language) && null == skip) {
+            String quoted = content
+                // quote values starting with $
+                .replaceAll("([:,])\\s*(\\$[^ ,\\n}]+)", "$1 \"$2\"")
+                // quote fields starting with $
+                .replaceAll("(\\$[^ ,\\n}]+)\\s*:", "\"$1\":");
+
+            JsonFactory jf = new JsonFactory();
+            jf.configure(JsonParser.Feature.ALLOW_BACKSLASH_ESCAPING_ANY_CHARACTER, true);
+            JsonParser jsonParser;
+
+            try {
+                jsonParser = jf.createParser(quoted);
+                while (jsonParser.isClosed() == false) {
+                    jsonParser.nextToken();
+                }
+            } catch (JsonParseException e) {
+                throw new InvalidUserDataException(
+                    "Invalid json in "
+                        + name
+                        + ". The error is:\n"
+                        + e.getMessage()
+                        + ".\n"
+                        + "After substitutions and munging, the json looks like:\n"
+                        + quoted,
+                    e
+                );
+            } catch (IOException e) {
+                throw new RuntimeException(e);
+            }
+        }
+    }
+
+    public SnippetBuilder withConsole(Boolean console) {
+        this.console = console;
+        return this;
+    }
+
+    public boolean consoleDefined() {
+        return console != null;
+
+    }
+}
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/SnippetParser.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/SnippetParser.java
index 064c1c460febf..c4ae0b90127a9 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/SnippetParser.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/SnippetParser.java
@@ -8,10 +8,251 @@
 
 package org.elasticsearch.gradle.internal.doc;
 
+import org.gradle.api.InvalidUserDataException;
+
 import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Stream;
+
+abstract class SnippetParser {
+    protected static final String SCHAR = "(?:\\\\\\/|[^\\/])";
+    protected static final String NON_JSON = "(non_json)";
+    protected static final String SKIP_REGEX = "skip:([^\\]]+)";
+    protected static final String SUBSTITUTION = "s\\/(" + SCHAR + "+)\\/(" + SCHAR + "*)\\/";
+
+    private static final String CATCH = "catch:\\s*((?:\\/[^\\/]+\\/)|[^ \\]]+)";
+    private static final String SETUP = "setup:([^ \\]]+)";
+    private static final String TEARDOWN = "teardown:([^ \\]]+)";
+    private static final String WARNING = "warning:(.+)";
+    private static final String TEST_SYNTAX = "(?:"
+        + CATCH
+        + "|"
+        + SUBSTITUTION
+        + "|"
+        + SKIP_REGEX
+        + "|(continued)|"
+        + SETUP
+        + "|"
+        + TEARDOWN
+        + "|"
+        + WARNING
+        + "|(skip_shard_failures)) ?";
+
+    protected final Map<String, String> defaultSubstitutions;
+
+    protected SnippetBuilder snippetBuilder = null;
+
+    private Path currentPath;
+
+    SnippetParser(Map<String, String> defaultSubstitutions) {
+        this.defaultSubstitutions = defaultSubstitutions;
+    }
+
+    public List<Snippet> parseDoc(File rootDir, File docFile) {
+        List<Snippet> snippets = new ArrayList<>();
+        this.currentPath = rootDir.toPath().relativize(docFile.toPath());
+        try (Stream<String> lines = Files.lines(docFile.toPath(), StandardCharsets.UTF_8)) {
+            List<String> linesList = lines.toList();
+            parseLines(docFile, linesList, snippets);
+        } catch (IOException e) {
+            throw new SnippetParserException("Failed to parse file " + docFile, e);
+        } finally {
+            this.currentPath = null;
+            this.snippetBuilder = null;
+        }
+        return snippets;
+    }
+
+    void parseLines(File file, List<String> linesList, List<Snippet> snippets) {
+        for (int lineNumber = 0; lineNumber < linesList.size(); lineNumber++) {
+            String line = linesList.get(lineNumber);
+            try {
+                parseLine(snippets, lineNumber, line);
+            } catch (InvalidUserDataException e) {
+                throw new SnippetParserException(file, lineNumber, e);
+            }
+        }
+        fileParsingFinished(snippets);
+    }
+
+    protected void handleCommons(List<Snippet> snippets, String line) {
+        if (consoleHandled(line, snippetBuilder)) {
+            return;
+        }
+        if (testHandled(line, snippetBuilder)) {
+            return;
+        }
+        if (testResponseHandled(line, snippetBuilder)) {
+            return;
+        }
+        if (line.matches(getTestSetupRegex())) {
+            snippetBuilder.withTestSetup(true);
+            return;
+        }
+        if (line.matches(getTeardownRegex())) {
+            snippetBuilder.withTestTearDown(true);
+            return;
+        }
+        if (snippetBuilder == null) {
+            // Outside
+            return;
+        }
+        if (snippetBuilder.notFinished()) {
+            // Inside
+            // We don't need the annotations
+            line = line.replaceAll("<\\d+>", "");
+            // nor bookmarks
+            line = line.replaceAll("\\[\\^\\d+\\]", "");
+            // Nor any trailing spaces
+            line = line.replaceAll("\s+$", "");
+            snippetBuilder.withContent(line, true);
+            return;
+        }
+        // Allow line continuations for console snippets within lists
+        if (snippetBuilder != null && line.trim().equals("+")) {
+            return;
+        }
+        snippets.add(snippetBuilder.build());
+        snippetBuilder = null;
+    }
+
+    protected SnippetBuilder newSnippetBuilder() {
+        snippetBuilder = new SnippetBuilder().withPath(currentPath);
+        return snippetBuilder;
+    }
+
+    void fileParsingFinished(List<Snippet> snippets) {
+        if (snippetBuilder != null) {
+            snippets.add(snippetBuilder.build());
+            snippetBuilder = null;
+        }
+    }
+
+    protected abstract void parseLine(List<Snippet> snippets, int lineNumber, String line);
+
+    boolean testResponseHandled(String line, SnippetBuilder snippetBuilder) {
+        Matcher matcher = testResponsePattern().matcher(line);
+        if (matcher.matches()) {
+            if (snippetBuilder == null) {
+                throw new InvalidUserDataException("TESTRESPONSE not paired with a snippet at ");
+            }
+            snippetBuilder.withTestResponse(true);
+            if (matcher.group(2) != null) {
+                ParsingUtils.parse(
+                    matcher.group(2),
+                    "(?:" + SUBSTITUTION + "|" + NON_JSON + "|" + SKIP_REGEX + ") ?",
+                    (Matcher m, Boolean last) -> {
+                        if (m.group(1) != null) {
+                            // TESTRESPONSE[s/adsf/jkl/]
+                            snippetBuilder.withSubstitution(m.group(1), m.group(2));
+                        } else if (m.group(3) != null) {
+                            // TESTRESPONSE[non_json]
+                            snippetBuilder.withSubstitution("^", "/");
+                            snippetBuilder.withSubstitution("\n$", "\\\\s*/");
+                            snippetBuilder.withSubstitution("( +)", "$1\\\\s+");
+                            snippetBuilder.withSubstitution("\n", "\\\\s*\n ");
+                        } else if (m.group(4) != null) {
+                            // TESTRESPONSE[skip:reason]
+                            snippetBuilder.withSkip(m.group(4));
+                        }
+                    }
+                );
+            }
+            return true;
+        }
+        return false;
+    }
+
+    protected boolean testHandled(String line, SnippetBuilder snippetBuilder) {
+        Matcher matcher = testPattern().matcher(line);
+        if (matcher.matches()) {
+            if (snippetBuilder == null) {
+                throw new InvalidUserDataException("TEST not paired with a snippet at ");
+            }
+            snippetBuilder.withTest(true);
+            if (matcher.group(2) != null) {
+                ParsingUtils.parse(matcher.group(2), TEST_SYNTAX, (Matcher m, Boolean last) -> {
+                    if (m.group(1) != null) {
+                        snippetBuilder.withCatchPart(m.group(1));
+                        return;
+                    }
+                    if (m.group(2) != null) {
+                        snippetBuilder.withSubstitution(m.group(2), m.group(3));
+                        return;
+                    }
+                    if (m.group(4) != null) {
+                        snippetBuilder.withSkip(m.group(4));
+                        return;
+                    }
+                    if (m.group(5) != null) {
+                        snippetBuilder.withContinued(true);
+                        return;
+                    }
+                    if (m.group(6) != null) {
+                        snippetBuilder.withSetup(m.group(6));
+                        return;
+                    }
+                    if (m.group(7) != null) {
+                        snippetBuilder.withTeardown(m.group(7));
+                        return;
+                    }
+                    if (m.group(8) != null) {
+                        snippetBuilder.withWarning(m.group(8));
+                        return;
+                    }
+                    if (m.group(9) != null) {
+                        snippetBuilder.withSkipShardsFailures(true);
+                        return;
+                    }
+                    throw new InvalidUserDataException("Invalid test marker: " + line);
+                });
+            }
+            return true;
+        }
+        return false;
+    }
+
+    protected boolean consoleHandled(String line, SnippetBuilder snippet) {
+        if (line.matches(getConsoleRegex())) {
+            if (snippetBuilder == null) {
+                throw new InvalidUserDataException("CONSOLE not paired with a snippet");
+            }
+            if (snippetBuilder.consoleDefined()) {
+                throw new InvalidUserDataException("Can't be both CONSOLE and NOTCONSOLE");
+            }
+            snippetBuilder.withConsole(Boolean.TRUE);
+            return true;
+        } else if (line.matches(getNotconsoleRegex())) {
+            if (snippet == null) {
+                throw new InvalidUserDataException("NOTCONSOLE not paired with a snippet");
+            }
+            if (snippetBuilder.consoleDefined()) {
+                throw new InvalidUserDataException("Can't be both CONSOLE and NOTCONSOLE");
+            }
+            snippet.withConsole(Boolean.FALSE);
+            return true;
+        }
+        return false;
+    }
+
+    protected abstract String getTestSetupRegex();
+
+    protected abstract String getTeardownRegex();
+
+    protected abstract String getConsoleRegex();
+
+    protected abstract String getNotconsoleRegex();
+
+    protected abstract Pattern testPattern();
+
+    protected abstract Pattern testResponsePattern();
 
-public interface SnippetParser {
-    List<Snippet> parseDoc(File rootDir, File docFile, List<Map.Entry<String, String>> substitutions);
 }
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/SnippetParserException.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/SnippetParserException.java
new file mode 100644
index 0000000000000..79563a97de119
--- /dev/null
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/SnippetParserException.java
@@ -0,0 +1,38 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.gradle.internal.doc;
+
+import org.gradle.api.InvalidUserDataException;
+
+import java.io.File;
+
+public class SnippetParserException extends RuntimeException {
+    private final File file;
+    private final int lineNumber;
+
+    public SnippetParserException(String message, Throwable cause) {
+        super(message, cause);
+        this.file = null;
+        this.lineNumber = -1;
+    }
+
+    public SnippetParserException(File file, int lineNumber, InvalidUserDataException e) {
+        super("Error parsing snippet in " + file.getName() + " at line " + lineNumber, e);
+        this.file = file;
+        this.lineNumber = lineNumber;
+    }
+
+    public File getFile() {
+        return file;
+    }
+
+    public int getLineNumber() {
+        return lineNumber;
+    }
+}
diff --git a/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/doc/AbstractSnippetParserSpec.groovy b/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/doc/AbstractSnippetParserSpec.groovy
new file mode 100644
index 0000000000000..8690c738f0d95
--- /dev/null
+++ b/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/doc/AbstractSnippetParserSpec.groovy
@@ -0,0 +1,191 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.gradle.internal.doc
+
+import spock.lang.Specification
+
+import org.gradle.api.InvalidUserDataException
+
+import java.nio.file.Path
+
+abstract class AbstractSnippetParserSpec extends Specification {
+
+    abstract SnippetParser parser()
+    abstract String docSnippetWithTestResponses()
+    abstract String docSnippetWithTest()
+    abstract String docSnippetWithRepetitiveSubstiutions()
+    abstract String docSnippetWithConsole()
+    abstract String docSnippetWithNotConsole()
+    abstract String docSnippetWithMixedConsoleNotConsole()
+
+    def "can parse snippet with console"() {
+        when:
+        def snippets = parse(docSnippetWithConsole())
+        then:
+        snippets*.console() == [true]
+    }
+
+    def "can parse snippet with notconsole"() {
+        when:
+        def snippets = parse(docSnippetWithNotConsole())
+        then:
+        snippets*.console() == [false]
+    }
+
+    def "fails on mixing console and notconsole"() {
+        when:
+        def snippets = parse(docSnippetWithMixedConsoleNotConsole())
+        then:
+        def e = thrown(SnippetParserException)
+        e.message.matches("Error parsing snippet in acme.xyz at line \\d")
+        e.file.name == "acme.xyz"
+        e.lineNumber > 0
+    }
+
+    def "can parse snippet with test"() {
+        when:
+        def snippets = parse(docSnippetWithTest())
+        then:
+        snippets*.test() == [true]
+        snippets*.testResponse() == [false]
+        snippets*.language() == ["console"]
+        snippets*.catchPart() == ["/painless_explain_error/"]
+        snippets*.teardown() == ["some_teardown"]
+        snippets*.setup() == ["seats"]
+        snippets*.warnings() == [["some_warning"]]
+        snippets*.contents() == ["""PUT /hockey/_doc/1?refresh
+{"first":"johnny","last":"gaudreau","goals":[9,27,1],"assists":[17,46,0],"gp":[26,82,1]}
+
+POST /hockey/_explain/1?error_trace=false
+{
+  "query": {
+    "script": {
+      "script": "Debug.explain(doc.goals)"
+    }
+  }
+}
+"""]
+    }
+
+    def "can parse snippet with test responses"() {
+        when:
+        def snippets = parse(docSnippetWithTestResponses())
+        then:
+        snippets*.testResponse() == [true]
+        snippets*.test() == [false]
+        snippets*.language() == ["console-result"]
+        snippets*.skip() == ["some_skip_message"]
+        snippets*.contents() == ["""{
+  "docs" : [
+    {
+      "processor_results" : [
+        {
+          "processor_type" : "set",
+          "status" : "success",
+          "doc" : {
+            "_index" : "index",
+            "_id" : "id",
+            "_version": "-3",
+            "_source" : {
+              "field2" : "_value2",
+              "foo" : "bar"
+            },
+            "_ingest" : {
+              "pipeline" : "_simulate_pipeline",
+              "timestamp" : \$body.docs.0.processor_results.0.doc._ingest.timestamp
+            }
+          }
+        },
+        {
+          "processor_type" : "set",
+          "status" : "success",
+          "doc" : {
+            "_index" : "index",
+            "_id" : "id",
+            "_version": "-3",
+            "_source" : {
+              "field3" : "_value3",
+              "field2" : "_value2",
+              "foo" : "bar"
+            },
+            "_ingest" : {
+              "pipeline" : "_simulate_pipeline",
+              "timestamp" : \$body.docs.0.processor_results.0.doc._ingest.timestamp
+            }
+          }
+        }
+      ]
+    },
+    {
+      "processor_results" : [
+        {
+          "processor_type" : "set",
+          "status" : "success",
+          "doc" : {
+            "_index" : "index",
+            "_id" : "id",
+            "_version": "-3",
+            "_source" : {
+              "field2" : "_value2",
+              "foo" : "rab"
+            },
+            "_ingest" : {
+              "pipeline" : "_simulate_pipeline",
+              "timestamp" : \$body.docs.1.processor_results.0.doc._ingest.timestamp
+            }
+          }
+        },
+        {
+          "processor_type" : "set",
+          "status" : "success",
+          "doc" : {
+            "_index" : "index",
+            "_id" : "id",
+            "_version": "-3",
+            "_source" : {
+              "field3" : "_value3",
+              "field2" : "_value2",
+              "foo" : "rab"
+            },
+            "_ingest" : {
+              "pipeline" : "_simulate_pipeline",
+              "timestamp" : \$body.docs.1.processor_results.0.doc._ingest.timestamp
+            }
+          }
+        }
+      ]
+    }
+  ]
+}
+"""]
+    }
+
+    def "can parse snippet with repetitive regex substitutions"() {
+        when:
+        def snippets = parse(docSnippetWithRepetitiveSubstiutions())
+        then:
+        snippets*.test() == [true]
+        snippets*.testResponse() == [false]
+        snippets*.language() == ["console"]
+        snippets*.contents() == ["""PUT /_snapshot/repo1
+{"type": "fs", "settings": {"location": "repo/1"}}
+PUT /_snapshot/repo1/snap2?wait_for_completion=true
+PUT /_snapshot/repo1/snap1?wait_for_completion=true
+GET /_cat/snapshots/repo1?v=true&s=id
+"""]
+    }
+
+    List<Snippet> parse(String docSnippet) {
+        List<Snippet> snippets = new ArrayList<>()
+        def lines = docSnippet.lines().toList()
+        parser().parseLines(new File("acme.xyz"), lines, snippets)
+        return snippets
+    }
+
+}
diff --git a/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/doc/AsciidocParserSpec.groovy b/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/doc/AsciidocParserSpec.groovy
index b7ac363ef7ad3..a80215cd82f0d 100644
--- a/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/doc/AsciidocParserSpec.groovy
+++ b/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/doc/AsciidocParserSpec.groovy
@@ -6,17 +6,11 @@
  * Side Public License, v 1.
  */
 
-package org.elasticsearch.gradle.internal.doc;
+package org.elasticsearch.gradle.internal.doc
 
-import spock.lang.Specification
-import spock.lang.Unroll
+import static org.elasticsearch.gradle.internal.doc.AsciidocSnippetParser.matchSource
 
-import org.gradle.api.InvalidUserDataException
-
-import static org.elasticsearch.gradle.internal.doc.AsciidocSnippetParser.finalizeSnippet;
-import static org.elasticsearch.gradle.internal.doc.AsciidocSnippetParser.matchSource;
-
-class AsciidocParserSpec extends Specification {
+class AsciidocParserSpec extends AbstractSnippetParserSpec {
 
     def testMatchSource() {
         expect:
@@ -78,107 +72,174 @@ class AsciidocParserSpec extends Specification {
         }
     }
 
-    @Unroll
-    def "checks for valid json for #languageParam"() {
-        given:
-        def snippet = snippet() {
-            language = languageParam
-            testResponse = true
-        }
-        def json = """{
-    "name": "John Doe",
-    "age": 30,
-    "isMarried": true,
-    "address": {
-        "street": "123 Main Street",
-        "city": "Springfield",
-        "state": "IL",
-        "zip": "62701"
-    },
-    "hobbies": ["Reading", "Cooking", "Traveling"]
-}"""
-        when:
-        def result = finalizeSnippet(snippet, json, [:], [:].entrySet())
-        then:
-        result != null
-
-        when:
-        finalizeSnippet(snippet, "some no valid json", [:], [:].entrySet())
-        then:
-        def e = thrown(InvalidUserDataException)
-        e.message.contains("Invalid json in")
-
-        when:
-        snippet.skip = "true"
-        result = finalizeSnippet(snippet, "some no valid json", [:], [:].entrySet())
-        then:
-        result != null
-
-        where:
-        languageParam << ["js", "console-result"]
+    @Override
+    SnippetParser parser() {
+        return new AsciidocSnippetParser([:]);
     }
 
-    def "test finalized snippet handles substitutions"() {
-        given:
-        def snippet = snippet() {
-            language = "console"
-        }
-        when:
-        finalizeSnippet(snippet, "snippet-content substDefault subst", [substDefault: "\$body"], [subst: 'substValue'].entrySet())
-        then:
-        snippet.contents == "snippet-content \$body substValue"
+    @Override
+    String docSnippetWithTest() {
+        return """[source,console]
+---------------------------------------------------------
+PUT /hockey/_doc/1?refresh
+{"first":"johnny","last":"gaudreau","goals":[9,27,1],"assists":[17,46,0],"gp":[26,82,1]}
+
+POST /hockey/_explain/1
+{
+  "query": {
+    "script": {
+      "script": "Debug.explain(doc.goals)"
+    }
+  }
+}
+---------------------------------------------------------
+// TEST[s/_explain\\/1/_explain\\/1?error_trace=false/ catch:/painless_explain_error/]
+// TEST[teardown:some_teardown]
+// TEST[setup:seats]
+// TEST[warning:some_warning]
+// TEST[skip_shard_failures]
+
+"""
     }
 
-    def snippetMustHaveLanguage() {
-        given:
-        def snippet = snippet()
-        when:
-        finalizeSnippet(snippet, "snippet-content", [:], [])
-        then:
-        def e = thrown(InvalidUserDataException)
-        e.message.contains("Snippet missing a language.")
+    @Override
+    String docSnippetWithRepetitiveSubstiutions() {
+        return """
+[source,console]
+--------------------------------------------------
+GET /_cat/snapshots/repo1?v=true&s=id
+--------------------------------------------------
+// TEST[s/^/PUT \\/_snapshot\\/repo1\\/snap1?wait_for_completion=true\\n/]
+// TEST[s/^/PUT \\/_snapshot\\/repo1\\/snap2?wait_for_completion=true\\n/]
+// TEST[s/^/PUT \\/_snapshot\\/repo1\\n{"type": "fs", "settings": {"location": "repo\\/1"}}\\n/]
+"""
     }
 
-    def testEmit() {
-        given:
-        def snippet = snippet() {
-            language = "console"
-        }
-        when:
-        finalizeSnippet(snippet, "snippet-content", [:], [])
-        then:
-        snippet.contents == "snippet-content"
+    @Override
+    String docSnippetWithConsole() {
+        return """
+[source,console]
+----
+// CONSOLE
+----
+"""
     }
 
-    def testSnippetsWithCurl() {
-        given:
-        def snippet = snippet() {
-            language = "sh"
-            name = "snippet-name-1"
-        }
-        when:
-        finalizeSnippet(snippet, "curl substDefault subst", [:], [:].entrySet())
-        then:
-        snippet.curl == true
+    @Override
+    String docSnippetWithNotConsole() {
+        return """
+[source,console]
+----
+// NOTCONSOLE
+----
+"""
     }
 
-    def "test snippets with no curl no console"() {
-        given:
-        def snippet = snippet() {
-            console = false
-            language = "shell"
-        }
-        when:
-        finalizeSnippet(snippet, "hello substDefault subst", [:], [:].entrySet())
-        then:
-        def e = thrown(InvalidUserDataException)
-        e.message.contains("No need for NOTCONSOLE if snippet doesn't contain `curl`")
+    @Override
+    String docSnippetWithMixedConsoleNotConsole() {
+        return """
+[source,console]
+----
+// NOTCONSOLE
+// CONSOLE
+----
+"""
     }
 
-    Snippet snippet(Closure<DocSnippetTask> configClosure = {}) {
-        def snippet = new Snippet(new File("SomePath").toPath(), 0, "snippet-name-1")
-        configClosure.delegate = snippet
-        configClosure()
-        return snippet
+    @Override
+    String docSnippetWithTestResponses() {
+        return """
+[source,console-result]
+----
+{
+  "docs" : [
+    {
+      "processor_results" : [
+        {
+          "processor_type" : "set",
+          "status" : "success",
+          "doc" : {
+            "_index" : "index",
+            "_id" : "id",
+            "_version": "-3",
+            "_source" : {
+              "field2" : "_value2",
+              "foo" : "bar"
+            },
+            "_ingest" : {
+              "pipeline" : "_simulate_pipeline",
+              "timestamp" : "2020-07-30T01:21:24.251836Z"
+            }
+          }
+        },
+        {
+          "processor_type" : "set",
+          "status" : "success",
+          "doc" : {
+            "_index" : "index",
+            "_id" : "id",
+            "_version": "-3",
+            "_source" : {
+              "field3" : "_value3",
+              "field2" : "_value2",
+              "foo" : "bar"
+            },
+            "_ingest" : {
+              "pipeline" : "_simulate_pipeline",
+              "timestamp" : "2020-07-30T01:21:24.251836Z"
+            }
+          }
+        }
+      ]
+    },
+    {
+      "processor_results" : [
+        {
+          "processor_type" : "set",
+          "status" : "success",
+          "doc" : {
+            "_index" : "index",
+            "_id" : "id",
+            "_version": "-3",
+            "_source" : {
+              "field2" : "_value2",
+              "foo" : "rab"
+            },
+            "_ingest" : {
+              "pipeline" : "_simulate_pipeline",
+              "timestamp" : "2020-07-30T01:21:24.251863Z"
+            }
+          }
+        },
+        {
+          "processor_type" : "set",
+          "status" : "success",
+          "doc" : {
+            "_index" : "index",
+            "_id" : "id",
+            "_version": "-3",
+            "_source" : {
+              "field3" : "_value3",
+              "field2" : "_value2",
+              "foo" : "rab"
+            },
+            "_ingest" : {
+              "pipeline" : "_simulate_pipeline",
+              "timestamp" : "2020-07-30T01:21:24.251863Z"
+            }
+          }
+        }
+      ]
+    }
+  ]
+}
+----
+// TESTRESPONSE[s/"2020-07-30T01:21:24.251836Z"/\$body.docs.0.processor_results.0.doc._ingest.timestamp/]
+// TESTRESPONSE[s/"2020-07-30T01:21:24.251836Z"/\$body.docs.0.processor_results.1.doc._ingest.timestamp/]
+// TESTRESPONSE[s/"2020-07-30T01:21:24.251863Z"/\$body.docs.1.processor_results.0.doc._ingest.timestamp/]
+// TESTRESPONSE[s/"2020-07-30T01:21:24.251863Z"/\$body.docs.1.processor_results.1.doc._ingest.timestamp/]
+// TESTRESPONSE[skip:some_skip_message]
+"""
     }
+
 }
diff --git a/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/doc/DocSnippetTaskSpec.groovy b/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/doc/DocSnippetTaskSpec.groovy
index 894e6e9b51ab8..2b6582bd633ef 100644
--- a/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/doc/DocSnippetTaskSpec.groovy
+++ b/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/doc/DocSnippetTaskSpec.groovy
@@ -10,8 +10,8 @@ package org.elasticsearch.gradle.internal.doc
 
 import spock.lang.Specification
 import spock.lang.TempDir
+import spock.lang.Unroll
 
-import org.gradle.api.InvalidUserDataException
 import org.gradle.testfixtures.ProjectBuilder
 
 import static org.elasticsearch.gradle.internal.test.TestUtils.normalizeString
@@ -21,42 +21,27 @@ class DocSnippetTaskSpec extends Specification {
     @TempDir
     File tempDir
 
-    def "handling test parsing multiple snippets per file"() {
-        given:
-        def project = ProjectBuilder.builder().build()
-        def task = project.tasks.register("docSnippetTask", DocSnippetTask).get()
+    @Unroll
+    def "handling test parsing multiple snippets per #fileType file"() {
         when:
-        def substitutions = []
-        def snippets = task.parseDocFile(
-            tempDir, docFile(
-            """
-[[mapper-annotated-text]]
-=== Mapper annotated text plugin
+        def snippets = parseFile("example-1.$fileType")
 
-experimental[]
-
-The mapper-annotated-text plugin provides the ability to index text that is a
-combination of free-text and special markup that is typically used to identify
-items of interest such as people or organisations (see NER or Named Entity Recognition
-tools).
-
-
-The elasticsearch markup allows one or more additional tokens to be injected, unchanged, into the token
-stream at the same position as the underlying text it annotates.
-
-:plugin_name: mapper-annotated-text
-include::install_remove.asciidoc[]
-
-[[mapper-annotated-text-usage]]
-==== Using the `annotated-text` field
-
-The `annotated-text` tokenizes text content as per the more common {ref}/text.html[`text`] field (see
-"limitations" below) but also injects any marked-up annotation tokens directly into
-the search index:
-
-[source,console]
---------------------------
-PUT my-index-000001
+        then:
+        snippets*.test == [false, false, false, false, false, false, false]
+        snippets*.catchPart == [null, null, null, null, null, null, null]
+        snippets*.setup == [null, null, null, null, null, null, null]
+        snippets*.teardown == [null, null, null, null, null, null, null]
+        snippets*.testResponse == [false, false, false, false, false, false, false]
+        snippets*.skip == [null, null, null, null, null, null, null]
+        snippets*.continued == [false, false, false, false, false, false, false]
+        snippets*.language == ["console", "js", "js", "console", "console", "console", "console"]
+        snippets*.contents*.empty == [false, false, false, false, false, false, false]
+        snippets*.start == expectedSnippetStarts
+        snippets*.end == expectedSnippetEnds
+
+        // test two snippet explicitly for content.
+        // More coverage on actual parsing is done in unit tests
+        normalizeString(snippets[0].contents) == """PUT my-index-000001
 {
   "mappings": {
     "properties": {
@@ -65,515 +50,31 @@ PUT my-index-000001
       }
     }
   }
-}
---------------------------
-
-Such a mapping would allow marked-up text eg wikipedia articles to be indexed as both text
-and structured tokens. The annotations use a markdown-like syntax using URL encoding of
-one or more values separated by the `&` symbol.
-
-
-We can use the "_analyze" api to test how an example annotation would be stored as tokens
-in the search index:
-
+}"""
 
-[source,js]
---------------------------
-GET my-index-000001/_analyze
+        normalizeString(snippets[1].contents) == """GET my-index-000001/_analyze
 {
   "field": "my_field",
   "text":"Investors in [Apple](Apple+Inc.) rejoiced."
-}
---------------------------
-// NOTCONSOLE
-
-Response:
-
-[source,js]
---------------------------------------------------
-{
-  "tokens": [
-    {
-      "token": "investors",
-      "start_offset": 0,
-      "end_offset": 9,
-      "type": "<ALPHANUM>",
-      "position": 0
-    },
-    {
-      "token": "in",
-      "start_offset": 10,
-      "end_offset": 12,
-      "type": "<ALPHANUM>",
-      "position": 1
-    },
-    {
-      "token": "Apple Inc.", <1>
-      "start_offset": 13,
-      "end_offset": 18,
-      "type": "annotation",
-      "position": 2
-    },
-    {
-      "token": "apple",
-      "start_offset": 13,
-      "end_offset": 18,
-      "type": "<ALPHANUM>",
-      "position": 2
-    },
-    {
-      "token": "rejoiced",
-      "start_offset": 19,
-      "end_offset": 27,
-      "type": "<ALPHANUM>",
-      "position": 3
-    }
-  ]
-}
---------------------------------------------------
-// NOTCONSOLE
-
-<1> Note the whole annotation token `Apple Inc.` is placed, unchanged as a single token in
-the token stream and at the same position (position 2) as the text token (`apple`) it annotates.
-
-
-We can now perform searches for annotations using regular `term` queries that don't tokenize
-the provided search values. Annotations are a more precise way of matching as can be seen
-in this example where a search for `Beck` will not match `Jeff Beck` :
-
-[source,console]
---------------------------
-# Example documents
-PUT my-index-000001/_doc/1
-{
-  "my_field": "[Beck](Beck) announced a new tour"<1>
-}
-
-PUT my-index-000001/_doc/2
-{
-  "my_field": "[Jeff Beck](Jeff+Beck&Guitarist) plays a strat"<2>
-}
-
-# Example search
-GET my-index-000001/_search
-{
-  "query": {
-    "term": {
-        "my_field": "Beck" <3>
-    }
-  }
-}
---------------------------
-
-<1> As well as tokenising the plain text into single words e.g. `beck`, here we
-inject the single token value `Beck` at the same position as `beck` in the token stream.
-<2> Note annotations can inject multiple tokens at the same position - here we inject both
-the very specific value `Jeff Beck` and the broader term `Guitarist`. This enables
-broader positional queries e.g. finding mentions of a `Guitarist` near to `strat`.
-<3> A benefit of searching with these carefully defined annotation tokens is that a query for
-`Beck` will not match document 2 that contains the tokens `jeff`, `beck` and `Jeff Beck`
-
-WARNING: Any use of `=` signs in annotation values eg `[Prince](person=Prince)` will
-cause the document to be rejected with a parse failure. In future we hope to have a use for
-the equals signs so wil actively reject documents that contain this today.
-
-
-[[mapper-annotated-text-tips]]
-==== Data modelling tips
-===== Use structured and unstructured fields
-
-Annotations are normally a way of weaving structured information into unstructured text for
-higher-precision search.
-
-`Entity resolution` is a form of document enrichment undertaken by specialist software or people
-where references to entities in a document are disambiguated by attaching a canonical ID.
-The ID is used to resolve any number of aliases or distinguish between people with the
-same name. The hyperlinks connecting Wikipedia's articles are a good example of resolved
-entity IDs woven into text.
-
-These IDs can be embedded as annotations in an annotated_text field but it often makes
-sense to include them in dedicated structured fields to support discovery via aggregations:
-
-[source,console]
---------------------------
-PUT my-index-000001
-{
-  "mappings": {
-    "properties": {
-      "my_unstructured_text_field": {
-        "type": "annotated_text"
-      },
-      "my_structured_people_field": {
-        "type": "text",
-        "fields": {
-          "keyword" : {
-            "type": "keyword"
-          }
-        }
-      }
-    }
-  }
-}
---------------------------
-
-Applications would then typically provide content and discover it as follows:
-
-[source,console]
---------------------------
-# Example documents
-PUT my-index-000001/_doc/1
-{
-  "my_unstructured_text_field": "[Shay](%40kimchy) created elasticsearch",
-  "my_twitter_handles": ["@kimchy"] <1>
-}
-
-GET my-index-000001/_search
-{
-  "query": {
-    "query_string": {
-        "query": "elasticsearch OR logstash OR kibana",<2>
-        "default_field": "my_unstructured_text_field"
-    }
-  },
-  "aggregations": {
-  \t"top_people" :{
-  \t    "significant_terms" : { <3>
-\t       "field" : "my_twitter_handles.keyword"
-  \t    }
-  \t}
-  }
-}
---------------------------
-
-<1> Note the `my_twitter_handles` contains a list of the annotation values
-also used in the unstructured text. (Note the annotated_text syntax requires escaping).
-By repeating the annotation values in a structured field this application has ensured that
-the tokens discovered in the structured field can be used for search and highlighting
-in the unstructured field.
-<2> In this example we search for documents that talk about components of the elastic stack
-<3> We use the `my_twitter_handles` field here to discover people who are significantly
-associated with the elastic stack.
-
-===== Avoiding over-matching annotations
-By design, the regular text tokens and the annotation tokens co-exist in the same indexed
-field but in rare cases this can lead to some over-matching.
-
-The value of an annotation often denotes a _named entity_ (a person, place or company).
-The tokens for these named entities are inserted untokenized, and differ from typical text
-tokens because they are normally:
-
-* Mixed case e.g. `Madonna`
-* Multiple words e.g. `Jeff Beck`
-* Can have punctuation or numbers e.g. `Apple Inc.` or `@kimchy`
-
-This means, for the most part, a search for a named entity in the annotated text field will
-not have any false positives e.g. when selecting `Apple Inc.` from an aggregation result
-you can drill down to highlight uses in the text without "over matching" on any text tokens
-like the word `apple` in this context:
-
-    the apple was very juicy
-
-However, a problem arises if your named entity happens to be a single term and lower-case e.g. the
-company `elastic`. In this case, a search on the annotated text field for the token `elastic`
-may match a text document such as this:
-
-    they fired an elastic band
-
-To avoid such false matches users should consider prefixing annotation values to ensure
-they don't name clash with text tokens e.g.
-
-    [elastic](Company_elastic) released version 7.0 of the elastic stack today
-
-
-
-
-[[mapper-annotated-text-highlighter]]
-==== Using the `annotated` highlighter
-
-The `annotated-text` plugin includes a custom highlighter designed to mark up search hits
-in a way which is respectful of the original markup:
-
-[source,console]
---------------------------
-# Example documents
-PUT my-index-000001/_doc/1
-{
-  "my_field": "The cat sat on the [mat](sku3578)"
-}
-
-GET my-index-000001/_search
-{
-  "query": {
-    "query_string": {
-        "query": "cats"
-    }
-  },
-  "highlight": {
-    "fields": {
-      "my_field": {
-        "type": "annotated", <1>
-        "require_field_match": false
-      }
-    }
-  }
-}
---------------------------
-
-<1> The `annotated` highlighter type is designed for use with annotated_text fields
-
-The annotated highlighter is based on the `unified` highlighter and supports the same
-settings but does not use the `pre_tags` or `post_tags` parameters. Rather than using
-html-like markup such as `<em>cat</em>` the annotated highlighter uses the same
-markdown-like syntax used for annotations and injects a key=value annotation where `_hit_term`
-is the key and the matched search term is the value e.g.
-
-    The [cat](_hit_term=cat) sat on the [mat](sku3578)
-
-The annotated highlighter tries to be respectful of any existing markup in the original
-text:
-
-* If the search term matches exactly the location of an existing annotation then the
-`_hit_term` key is merged into the url-like syntax used in the `(...)` part of the
-existing annotation.
-* However, if the search term overlaps the span of an existing annotation it would break
-the markup formatting so the original annotation is removed in favour of a new annotation
-with just the search hit information in the results.
-* Any non-overlapping annotations in the original text are preserved in highlighter
-selections
-
-
-[[mapper-annotated-text-limitations]]
-==== Limitations
-
-The annotated_text field type supports the same mapping settings as the `text` field type
-but with the following exceptions:
-
-* No support for `fielddata` or `fielddata_frequency_filter`
-* No support for `index_prefixes` or `index_phrases` indexing
-
-"""
-        ), substitutions
-        )
-        then:
-        snippets*.test == [false, false, false, false, false, false, false]
-        snippets*.catchPart == [null, null, null, null, null, null, null]
-    }
-
-    def "handling test parsing"() {
-        when:
-        def substitutions = []
-        def snippets = task().parseDocFile(
-            tempDir, docFile(
-            """
-[source,console]
-----
-POST logs-my_app-default/_rollover/
-----
-// TEST[s/_explain\\/1/_explain\\/1?error_trace=false/ catch:/painless_explain_error/]
-"""
-        ), substitutions
-        )
-        then:
-        snippets*.test == [true]
-        snippets*.catchPart == ["/painless_explain_error/"]
-        substitutions.size() == 1
-        substitutions[0].key == "_explain\\/1"
-        substitutions[0].value == "_explain\\/1?error_trace=false"
-
-        when:
-        substitutions = []
-        snippets = task().parseDocFile(
-            tempDir, docFile(
-            """
-
-[source,console]
-----
-PUT _snapshot/my_hdfs_repository
-{
-  "type": "hdfs",
-  "settings": {
-    "uri": "hdfs://namenode:8020/",
-    "path": "elasticsearch/repositories/my_hdfs_repository",
-    "conf.dfs.client.read.shortcircuit": "true"
-  }
-}
-----
-// TEST[skip:we don't have hdfs set up while testing this]
-"""
-        ), substitutions
-        )
-        then:
-        snippets*.test == [true]
-        snippets*.skip == ["we don't have hdfs set up while testing this"]
-    }
-
-    def "handling testresponse parsing"() {
-        when:
-        def substitutions = []
-        def snippets = task().parseDocFile(
-            tempDir, docFile(
-            """
-[source,console]
-----
-POST logs-my_app-default/_rollover/
-----
-// TESTRESPONSE[s/\\.\\.\\./"script_stack": \$body.error.caused_by.script_stack, "script": \$body.error.caused_by.script, "lang": \$body.error.caused_by.lang, "position": \$body.error.caused_by.position, "caused_by": \$body.error.caused_by.caused_by, "reason": \$body.error.caused_by.reason/]
-"""
-        ), substitutions
-        )
-        then:
-        snippets*.test == [false]
-        snippets*.testResponse == [true]
-        substitutions.size() == 1
-        substitutions[0].key == "\\.\\.\\."
-        substitutions[0].value ==
-            "\"script_stack\": \$body.error.caused_by.script_stack, \"script\": \$body.error.caused_by.script, \"lang\": \$body.error.caused_by.lang, \"position\": \$body.error.caused_by.position, \"caused_by\": \$body.error.caused_by.caused_by, \"reason\": \$body.error.caused_by.reason"
-
-        when:
-        snippets = task().parseDocFile(
-            tempDir, docFile(
-            """
-[source,console]
-----
-POST logs-my_app-default/_rollover/
-----
-// TESTRESPONSE[skip:no setup made for this example yet]
-"""
-        ), []
-        )
-        then:
-        snippets*.test == [false]
-        snippets*.testResponse == [true]
-        snippets*.skip == ["no setup made for this example yet"]
-
-        when:
-        substitutions = []
-        snippets = task().parseDocFile(
-            tempDir, docFile(
-            """
-[source,txt]
----------------------------------------------------------------------------
-my-index-000001 0 p RELOCATING 3014 31.1mb 192.168.56.10 H5dfFeA -> -> 192.168.56.30 bGG90GE
----------------------------------------------------------------------------
-// TESTRESPONSE[non_json]
-"""
-        ), substitutions
-        )
-        then:
-        snippets*.test == [false]
-        snippets*.testResponse == [true]
-        substitutions.size() == 4
-    }
-
-
-    def "handling console parsing"() {
-        when:
-        def snippets = task().parseDocFile(
-            tempDir, docFile(
-            """
-[source,console]
-----
-
-// $firstToken
-----
-"""
-        ), []
-        )
-        then:
-        snippets*.console == [firstToken.equals("CONSOLE")]
-
-
-        when:
-        task().parseDocFile(
-            tempDir, docFile(
-            """
-[source,console]
-----
-// $firstToken
-// $secondToken
-----
-"""
-        ), []
-        )
-        then:
-        def e = thrown(InvalidUserDataException)
-        e.message == "mapping-charfilter.asciidoc:4: Can't be both CONSOLE and NOTCONSOLE"
-
-        when:
-        task().parseDocFile(
-            tempDir, docFile(
-            """
-// $firstToken
-// $secondToken
-"""
-        ), []
-        )
-        then:
-        e = thrown(InvalidUserDataException)
-        e.message == "mapping-charfilter.asciidoc:1: $firstToken not paired with a snippet"
+}"""
 
         where:
-        firstToken << ["CONSOLE", "NOTCONSOLE"]
-        secondToken << ["NOTCONSOLE", "CONSOLE"]
+        fileType << ["asciidoc", "mdx"]
+        expectedSnippetStarts << [[10, 24, 36, 59, 86, 108, 135], [9, 22, 33, 55, 80, 101, 127]]
+        expectedSnippetEnds << [[21, 30, 55, 75, 105, 132, 158], [20, 28, 52, 71, 99, 125, 150]]
     }
 
-    def "test parsing snippet from doc"() {
-        def doc = docFile(
-            """
-[source,console]
-----
-GET /_analyze
-{
-  "tokenizer": "keyword",
-  "char_filter": [
-    {
-      "type": "mapping",
-      "mappings": [
-        "e => 0",
-        "m => 1",
-        "p => 2",
-        "t => 3",
-        "y => 4"
-      ]
-    }
-  ],
-  "text": "My license plate is empty"
-}
-----
-"""
+    List<Snippet> parseFile(String fileName) {
+        def task = ProjectBuilder.builder().build().tasks.register("docSnippetTask", DocSnippetTask).get()
+        def docFileToParse = docFile(fileName, DocTestUtils.SAMPLE_TEST_DOCS[fileName])
+        return task.parseDocFile(
+            tempDir, docFileToParse
         )
-        def snippets = task().parseDocFile(tempDir, doc, [])
-        expect:
-        snippets[0].start == 3
-        snippets[0].language == "console"
-        normalizeString(snippets[0].contents, tempDir) == """GET /_analyze
-{
-  "tokenizer": "keyword",
-  "char_filter": [
-    {
-      "type": "mapping",
-      "mappings": [
-        "e => 0",
-        "m => 1",
-        "p => 2",
-        "t => 3",
-        "y => 4"
-      ]
-    }
-  ],
-  "text": "My license plate is empty"
-}"""
     }
 
-    File docFile(String docContent) {
-        def file = tempDir.toPath().resolve("mapping-charfilter.asciidoc").toFile()
+    File docFile(String filename, String docContent) {
+        def file = tempDir.toPath().resolve(filename).toFile()
         file.text = docContent
         return file
     }
-
-
-    private DocSnippetTask task() {
-        ProjectBuilder.builder().build().tasks.register("docSnippetTask", DocSnippetTask).get()
-    }
-
 }
diff --git a/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/doc/DocTestUtils.groovy b/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/doc/DocTestUtils.groovy
new file mode 100644
index 0000000000000..350d8638c8005
--- /dev/null
+++ b/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/doc/DocTestUtils.groovy
@@ -0,0 +1,745 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.gradle.internal.doc;
+
+class DocTestUtils {
+    public static Map<String,String> SAMPLE_TEST_DOCS = Map.of(
+        "example-1.mdx", """
+# mapper-annotated-text
+### Mapper annotated text plugin
+
+experimental[]
+
+some text
+
+```console
+PUT my-index-000001
+{
+  "mappings": {
+    "properties": {
+      "my_field": {
+        "type": "annotated_text"
+      }
+    }
+  }
+}
+```
+
+```js
+GET my-index-000001/_analyze
+{
+  "field": "my_field",
+  "text":"Investors in [Apple](Apple+Inc.) rejoiced."
+}
+```
+{/* NOTCONSOLE */}
+
+Response:
+
+```js
+{
+  "tokens": [
+    {
+      "token": "investors",
+      "start_offset": 0,
+      "end_offset": 9,
+      "type": "<ALPHANUM>",
+      "position": 0
+    },
+    {
+      "token": "in",
+      "start_offset": 10,
+      "end_offset": 12,
+      "type": "<ALPHANUM>",
+      "position": 1
+    }
+  ]
+}
+```
+{/* NOTCONSOLE */}
+
+```console
+# Example documents
+PUT my-index-000001/_doc/1
+{
+  "my_field": "[Jeff Beck](Jeff+Beck&Guitarist) plays a strat"<2>
+}
+
+# Example search
+GET my-index-000001/_search
+{
+  "query": {
+    "term": {
+        "my_field": "Beck" <3>
+    }
+  }
+}
+```
+
+<1> More text
+<2> Even More
+<3> More
+
+### Headline
+#### a smaller headline
+
+```console
+PUT my-index-000001
+{
+  "mappings": {
+    "properties": {
+      "my_unstructured_text_field": {
+        "type": "annotated_text"
+      },
+      "my_structured_people_field": {
+        "type": "text",
+        "fields": {
+          "keyword" : {
+            "type": "keyword"
+          }
+        }
+      }
+    }
+  }
+}
+```
+
+```console
+# Example documents
+PUT my-index-000001/_doc/1
+{
+  "my_unstructured_text_field": "[Shay](%40kimchy) created elasticsearch",
+  "my_twitter_handles": ["@kimchy"] <1>
+}
+
+GET my-index-000001/_search
+{
+  "query": {
+    "query_string": {
+        "query": "elasticsearch OR logstash OR kibana",<2>
+        "default_field": "my_unstructured_text_field"
+    }
+  },
+  "aggregations": {
+  \t"top_people" :{
+  \t    "significant_terms" : { <3>
+\t       "field" : "my_twitter_handles.keyword"
+  \t    }
+  \t}
+  }
+}
+```
+
+```console
+# Example documents
+PUT my-index-000001/_doc/1
+{
+  "my_field": "The cat sat on the [mat](sku3578)"
+}
+
+GET my-index-000001/_search
+{
+  "query": {
+    "query_string": {
+        "query": "cats"
+    }
+  },
+  "highlight": {
+    "fields": {
+      "my_field": {
+        "type": "annotated", <1>
+        "require_field_match": false
+      }
+    }
+  }
+}
+```
+
+* No support for `fielddata` or `fielddata_frequency_filter`
+* No support for `index_prefixes` or `index_phrases` indexing
+
+""",
+
+        "example-1.asciidoc", """
+[[mapper-annotated-text]]
+=== Mapper annotated text plugin
+
+experimental[]
+
+some text
+
+[source,console]
+--------------------------
+PUT my-index-000001
+{
+  "mappings": {
+    "properties": {
+      "my_field": {
+        "type": "annotated_text"
+      }
+    }
+  }
+}
+--------------------------
+
+[source,js]
+--------------------------
+GET my-index-000001/_analyze
+{
+  "field": "my_field",
+  "text":"Investors in [Apple](Apple+Inc.) rejoiced."
+}
+--------------------------
+// NOTCONSOLE
+
+Response:
+
+[source,js]
+--------------------------------------------------
+{
+  "tokens": [
+    {
+      "token": "investors",
+      "start_offset": 0,
+      "end_offset": 9,
+      "type": "<ALPHANUM>",
+      "position": 0
+    },
+    {
+      "token": "in",
+      "start_offset": 10,
+      "end_offset": 12,
+      "type": "<ALPHANUM>",
+      "position": 1
+    }
+  ]
+}
+--------------------------------------------------
+// NOTCONSOLE
+
+[source,console]
+--------------------------
+# Example documents
+PUT my-index-000001/_doc/1
+{
+  "my_field": "[Jeff Beck](Jeff+Beck&Guitarist) plays a strat"<2>
+}
+
+# Example search
+GET my-index-000001/_search
+{
+  "query": {
+    "term": {
+        "my_field": "Beck" <3>
+    }
+  }
+}
+--------------------------
+
+<1> More text
+<2> Even More
+<3> More
+
+[[mapper-annotated-text-tips]]
+==== Headline
+===== a smaller headline
+
+[source,console]
+--------------------------
+PUT my-index-000001
+{
+  "mappings": {
+    "properties": {
+      "my_unstructured_text_field": {
+        "type": "annotated_text"
+      },
+      "my_structured_people_field": {
+        "type": "text",
+        "fields": {
+          "keyword" : {
+            "type": "keyword"
+          }
+        }
+      }
+    }
+  }
+}
+--------------------------
+
+[source,console]
+--------------------------
+# Example documents
+PUT my-index-000001/_doc/1
+{
+  "my_unstructured_text_field": "[Shay](%40kimchy) created elasticsearch",
+  "my_twitter_handles": ["@kimchy"] <1>
+}
+
+GET my-index-000001/_search
+{
+  "query": {
+    "query_string": {
+        "query": "elasticsearch OR logstash OR kibana",<2>
+        "default_field": "my_unstructured_text_field"
+    }
+  },
+  "aggregations": {
+  \t"top_people" :{
+  \t    "significant_terms" : { <3>
+\t       "field" : "my_twitter_handles.keyword"
+  \t    }
+  \t}
+  }
+}
+--------------------------
+
+[source,console]
+--------------------------
+# Example documents
+PUT my-index-000001/_doc/1
+{
+  "my_field": "The cat sat on the [mat](sku3578)"
+}
+
+GET my-index-000001/_search
+{
+  "query": {
+    "query_string": {
+        "query": "cats"
+    }
+  },
+  "highlight": {
+    "fields": {
+      "my_field": {
+        "type": "annotated", <1>
+        "require_field_match": false
+      }
+    }
+  }
+}
+--------------------------
+
+* No support for `fielddata` or `fielddata_frequency_filter`
+* No support for `index_prefixes` or `index_phrases` indexing
+
+""",
+
+
+        "example-2.asciidoc", """
+[[example-2]]
+=== Field context
+
+Use a Painless script to create a
+{ref}/search-fields.html#script-fields[script field] to return
+a customized value for each document in the results of a query.
+
+*Variables*
+
+`params` (`Map`, read-only)::
+        User-defined parameters passed in as part of the query.
+
+`doc` (`Map`, read-only)::
+        Contains the fields of the specified document where each field is a
+        `List` of values.
+
+{ref}/mapping-source-field.html[`params['_source']`] (`Map`, read-only)::
+        Contains extracted JSON in a `Map` and `List` structure for the fields
+        existing in a stored document.
+
+*Return*
+
+`Object`::
+        The customized value for each document.
+
+*API*
+
+Both the standard <<painless-api-reference-shared, Painless API>> and
+<<painless-api-reference-field, Specialized Field API>> are available.
+
+
+*Example*
+
+To run this example, first follow the steps in
+<<painless-context-examples, context examples>>.
+
+You can then use these two example scripts to compute custom information
+for each search hit and output it to two new fields.
+
+The first script gets the doc value for the `datetime` field and calls
+the `getDayOfWeekEnum` function to determine the corresponding day of the week.
+
+[source,Painless]
+----
+doc['datetime'].value.getDayOfWeekEnum().getDisplayName(TextStyle.FULL, Locale.ROOT)
+----
+
+The second script calculates the number of actors. Actors' names are stored
+as a keyword array in the `actors` field.
+
+[source,Painless]
+----
+doc['actors'].size()  <1>
+----
+
+<1> By default, doc values are not available for `text` fields. If `actors` was
+a `text` field, you could still calculate the number of actors by extracting
+values from `_source` with `params['_source']['actors'].size()`.
+
+The following request returns the calculated day of week and the number of
+actors that appear in each play:
+
+[source,console]
+----
+GET seats/_search
+{
+  "size": 2,
+  "query": {
+    "match_all": {}
+  },
+  "script_fields": {
+    "day-of-week": {
+      "script": {
+        "source": "doc['datetime'].value.getDayOfWeekEnum().getDisplayName(TextStyle.FULL, Locale.ROOT)"
+      }
+    },
+    "number-of-actors": {
+      "script": {
+        "source": "doc['actors'].size()"
+      }
+    }
+  }
+}
+----
+// TEST[setup:seats]
+
+[source,console-result]
+----
+{
+  "took" : 68,
+  "timed_out" : false,
+  "_shards" : {
+    "total" : 1,
+    "successful" : 1,
+    "skipped" : 0,
+    "failed" : 0
+  },
+  "hits" : {
+    "total" : {
+      "value" : 11,
+      "relation" : "eq"
+    },
+    "max_score" : 1.0,
+    "hits" : [
+      {
+        "_index" : "seats",
+        "_id" : "1",
+        "_score" : 1.0,
+        "fields" : {
+          "day-of-week" : [
+            "Thursday"
+          ],
+          "number-of-actors" : [
+            4
+          ]
+        }
+      },
+      {
+        "_index" : "seats",
+        "_id" : "2",
+        "_score" : 1.0,
+        "fields" : {
+          "day-of-week" : [
+            "Thursday"
+          ],
+          "number-of-actors" : [
+            1
+          ]
+        }
+      }
+    ]
+  }
+}
+----
+// TESTRESPONSE[s/"took" : 68/"took" : "\$body.took"/]
+""",
+        "example-2.mdx", """---
+id: enElasticsearchPainlessPainlessFieldContext
+slug: /en/elasticsearch/painless/example-2
+title: Field context
+description: Description to be written
+tags: []
+---
+
+<div id="example-2"></div>
+
+Use a Painless script to create a
+[script field](((ref))/search-fields.html#script-fields) to return
+a customized value for each document in the results of a query.
+
+**Variables**
+
+`params` (`Map`, read-only)
+    : User-defined parameters passed in as part of the query.
+
+`doc` (`Map`, read-only)
+    : Contains the fields of the specified document where each field is a
+    `List` of values.
+
+[`params['_source']`](((ref))/mapping-source-field.html) (`Map`, read-only)
+    : Contains extracted JSON in a `Map` and `List` structure for the fields
+    existing in a stored document.
+
+**Return**
+
+`Object`
+    : The customized value for each document.
+
+**API**
+
+Both the standard <DocLink id="enElasticsearchPainlessPainlessApiReferenceShared">Painless API</DocLink> and
+<DocLink id="enElasticsearchPainlessPainlessApiReferenceField">Specialized Field API</DocLink> are available.
+
+**Example**
+
+To run this example, first follow the steps in
+<DocLink id="enElasticsearchPainlessPainlessContextExamples">context examples</DocLink>.
+
+You can then use these two example scripts to compute custom information
+for each search hit and output it to two new fields.
+
+The first script gets the doc value for the `datetime` field and calls
+the `getDayOfWeekEnum` function to determine the corresponding day of the week.
+
+```Painless
+doc['datetime'].value.getDayOfWeekEnum().getDisplayName(TextStyle.FULL, Locale.ROOT)
+```
+
+The second script calculates the number of actors. Actors' names are stored
+as a keyword array in the `actors` field.
+
+```Painless
+doc['actors'].size()   [^1]
+```
+[^1]: By default, doc values are not available for `text` fields. If `actors` was
+a `text` field, you could still calculate the number of actors by extracting
+values from `_source` with `params['_source']['actors'].size()`.
+
+The following request returns the calculated day of week and the number of
+actors that appear in each play:
+
+```console
+GET seats/_search
+{
+  "size": 2,
+  "query": {
+    "match_all": {}
+  },
+  "script_fields": {
+    "day-of-week": {
+      "script": {
+        "source": "doc['datetime'].value.getDayOfWeekEnum().getDisplayName(TextStyle.FULL, Locale.ROOT)"
+      }
+    },
+    "number-of-actors": {
+      "script": {
+        "source": "doc['actors'].size()"
+      }
+    }
+  }
+}
+```
+{/* TEST[setup:seats] */}
+
+```console-result
+{
+  "took" : 68,
+  "timed_out" : false,
+  "_shards" : {
+    "total" : 1,
+    "successful" : 1,
+    "skipped" : 0,
+    "failed" : 0
+  },
+  "hits" : {
+    "total" : {
+      "value" : 11,
+      "relation" : "eq"
+    },
+    "max_score" : 1.0,
+    "hits" : [
+      {
+        "_index" : "seats",
+        "_id" : "1",
+        "_score" : 1.0,
+        "fields" : {
+          "day-of-week" : [
+            "Thursday"
+          ],
+          "number-of-actors" : [
+            4
+          ]
+        }
+      },
+      {
+        "_index" : "seats",
+        "_id" : "2",
+        "_score" : 1.0,
+        "fields" : {
+          "day-of-week" : [
+            "Thursday"
+          ],
+          "number-of-actors" : [
+            1
+          ]
+        }
+      }
+    ]
+  }
+}
+```
+{/* TESTRESPONSE[s/"took" : 68/"took" : "\$body.took"/] */}
+""",
+        "example-2-different.mdx", """---
+id: enElasticsearchPainlessPainlessFieldContext
+slug: /en/elasticsearch/painless/example-2
+title: Field context
+description: Description to be written
+tags: []
+---
+
+<div id="example-2"></div>
+
+Use a Painless script to create a
+[script field](((ref))/search-fields.html#script-fields) to return
+a customized value for each document in the results of a query.
+
+**Variables**
+
+`params` (`Map`, read-only)
+    : User-defined parameters passed in as part of the query.
+
+`doc` (`Map`, read-only)
+    : Contains the fields of the specified document where each field is a
+    `List` of values.
+
+[`params['_source']`](((ref))/mapping-source-field.html) (`Map`, read-only)
+    : Contains extracted JSON in a `Map` and `List` structure for the fields
+    existing in a stored document.
+
+**Return**
+
+`Object`
+    : The customized value for each document.
+
+**API**
+
+Both the standard <DocLink id="enElasticsearchPainlessPainlessApiReferenceShared">Painless API</DocLink> and
+<DocLink id="enElasticsearchPainlessPainlessApiReferenceField">Specialized Field API</DocLink> are available.
+
+**Example**
+
+To run this example, first follow the steps in
+<DocLink id="enElasticsearchPainlessPainlessContextExamples">context examples</DocLink>.
+
+You can then use these two example scripts to compute custom information
+for each search hit and output it to two new fields.
+
+The first script gets the doc value for the `datetime` field and calls
+the `getDayOfWeekEnum` function to determine the corresponding day of the week.
+
+```Painless
+doc['datetime'].value.getDayOfWeekEnum().getDisplayName(TextStyle.FULL, Locale.ROOT)
+```
+
+The second script calculates the number of actors. Actors' names are stored
+as a keyword array in the `actors` field.
+
+```Painless
+doc['actresses'].size()   [^1]
+```
+[^1]: By default, doc values are not available for `text` fields. If `actors` was
+a `text` field, you could still calculate the number of actors by extracting
+values from `_source` with `params['_source']['actors'].size()`.
+
+The following request returns the calculated day of week and the number of
+actors that appear in each play:
+
+```console
+GET seats/_search
+{
+  "size": 2,
+  "query": {
+    "match_all": {}
+  },
+  "script_fields": {
+    "day-of-week": {
+      "script": {
+        "source": "doc['datetime'].value.getDayOfWeekEnum().getDisplayName(TextStyle.FULL, Locale.ROOT)"
+      }
+    },
+    "number-of-actors": {
+      "script": {
+        "source": "doc['actors'].size()"
+      }
+    }
+  }
+}
+```
+{/* TEST[setup:seats] */}
+
+```console-result
+{
+  "took" : 68,
+  "timed_out" : false,
+  "_shards" : {
+    "total" : 1,
+    "successful" : 1,
+    "skipped" : 0,
+    "failed" : 0
+  },
+  "hits" : {
+    "total" : {
+      "value" : 11,
+      "relation" : "eq"
+    },
+    "max_score" : 1.0,
+    "hits" : [
+      {
+        "_index" : "seats",
+        "_id" : "1",
+        "_score" : 1.0,
+        "fields" : {
+          "day-of-week" : [
+            "Thursday"
+          ],
+          "number-of-actors" : [
+            4
+          ]
+        }
+      },
+      {
+        "_index" : "seats",
+        "_id" : "2",
+        "_score" : 1.0,
+        "fields" : {
+          "day-of-week" : [
+            "Thursday"
+          ],
+          "number-of-actors" : [
+            1
+          ]
+        }
+      }
+    ]
+  }
+}
+```
+{/* TESTRESPONSE[s/"took" : 68/"took" : "\$body.took"/] */}
+""")
+}
diff --git a/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/doc/MdxSnippetParserSpec.groovy b/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/doc/MdxSnippetParserSpec.groovy
new file mode 100644
index 0000000000000..020b920de3d0e
--- /dev/null
+++ b/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/doc/MdxSnippetParserSpec.groovy
@@ -0,0 +1,173 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.gradle.internal.doc
+
+class MdxSnippetParserSpec extends AbstractSnippetParserSpec {
+
+    @Override
+    SnippetParser parser(Map<String,String> defaultSubstitutions = [:]) {
+        return new MdxSnippetParser(defaultSubstitutions)
+    }
+
+    @Override
+    String docSnippetWithTest() {
+        return """```console
+PUT /hockey/_doc/1?refresh
+{"first":"johnny","last":"gaudreau","goals":[9,27,1],"assists":[17,46,0],"gp":[26,82,1]}
+
+POST /hockey/_explain/1
+{
+  "query": {
+    "script": {
+      "script": "Debug.explain(doc.goals)"
+    }
+  }
+}
+```
+{/* TEST[s/_explain\\/1/_explain\\/1?error_trace=false/ catch:/painless_explain_error/] */}
+{/* TEST[teardown:some_teardown] */}
+{/* TEST[setup:seats] */}
+{/* TEST[warning:some_warning] */}
+{/* TEST[skip_shard_failures] */}
+
+"""
+    }
+
+    @Override
+    String docSnippetWithRepetitiveSubstiutions() {
+        return """```console
+GET /_cat/snapshots/repo1?v=true&s=id
+```
+{/* TEST[s/^/PUT \\/_snapshot\\/repo1\\/snap1?wait_for_completion=true\\n/] */}
+{/* TEST[s/^/PUT \\/_snapshot\\/repo1\\/snap2?wait_for_completion=true\\n/] */}
+{/* TEST[s/^/PUT \\/_snapshot\\/repo1\\n{"type": "fs", "settings": {"location": "repo\\/1"}}\\n/] */}
+"""
+    }
+
+    @Override
+    String docSnippetWithConsole() {
+        return """
+```console
+{/* CONSOLE */}
+```
+"""
+    }
+
+    @Override
+    String docSnippetWithNotConsole() {
+        return """
+```console
+{/* NOTCONSOLE */}
+```
+"""    }
+
+    @Override
+    String docSnippetWithMixedConsoleNotConsole() {
+        return """
+```console
+{/* CONSOLE */}
+{/* NOTCONSOLE */}
+```
+"""    }
+
+    @Override
+    String docSnippetWithTestResponses() {
+        return """```console-result
+{
+  "docs" : [
+    {
+      "processor_results" : [
+        {
+          "processor_type" : "set",
+          "status" : "success",
+          "doc" : {
+            "_index" : "index",
+            "_id" : "id",
+            "_version": "-3",
+            "_source" : {
+              "field2" : "_value2",
+              "foo" : "bar"
+            },
+            "_ingest" : {
+              "pipeline" : "_simulate_pipeline",
+              "timestamp" : "2020-07-30T01:21:24.251836Z"
+            }
+          }
+        },
+        {
+          "processor_type" : "set",
+          "status" : "success",
+          "doc" : {
+            "_index" : "index",
+            "_id" : "id",
+            "_version": "-3",
+            "_source" : {
+              "field3" : "_value3",
+              "field2" : "_value2",
+              "foo" : "bar"
+            },
+            "_ingest" : {
+              "pipeline" : "_simulate_pipeline",
+              "timestamp" : "2020-07-30T01:21:24.251836Z"
+            }
+          }
+        }
+      ]
+    },
+    {
+      "processor_results" : [
+        {
+          "processor_type" : "set",
+          "status" : "success",
+          "doc" : {
+            "_index" : "index",
+            "_id" : "id",
+            "_version": "-3",
+            "_source" : {
+              "field2" : "_value2",
+              "foo" : "rab"
+            },
+            "_ingest" : {
+              "pipeline" : "_simulate_pipeline",
+              "timestamp" : "2020-07-30T01:21:24.251863Z"
+            }
+          }
+        },
+        {
+          "processor_type" : "set",
+          "status" : "success",
+          "doc" : {
+            "_index" : "index",
+            "_id" : "id",
+            "_version": "-3",
+            "_source" : {
+              "field3" : "_value3",
+              "field2" : "_value2",
+              "foo" : "rab"
+            },
+            "_ingest" : {
+              "pipeline" : "_simulate_pipeline",
+              "timestamp" : "2020-07-30T01:21:24.251863Z"
+            }
+          }
+        }
+      ]
+    }
+  ]
+}
+```
+{/* TESTRESPONSE[s/"2020-07-30T01:21:24.251836Z"/\$body.docs.0.processor_results.0.doc._ingest.timestamp/] */}
+{/* TESTRESPONSE[s/"2020-07-30T01:21:24.251836Z"/\$body.docs.0.processor_results.1.doc._ingest.timestamp/] */}
+{/* TESTRESPONSE[s/"2020-07-30T01:21:24.251863Z"/\$body.docs.1.processor_results.0.doc._ingest.timestamp/] */}
+{/* TESTRESPONSE[s/"2020-07-30T01:21:24.251863Z"/\$body.docs.1.processor_results.1.doc._ingest.timestamp/] */}
+{/* TESTRESPONSE[skip:some_skip_message] */}
+"""
+    }
+
+}
diff --git a/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/doc/RestTestsFromDocSnippetTaskSpec.groovy b/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/doc/RestTestsFromDocSnippetTaskSpec.groovy
index 45d3892121952..dde1931afaa41 100644
--- a/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/doc/RestTestsFromDocSnippetTaskSpec.groovy
+++ b/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/doc/RestTestsFromDocSnippetTaskSpec.groovy
@@ -11,9 +11,11 @@ package org.elasticsearch.gradle.internal.doc
 import spock.lang.Specification
 import spock.lang.TempDir
 
+import org.gradle.api.GradleException
 import org.gradle.api.InvalidUserDataException
 import org.gradle.testfixtures.ProjectBuilder
 
+import static org.elasticsearch.gradle.internal.doc.DocTestUtils.SAMPLE_TEST_DOCS
 import static org.elasticsearch.gradle.internal.doc.RestTestsFromDocSnippetTask.replaceBlockQuote
 import static org.elasticsearch.gradle.internal.doc.RestTestsFromDocSnippetTask.shouldAddShardFailureCheck
 import static org.elasticsearch.gradle.internal.test.TestUtils.normalizeString
@@ -56,778 +58,149 @@ class RestTestsFromDocSnippetTaskSpec extends Specification {
         shouldAddShardFailureCheck("_ml/datafeeds/datafeed-id/_preview") == false
     }
 
-    def "can create rest tests from docs"() {
-        def build = ProjectBuilder.builder().build()
+    def "can generate tests files from asciidoc and mdx"() {
         given:
-        def task = build.tasks.create("restTestFromSnippet", RestTestsFromDocSnippetTask)
-        task.expectedUnconvertedCandidates = ["ml-update-snapshot.asciidoc", "reference/security/authorization/run-as-privilege.asciidoc"]
-        docs()
+        def build = ProjectBuilder.builder().build()
+        def task = build.tasks.register("restTestFromSnippet", RestTestsFromDocSnippetTask).get()
+        task.expectedUnconvertedCandidates = []
         task.docs = build.fileTree(new File(tempDir, "docs"))
         task.testRoot.convention(build.getLayout().buildDirectory.dir("rest-tests"));
-
+        docFile('docs/example-2-asciidoc.asciidoc', SAMPLE_TEST_DOCS['example-2.asciidoc'])
+        docFile('docs/example-2-mdx.mdx', SAMPLE_TEST_DOCS['example-2.mdx'])
+        task.getSetups().put(
+            "seats", """
+'''
+  - do:
+        indices.create:
+          index: seats
+          body:
+            settings:
+              number_of_shards: 1
+              number_of_replicas: 0
+            mappings:
+              properties:
+                theatre:
+                  type: keyword
+"""
+        )
         when:
         task.getActions().forEach { it.execute(task) }
-        def restSpec = new File(task.getTestRoot().get().getAsFile(), "rest-api-spec/test/painless-debugging.yml")
 
         then:
-        restSpec.exists()
-        normalizeString(restSpec.text, tempDir) == """---
-"line_22":
-  - skip:
-      features:
-        - default_shards
-        - stash_in_key
-        - stash_in_path
-        - stash_path_replace
-        - warnings
-  - do:
-      raw:
-        method: PUT
-        path: "hockey/_doc/1"
-        refresh: ""
-        body: |
-          {"first":"johnny","last":"gaudreau","goals":[9,27,1],"assists":[17,46,0],"gp":[26,82,1]}
-  - is_false: _shards.failures
-  - do:
-      catch: /painless_explain_error/
-      raw:
-        method: POST
-        path: "hockey/_explain/1"
-        error_trace: "false"
-        body: |
-          {
-            "query": {
-              "script": {
-                "script": "Debug.explain(doc.goals)"
-              }
-            }
-          }
-  - is_false: _shards.failures
-  - match:
-      \$body:
-        {
-           "error": {
-              "type": "script_exception",
-              "to_string": "[1, 9, 27]",
-              "painless_class": "org.elasticsearch.index.fielddata.ScriptDocValues.Longs",
-              "java_class": "org.elasticsearch.index.fielddata.ScriptDocValues\$Longs",
-              "script_stack": \$body.error.script_stack, "script": \$body.error.script, "lang": \$body.error.lang, "position": \$body.error.position, "caused_by": \$body.error.caused_by, "root_cause": \$body.error.root_cause, "reason": \$body.error.reason
-           },
-           "status": 400
-        }
-  - do:
-      catch: /painless_explain_error/
-      raw:
-        method: POST
-        path: "hockey/_update/1"
-        error_trace: "false"
-        body: |
-          {
-            "script": "Debug.explain(ctx._source)"
-          }
-  - is_false: _shards.failures
-  - match:
-      \$body:
-        {
-          "error" : {
-            "root_cause": \$body.error.root_cause,
-            "type": "illegal_argument_exception",
-            "reason": "failed to execute script",
-            "caused_by": {
-              "type": "script_exception",
-              "to_string": \$body.error.caused_by.to_string,
-              "painless_class": "java.util.LinkedHashMap",
-              "java_class": "java.util.LinkedHashMap",
-              "script_stack": \$body.error.caused_by.script_stack, "script": \$body.error.caused_by.script, "lang": \$body.error.caused_by.lang, "position": \$body.error.caused_by.position, "caused_by": \$body.error.caused_by.caused_by, "reason": \$body.error.caused_by.reason
-            }
-          },
-          "status": 400
-        }"""
-        def restSpec2 = new File(task.testRoot.get().getAsFile(), "rest-api-spec/test/ml-update-snapshot.yml")
-        restSpec2.exists()
-        normalizeString(restSpec2.text, tempDir) == """---
-"line_50":
-  - skip:
-      features:
-        - default_shards
-        - stash_in_key
-        - stash_in_path
-        - stash_path_replace
-        - warnings
-        - always_skip
-      reason: todo
-  - do:
-      raw:
-        method: POST
-        path: "_ml/anomaly_detectors/it_ops_new_logs/model_snapshots/1491852978/_update"
-        body: |
-          {
-            "description": "Snapshot 1",
-            "retain": true
-          }
-  - is_false: _shards.failures"""
-        def restSpec3 = new File(task.testRoot.get().getAsFile(), "rest-api-spec/test/reference/sql/getting-started.yml")
-        restSpec3.exists()
-        normalizeString(restSpec3.text, tempDir) == """---
-"line_10":
-  - skip:
-      features:
-        - default_shards
-        - stash_in_key
-        - stash_in_path
-        - stash_path_replace
-        - warnings
-  - do:
-      raw:
-        method: PUT
-        path: "library/_bulk"
-        refresh: ""
-        body: |
-          {"index":{"_id": "Leviathan Wakes"}}
-          {"name": "Leviathan Wakes", "author": "James S.A. Corey", "release_date": "2011-06-02", "page_count": 561}
-          {"index":{"_id": "Hyperion"}}
-          {"name": "Hyperion", "author": "Dan Simmons", "release_date": "1989-05-26", "page_count": 482}
-          {"index":{"_id": "Dune"}}
-          {"name": "Dune", "author": "Frank Herbert", "release_date": "1965-06-01", "page_count": 604}
-  - is_false: _shards.failures
-  - do:
-      raw:
-        method: POST
-        path: "_sql"
-        format: "txt"
-        body: |
-          {
-            "query": "SELECT * FROM library WHERE release_date < '2000-01-01'"
-          }
-  - is_false: _shards.failures
-  - match:
-      \$body:
-        /    /s+author     /s+/|     /s+name      /s+/|  /s+page_count   /s+/| /s+release_date/s*
-         ---------------/+---------------/+---------------/+------------------------/s*
-         Dan /s+Simmons    /s+/|Hyperion       /s+/|482            /s+/|1989-05-26T00:00:00.000Z/s*
-         Frank /s+Herbert  /s+/|Dune           /s+/|604            /s+/|1965-06-01T00:00:00.000Z/s*/"""
-    def restSpec4 = new File(task.testRoot.get().getAsFile(), "rest-api-spec/test/reference/security/authorization/run-as-privilege.yml")
-    restSpec4.exists()
-    normalizeString(restSpec4.text, tempDir) == """---
-"line_51":
-  - skip:
-      features:
-        - default_shards
-        - stash_in_key
-        - stash_in_path
-        - stash_path_replace
-        - warnings
-  - do:
-      raw:
-        method: POST
-        path: "_security/role/my_director"
-        refresh: "true"
-        body: |
-          {
-            "cluster": ["manage"],
-            "indices": [
-              {
-                "names": [ "index1", "index2" ],
-                "privileges": [ "manage" ]
-              }
-            ],
-            "run_as": [ "jacknich", "rdeniro" ],
-            "metadata" : {
-              "version" : 1
-            }
-          }
-  - is_false: _shards.failures
----
-"line_114":
-  - skip:
-      features:
-        - default_shards
-        - stash_in_key
-        - stash_in_path
-        - stash_path_replace
-        - warnings
-  - do:
-      raw:
-        method: POST
-        path: "_security/role/my_admin_role"
-        refresh: "true"
-        body: |
-          {
-            "cluster": ["manage"],
-            "indices": [
-              {
-                "names": [ "index1", "index2" ],
-                "privileges": [ "manage" ]
-              }
-            ],
-            "applications": [
-              {
-                "application": "myapp",
-                "privileges": [ "admin", "read" ],
-                "resources": [ "*" ]
-              }
-            ],
-            "run_as": [ "analyst_user" ],
-            "metadata" : {
-              "version" : 1
-            }
-          }
-  - is_false: _shards.failures
----
-"line_143":
-  - skip:
-      features:
-        - default_shards
-        - stash_in_key
-        - stash_in_path
-        - stash_path_replace
-        - warnings
-  - do:
-      raw:
-        method: POST
-        path: "_security/role/my_analyst_role"
-        refresh: "true"
-        body: |
-          {
-            "cluster": [ "monitor"],
-            "indices": [
-              {
-                "names": [ "index1", "index2" ],
-                "privileges": ["manage"]
-              }
-            ],
-            "applications": [
-              {
-                "application": "myapp",
-                "privileges": [ "read" ],
-                "resources": [ "*" ]
-              }
-            ],
-            "metadata" : {
-              "version" : 1
-            }
-          }
-  - is_false: _shards.failures
----
-"line_170":
-  - skip:
-      features:
-        - default_shards
-        - stash_in_key
-        - stash_in_path
-        - stash_path_replace
-        - warnings
-  - do:
-      raw:
-        method: POST
-        path: "_security/user/admin_user"
-        refresh: "true"
-        body: |
-          {
-            "password": "l0ng-r4nd0m-p@ssw0rd",
-            "roles": [ "my_admin_role" ],
-            "full_name": "Eirian Zola",
-            "metadata": { "intelligence" : 7}
-          }
-  - is_false: _shards.failures
----
-"line_184":
-  - skip:
-      features:
-        - default_shards
-        - stash_in_key
-        - stash_in_path
-        - stash_path_replace
-        - warnings
-  - do:
-      raw:
-        method: POST
-        path: "_security/user/analyst_user"
-        refresh: "true"
-        body: |
-          {
-            "password": "l0nger-r4nd0mer-p@ssw0rd",
-            "roles": [ "my_analyst_role" ],
-            "full_name": "Monday Jaffe",
-            "metadata": { "innovation" : 8}
-          }
-  - is_false: _shards.failures"""
-}
-
-    File docFile(String fileName, String docContent) {
-        def file = tempDir.toPath().resolve(fileName).toFile()
-        file.parentFile.mkdirs()
-        file.text = docContent
-        return file
+        def restSpecFromAsciidoc = new File(task.getTestRoot().get().getAsFile(), "rest-api-spec/test/example-2-asciidoc.yml")
+        def restSpecFromMdx = new File(task.getTestRoot().get().getAsFile(), "rest-api-spec/test/example-2-mdx.yml")
+        normalizeRestSpec(restSpecFromAsciidoc.text) == normalizeRestSpec(restSpecFromMdx.text)
     }
 
-
-    void docs() {
-        docFile(
-            "docs/reference/sql/getting-started.asciidoc", """
-[role="xpack"]
-[[sql-getting-started]]
-== Getting Started with SQL
-
-To start using {es-sql}, create
-an index with some data to experiment with:
-
-[source,console]
---------------------------------------------------
-PUT /library/_bulk?refresh
-{"index":{"_id": "Leviathan Wakes"}}
-{"name": "Leviathan Wakes", "author": "James S.A. Corey", "release_date": "2011-06-02", "page_count": 561}
-{"index":{"_id": "Hyperion"}}
-{"name": "Hyperion", "author": "Dan Simmons", "release_date": "1989-05-26", "page_count": 482}
-{"index":{"_id": "Dune"}}
-{"name": "Dune", "author": "Frank Herbert", "release_date": "1965-06-01", "page_count": 604}
---------------------------------------------------
-
-And now you can execute SQL using the <<sql-search-api,SQL search API>>:
-
-[source,console]
---------------------------------------------------
-POST /_sql?format=txt
-{
-  "query": "SELECT * FROM library WHERE release_date < '2000-01-01'"
-}
---------------------------------------------------
-// TEST[continued]
-
-Which should return something along the lines of:
-
-[source,text]
---------------------------------------------------
-    author     |     name      |  page_count   | release_date
----------------+---------------+---------------+------------------------
-Dan Simmons    |Hyperion       |482            |1989-05-26T00:00:00.000Z
-Frank Herbert  |Dune           |604            |1965-06-01T00:00:00.000Z
---------------------------------------------------
-// TESTRESPONSE[s/\\|/\\\\|/ s/\\+/\\\\+/]
-// TESTRESPONSE[non_json]
-
-You can also use the <<sql-cli>>. There is a script to start it
-shipped in x-pack's bin directory:
-
-[source,bash]
---------------------------------------------------
-\$ ./bin/elasticsearch-sql-cli
---------------------------------------------------
-
-From there you can run the same query:
-
-[source,sqlcli]
---------------------------------------------------
-sql> SELECT * FROM library WHERE release_date < '2000-01-01';
-    author     |     name      |  page_count   | release_date
----------------+---------------+---------------+------------------------
-Dan Simmons    |Hyperion       |482            |1989-05-26T00:00:00.000Z
-Frank Herbert  |Dune           |604            |1965-06-01T00:00:00.000Z
---------------------------------------------------
+    def "task fails on same doc source file with supported different extension"() {
+        given:
+        def build = ProjectBuilder.builder().build()
+        def task = build.tasks.register("restTestFromSnippet", RestTestsFromDocSnippetTask).get()
+        task.expectedUnconvertedCandidates = []
+        task.docs = build.fileTree(new File(tempDir, "docs"))
+        task.testRoot.convention(build.getLayout().buildDirectory.dir("rest-tests"));
+        docFile('docs/example-2.asciidoc', SAMPLE_TEST_DOCS['example-2.asciidoc'])
+        docFile('docs/example-2.mdx', SAMPLE_TEST_DOCS['example-2.mdx'])
+        task.getSetups().put(
+            "seats", """
+'''
+  - do:
+        indices.create:
+          index: seats
+          body:
+            settings:
+              number_of_shards: 1
+              number_of_replicas: 0
+            mappings:
+              properties:
+                theatre:
+                  type: keyword
 """
         )
-        docFile(
-            "docs/ml-update-snapshot.asciidoc",
-            """
-[role="xpack"]
-[[ml-update-snapshot]]
-= Update model snapshots API
-++++
-<titleabbrev>Update model snapshots</titleabbrev>
-++++
-
-Updates certain properties of a snapshot.
-
-[[ml-update-snapshot-request]]
-== {api-request-title}
-
-`POST _ml/anomaly_detectors/<job_id>/model_snapshots/<snapshot_id>/_update`
-
-[[ml-update-snapshot-prereqs]]
-== {api-prereq-title}
-
-Requires the `manage_ml` cluster privilege. This privilege is included in the
-`machine_learning_admin` built-in role.
-
-[[ml-update-snapshot-path-parms]]
-== {api-path-parms-title}
-
-`<job_id>`::
-(Required, string)
-include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection]
-
-`<snapshot_id>`::
-(Required, string)
-include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=snapshot-id]
-
-[[ml-update-snapshot-request-body]]
-== {api-request-body-title}
-
-The following properties can be updated after the model snapshot is created:
-
-`description`::
-(Optional, string) A description of the model snapshot.
-
-`retain`::
-(Optional, Boolean)
-include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=retain]
-
-
-[[ml-update-snapshot-example]]
-== {api-examples-title}
-
-[source,console]
---------------------------------------------------
-POST
-_ml/anomaly_detectors/it_ops_new_logs/model_snapshots/1491852978/_update
-{
-  "description": "Snapshot 1",
-  "retain": true
-}
---------------------------------------------------
-// TEST[skip:todo]
+        when:
+        task.getActions().forEach { it.execute(task) }
 
-When the snapshot is updated, you receive the following results:
-[source,js]
-----
-{
-  "acknowledged": true,
-  "model": {
-    "job_id": "it_ops_new_logs",
-    "timestamp": 1491852978000,
-    "description": "Snapshot 1",
-...
-    "retain": true
-  }
-}
-----
+        then:
+        def e = thrown(GradleException)
+        e.message == "Found multiple files with the same name 'example-2' but different extensions: [asciidoc, mdx]"
+    }
 
+    def "can run in migration mode to compare same doc source file with supported different extension"() {
+        given:
+        def build = ProjectBuilder.builder().build()
+        def task = build.tasks.register("restTestFromSnippet", RestTestsFromDocSnippetTask).get()
+        task.expectedUnconvertedCandidates = []
+        task.migrationMode = true
+        task.docs = build.fileTree(new File(tempDir, "docs"))
+        task.testRoot.convention(build.getLayout().buildDirectory.dir("rest-tests"));
+        docFile('docs/example-2.asciidoc', SAMPLE_TEST_DOCS['example-2.asciidoc'])
+        docFile('docs/example-2.mdx', SAMPLE_TEST_DOCS['example-2.mdx'])
+        task.getSetups().put(
+            "seats", """
+'''
+  - do:
+        indices.create:
+          index: seats
+          body:
+            settings:
+              number_of_shards: 1
+              number_of_replicas: 0
+            mappings:
+              properties:
+                theatre:
+                  type: keyword
 """
         )
+        when:
+        task.getActions().forEach { it.execute(task) }
 
-        docFile(
-            "docs/painless-debugging.asciidoc",
-            """
-
-[[painless-debugging]]
-=== Painless Debugging
-
-==== Debug.Explain
-
-Painless doesn't have a
-{wikipedia}/Read%E2%80%93eval%E2%80%93print_loop[REPL]
-and while it'd be nice for it to have one day, it wouldn't tell you the
-whole story around debugging painless scripts embedded in Elasticsearch because
-the data that the scripts have access to or "context" is so important. For now
-the best way to debug embedded scripts is by throwing exceptions at choice
-places. While you can throw your own exceptions
-(`throw new Exception('whatever')`), Painless's sandbox prevents you from
-accessing useful information like the type of an object. So Painless has a
-utility method, `Debug.explain` which throws the exception for you. For
-example, you can use {ref}/search-explain.html[`_explain`] to explore the
-context available to a {ref}/query-dsl-script-query.html[script query].
-
-[source,console]
----------------------------------------------------------
-PUT /hockey/_doc/1?refresh
-{"first":"johnny","last":"gaudreau","goals":[9,27,1],"assists":[17,46,0],"gp":[26,82,1]}
-
-POST /hockey/_explain/1
-{
-  "query": {
-    "script": {
-      "script": "Debug.explain(doc.goals)"
-    }
-  }
-}
----------------------------------------------------------
-// TEST[s/_explain\\/1/_explain\\/1?error_trace=false/ catch:/painless_explain_error/]
-// The test system sends error_trace=true by default for easier debugging so
-// we have to override it to get a normal shaped response
-
-Which shows that the class of `doc.first` is
-`org.elasticsearch.index.fielddata.ScriptDocValues.Longs` by responding with:
-
-[source,console-result]
----------------------------------------------------------
-{
-   "error": {
-      "type": "script_exception",
-      "to_string": "[1, 9, 27]",
-      "painless_class": "org.elasticsearch.index.fielddata.ScriptDocValues.Longs",
-      "java_class": "org.elasticsearch.index.fielddata.ScriptDocValues\$Longs",
-      ...
-   },
-   "status": 400
-}
----------------------------------------------------------
-// TESTRESPONSE[s/\\.\\.\\./"script_stack": \$body.error.script_stack, "script": \$body.error.script, "lang": \$body.error.lang, "position": \$body.error.position, "caused_by": \$body.error.caused_by, "root_cause": \$body.error.root_cause, "reason": \$body.error.reason/]
-
-You can use the same trick to see that `_source` is a `LinkedHashMap`
-in the `_update` API:
-
-[source,console]
----------------------------------------------------------
-POST /hockey/_update/1
-{
-  "script": "Debug.explain(ctx._source)"
-}
----------------------------------------------------------
-// TEST[continued s/_update\\/1/_update\\/1?error_trace=false/ catch:/painless_explain_error/]
-
-The response looks like:
-
-[source,console-result]
----------------------------------------------------------
-{
-  "error" : {
-    "root_cause": ...,
-    "type": "illegal_argument_exception",
-    "reason": "failed to execute script",
-    "caused_by": {
-      "type": "script_exception",
-      "to_string": "{gp=[26, 82, 1], last=gaudreau, assists=[17, 46, 0], first=johnny, goals=[9, 27, 1]}",
-      "painless_class": "java.util.LinkedHashMap",
-      "java_class": "java.util.LinkedHashMap",
-      ...
+        then:
+        new File(task.getTestRoot().get().getAsFile(), "rest-api-spec/test/example-2.asciidoc.yml").exists()
+        new File(task.getTestRoot().get().getAsFile(), "rest-api-spec/test/example-2.mdx.yml").exists()
     }
-  },
-  "status": 400
-}
----------------------------------------------------------
-// TESTRESPONSE[s/"root_cause": \\.\\.\\./"root_cause": \$body.error.root_cause/]
-// TESTRESPONSE[s/\\.\\.\\./"script_stack": \$body.error.caused_by.script_stack, "script": \$body.error.caused_by.script, "lang": \$body.error.caused_by.lang, "position": \$body.error.caused_by.position, "caused_by": \$body.error.caused_by.caused_by, "reason": \$body.error.caused_by.reason/]
-// TESTRESPONSE[s/"to_string": ".+"/"to_string": \$body.error.caused_by.to_string/]
-
-Once you have a class you can go to <<painless-api-reference>> to see a list of
-available methods.
 
+    def "fails in migration mode for same doc source file with different extension generates different spec"() {
+        given:
+        def build = ProjectBuilder.builder().build()
+        def task = build.tasks.register("restTestFromSnippet", RestTestsFromDocSnippetTask).get()
+        task.getMigrationMode().set(true)
+        task.docs = build.fileTree(new File(tempDir, "docs"))
+        task.testRoot.convention(build.getLayout().buildDirectory.dir("rest-tests"));
+        docFile('docs/example-2.asciidoc', SAMPLE_TEST_DOCS['example-2.asciidoc'])
+        docFile('docs/example-2.mdx', SAMPLE_TEST_DOCS['example-2-different.mdx'])
+        task.getSetups().put(
+            "seats", """
+'''
+  - do:
+        indices.create:
+          index: seats
+          body:
+            settings:
+              number_of_shards: 1
+              number_of_replicas: 0
+            mappings:
+              properties:
+                theatre:
+                  type: keyword
 """
         )
-        docFile(
-            "docs/reference/security/authorization/run-as-privilege.asciidoc",
-            """[role="xpack"]
-[[run-as-privilege]]
-= Submitting requests on behalf of other users
-
-{es} roles support a `run_as` privilege that enables an authenticated user to
-submit requests on behalf of other users. For example, if your external
-application is trusted to authenticate users, {es} can authenticate the external
-application and use the _run as_ mechanism to issue authorized requests as
-other users without having to re-authenticate each user.
-
-To "run as" (impersonate) another user, the first user (the authenticating user)
-must be authenticated by a mechanism that supports run-as delegation. The second
-user (the `run_as` user) must be authorized by a mechanism that supports
-delegated run-as lookups by username.
-
-The `run_as` privilege essentially operates like a secondary form of
-<<authorization_realms,delegated authorization>>. Delegated authorization applies
-to the authenticating user, and the `run_as` privilege applies to the user who
-is being impersonated.
-
-Authenticating user::
---
-For the authenticating user, the following realms (plus API keys) all support
-`run_as` delegation: `native`, `file`, Active Directory, JWT, Kerberos, LDAP and
-PKI.
-
-Service tokens, the {es} Token Service, SAML 2.0, and OIDC 1.0 do not
-support `run_as` delegation.
---
-
-`run_as` user::
---
-{es} supports `run_as` for any realm that supports user lookup.
-Not all realms support user lookup. Refer to the list of <<user-lookup,supported realms>>
-and ensure that the realm you wish to use is configured in a manner that
-supports user lookup.
-
-The `run_as` user must be retrieved from a <<realms,realm>> - it is not
-possible to run as a
-<<service-accounts,service account>>,
-<<token-authentication-api-key,API key>> or
-<<token-authentication-access-token,access token>>.
---
-
-To submit requests on behalf of other users, you need to have the `run_as`
-privilege in your <<defining-roles,roles>>. For example, the following request
-creates a `my_director` role that grants permission to submit request on behalf
-of `jacknich` or `redeniro`:
-
-[source,console]
-----
-POST /_security/role/my_director?refresh=true
-{
-  "cluster": ["manage"],
-  "indices": [
-    {
-      "names": [ "index1", "index2" ],
-      "privileges": [ "manage" ]
-    }
-  ],
-  "run_as": [ "jacknich", "rdeniro" ],
-  "metadata" : {
-    "version" : 1
-  }
-}
-----
-
-To submit a request as another user, you specify the user in the
-`es-security-runas-user` request header. For example:
-
-[source,sh]
-----
-curl -H "es-security-runas-user: jacknich" -u es-admin -X GET http://localhost:9200/
-----
-
-The `run_as` user passed in through the `es-security-runas-user` header must be
-available from a realm that supports delegated authorization lookup by username.
-Realms that don't support user lookup can't be used by `run_as` delegation from
-other realms.
-
-For example, JWT realms can authenticate external users specified in JWTs, and
-execute requests as a `run_as` user in the `native` realm. {es} will retrieve the
-indicated `runas` user and execute the request as that user using their roles.
-
-[[run-as-privilege-apply]]
-== Apply the `run_as` privilege to roles
-You can apply the `run_as` privilege when creating roles with the
-<<security-api-put-role,create or update roles API>>. Users who are assigned
-a role that contains the `run_as` privilege inherit all privileges from their
-role, and can also submit requests on behalf of the indicated users.
-
-NOTE: Roles for the authenticated user and the `run_as` user are not merged. If
-a user authenticates without specifying the `run_as` parameter, only the
-authenticated user's roles are used. If a user authenticates and their roles
-include the `run_as` parameter, only the `run_as` user's roles are used.
-
-After a user successfully authenticates to {es}, an authorization process determines whether the user behind an incoming request is allowed to run
-that request. If the authenticated user has the `run_as` privilege in their list
-of permissions and specifies the run-as header, {es} _discards_ the authenticated
-user and associated roles. It then looks in each of the configured realms in the
-realm chain until it finds the username that's associated with the `run_as` user,
-and uses those roles to execute any requests.
-
-Consider an admin role and an analyst role. The admin role has higher privileges,
-but might also want to submit requests as another user to test and verify their
-permissions.
-
-First, we'll create an admin role named `my_admin_role`. This role has `manage`
-<<security-privileges,privileges>> on the entire cluster, and on a subset of
-indices. This role also contains the `run_as` privilege, which enables any user
-with this role to submit requests on behalf of the specified `analyst_user`.
+        when:
+        task.getActions().forEach { it.execute(task) }
 
-[source,console]
-----
-POST /_security/role/my_admin_role?refresh=true
-{
-  "cluster": ["manage"],
-  "indices": [
-    {
-      "names": [ "index1", "index2" ],
-      "privileges": [ "manage" ]
-    }
-  ],
-  "applications": [
-    {
-      "application": "myapp",
-      "privileges": [ "admin", "read" ],
-      "resources": [ "*" ]
+        then:
+        new File(task.getTestRoot().get().getAsFile(), "rest-api-spec/test/example-2.asciidoc.yml").exists()
+        new File(task.getTestRoot().get().getAsFile(), "rest-api-spec/test/example-2.mdx.yml").exists()
     }
-  ],
-  "run_as": [ "analyst_user" ],
-  "metadata" : {
-    "version" : 1
-  }
-}
-----
 
-Next, we'll create an analyst role named `my_analyst_role`, which has more
-restricted `monitor` cluster privileges and `manage` privileges on a subset of
-indices.
-
-[source,console]
-----
-POST /_security/role/my_analyst_role?refresh=true
-{
-  "cluster": [ "monitor"],
-  "indices": [
-    {
-      "names": [ "index1", "index2" ],
-      "privileges": ["manage"]
-    }
-  ],
-  "applications": [
-    {
-      "application": "myapp",
-      "privileges": [ "read" ],
-      "resources": [ "*" ]
+    File docFile(String fileName, String docContent) {
+        def file = tempDir.toPath().resolve(fileName).toFile()
+        file.parentFile.mkdirs()
+        file.text = docContent
+        return file
     }
-  ],
-  "metadata" : {
-    "version" : 1
-  }
-}
-----
-
-We'll create an administrator user and assign them the role named `my_admin_role`,
-which allows this user to submit requests as the `analyst_user`.
-
-[source,console]
-----
-POST /_security/user/admin_user?refresh=true
-{
-  "password": "l0ng-r4nd0m-p@ssw0rd",
-  "roles": [ "my_admin_role" ],
-  "full_name": "Eirian Zola",
-  "metadata": { "intelligence" : 7}
-}
-----
-
-We can also create an analyst user and assign them the role named
-`my_analyst_role`.
-
-[source,console]
-----
-POST /_security/user/analyst_user?refresh=true
-{
-  "password": "l0nger-r4nd0mer-p@ssw0rd",
-  "roles": [ "my_analyst_role" ],
-  "full_name": "Monday Jaffe",
-  "metadata": { "innovation" : 8}
-}
-----
-
-You can then authenticate to {es} as the `admin_user` or `analyst_user`. However, the `admin_user` could optionally submit requests on
-behalf of the `analyst_user`. The following request authenticates to {es} with a
-`Basic` authorization token and submits the request as the `analyst_user`:
-
-[source,sh]
-----
-curl -s -X GET -H "Authorization: Basic YWRtaW5fdXNlcjpsMG5nLXI0bmQwbS1wQHNzdzByZA==" -H "es-security-runas-user: analyst_user" https://localhost:9200/_security/_authenticate
-----
-
-The response indicates that the `analyst_user` submitted this request, using the
-`my_analyst_role` that's assigned to that user. When the `admin_user` submitted
-the request, {es} authenticated that user, discarded their roles, and then used
-the roles of the `run_as` user.
-
-[source,sh]
-----
-{"username":"analyst_user","roles":["my_analyst_role"],"full_name":"Monday Jaffe","email":null,
-"metadata":{"innovation":8},"enabled":true,"authentication_realm":{"name":"native",
-"type":"native"},"lookup_realm":{"name":"native","type":"native"},"authentication_type":"realm"}
-%
-----
-
-The `authentication_realm` and `lookup_realm` in the response both specify
-the `native` realm because both the `admin_user` and `analyst_user` are from
-that realm. If the two users are in different realms, the values for
-`authentication_realm` and `lookup_realm` are different (such as `pki` and
-`native`).
-"""
-        )
 
+    String normalizeRestSpec(String inputString) {
+        def withNormalizedLines = inputString.replaceAll(/"line_\d+":/, "\"line_0\":")
+        return withNormalizedLines
     }
 }
diff --git a/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/doc/SnippetBuilderSpec.groovy b/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/doc/SnippetBuilderSpec.groovy
new file mode 100644
index 0000000000000..278728ec176c1
--- /dev/null
+++ b/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/doc/SnippetBuilderSpec.groovy
@@ -0,0 +1,107 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.gradle.internal.doc
+
+import spock.lang.Specification
+import spock.lang.Unroll
+
+import org.gradle.api.InvalidUserDataException
+
+class SnippetBuilderSpec extends Specification {
+
+    @Unroll
+    def "checks for valid json for #languageParam"() {
+        when:
+        def snippet1 = snippetBuilder().withLanguage(languageParam).withTestResponse(true).withConsole(true)
+            .withContent(
+                """{
+    "name": "John Doe",
+    "age": 30,
+    "isMarried": true,
+    "address": {
+        "street": "123 Main Street",
+        "city": "Springfield",
+        "state": "IL",
+        "zip": "62701"
+    },
+    "hobbies": ["Reading", "Cooking", "Traveling"]
+}"""
+            ).build()
+        then:
+        snippet1 != null
+
+
+        when:
+        snippetBuilder().withLanguage(languageParam).withTestResponse(true).withConsole(true)
+            .withContent(
+                "some no valid json"
+            ).build()
+
+        then:
+        def e = thrown(InvalidUserDataException)
+        e.message.contains("Invalid json in")
+
+        when:
+        def snippet2 = snippetBuilder().withLanguage(languageParam).withTestResponse(true).withConsole(true)
+            .withSkip("skipping")
+            .withContent(
+                "some no valid json"
+            ).build()
+
+        then:
+        snippet2 != null
+
+        where:
+        languageParam << ["js", "console-result"]
+    }
+
+    def "language must be defined"() {
+        when:
+        snippetBuilder().withContent("snippet-content").build()
+        then:
+        def e = thrown(InvalidUserDataException)
+        e.message.contains("Snippet missing a language.")
+    }
+
+    def "handles snippets with curl"() {
+        expect:
+        snippetBuilder().withLanguage("sh")
+            .withName("snippet-name-1")
+            .withContent("curl substDefault subst")
+            .build()
+            .curl() == true
+
+    }
+
+    def "snippet builder handles substitutions"() {
+        when:
+        def snippet = snippetBuilder().withLanguage("console").withContent("snippet-content substDefault subst")
+            .withSubstitutions([substDefault: "\$body", subst: 'substValue']).build()
+
+        then:
+        snippet.contents == "snippet-content \$body substValue"
+    }
+
+    def "test snippets with no curl no console"() {
+        when:
+        snippetBuilder()
+            .withConsole(false)
+            .withLanguage("shell")
+            .withContent("hello substDefault subst")
+            .build()
+        then:
+        def e = thrown(InvalidUserDataException)
+        e.message.contains("No need for NOTCONSOLE if snippet doesn't contain `curl`")
+    }
+
+    SnippetBuilder snippetBuilder() {
+        return new SnippetBuilder()
+    }
+
+}
diff --git a/build-tools/src/testFixtures/java/org/elasticsearch/gradle/internal/test/TestUtils.java b/build-tools/src/testFixtures/java/org/elasticsearch/gradle/internal/test/TestUtils.java
index 17d3375c4e83c..e4513bfe4c0f0 100644
--- a/build-tools/src/testFixtures/java/org/elasticsearch/gradle/internal/test/TestUtils.java
+++ b/build-tools/src/testFixtures/java/org/elasticsearch/gradle/internal/test/TestUtils.java
@@ -14,6 +14,10 @@
 
 public class TestUtils {
 
+    public static String normalizeString(String input) {
+        return normalizeString(input, new File("."));
+    }
+
     public static String normalizeString(String input, File projectRootDir) {
         try {
             String canonicalNormalizedPathPrefix = projectRootDir.getCanonicalPath().replace('\\', '/');
diff --git a/docs/build.gradle b/docs/build.gradle
index e38b0129b219e..0eba980e8cc31 100644
--- a/docs/build.gradle
+++ b/docs/build.gradle
@@ -1,6 +1,6 @@
 import org.elasticsearch.gradle.Version
 import org.elasticsearch.gradle.internal.info.BuildParams
-
+import org.elasticsearch.gradle.internal.doc.DocSnippetTask
 import static org.elasticsearch.gradle.testclusters.TestDistribution.DEFAULT
 
 /*
@@ -16,6 +16,7 @@ apply plugin: 'elasticsearch.rest-resources'
 
 ext.docsFileTree = fileTree(projectDir) {
   include '**/*.asciidoc'
+  include '**/*.mdx'
   // That is where the snippets go, not where they come from!
   exclude 'build/**'
   exclude 'build-idea/**'
@@ -37,7 +38,7 @@ ext.docsFileTree = fileTree(projectDir) {
 
 /* List of files that have snippets that will not work until platinum tests can occur ... */
 tasks.named("buildRestTests").configure {
-  expectedUnconvertedCandidates = [
+  getExpectedUnconvertedCandidates().addAll(
           'reference/ml/anomaly-detection/ml-configuring-transform.asciidoc',
           'reference/ml/anomaly-detection/apis/delete-calendar-event.asciidoc',
           'reference/ml/anomaly-detection/apis/get-bucket.asciidoc',
@@ -58,7 +59,7 @@ tasks.named("buildRestTests").configure {
           'reference/rest-api/watcher/put-watch.asciidoc',
           'reference/rest-api/watcher/stats.asciidoc',
           'reference/watcher/example-watches/watching-time-series-data.asciidoc'
-  ]
+  )
 }
 
 restResources {
@@ -176,16 +177,8 @@ tasks.named("forbiddenPatterns").configure {
   exclude '**/*.mmdb'
 }
 
-tasks.named("buildRestTests").configure {
-  docs = docsFileTree
-}
-
-tasks.named("listSnippets").configure {
-  docs = docsFileTree
-}
-
-tasks.named("listConsoleCandidates").configure {
-  docs = docsFileTree
+tasks.withType(DocSnippetTask).configureEach {
+    docs = docsFileTree
 }
 
 Closure setupMyIndex = { String name, int count ->

From 6dfb78f6f19a07fe8b50c8c0b8e4f12375ccaab9 Mon Sep 17 00:00:00 2001
From: Henning Andersen <33268011+henningandersen@users.noreply.github.com>
Date: Thu, 25 Apr 2024 09:44:58 +0200
Subject: [PATCH 15/58] Fix BlobCacheUtils.toPageAlignedSize (#107858)

toPageAlignedSize could round wrongly when the length did not fit
in an integer.
---
 .../java/org/elasticsearch/blobcache/BlobCacheUtils.java  | 2 +-
 .../org/elasticsearch/blobcache/BlobCacheUtilsTests.java  | 8 ++++++++
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/BlobCacheUtils.java b/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/BlobCacheUtils.java
index 940578d3dafb2..ff273d99d3c41 100644
--- a/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/BlobCacheUtils.java
+++ b/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/BlobCacheUtils.java
@@ -36,7 +36,7 @@ public static int toIntBytes(long l) {
      * Rounds the length up so that it is aligned on the next page size (defined by SharedBytes.PAGE_SIZE). For example
      */
     public static long toPageAlignedSize(long length) {
-        int remainder = (int) length % SharedBytes.PAGE_SIZE;
+        int remainder = (int) (length % SharedBytes.PAGE_SIZE);
         if (remainder > 0L) {
             return length + (SharedBytes.PAGE_SIZE - remainder);
         }
diff --git a/x-pack/plugin/blob-cache/src/test/java/org/elasticsearch/blobcache/BlobCacheUtilsTests.java b/x-pack/plugin/blob-cache/src/test/java/org/elasticsearch/blobcache/BlobCacheUtilsTests.java
index 2f78797e556ac..2b615dad05655 100644
--- a/x-pack/plugin/blob-cache/src/test/java/org/elasticsearch/blobcache/BlobCacheUtilsTests.java
+++ b/x-pack/plugin/blob-cache/src/test/java/org/elasticsearch/blobcache/BlobCacheUtilsTests.java
@@ -6,8 +6,10 @@
  */
 package org.elasticsearch.blobcache;
 
+import org.elasticsearch.blobcache.shared.SharedBytes;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.test.ESTestCase;
+import org.hamcrest.Matchers;
 
 import java.io.EOFException;
 import java.nio.ByteBuffer;
@@ -19,4 +21,10 @@ public void testReadSafeThrows() {
         final int remaining = randomIntBetween(1, 1025);
         expectThrows(EOFException.class, () -> BlobCacheUtils.readSafe(BytesArray.EMPTY.streamInput(), buffer, 0, remaining));
     }
+
+    public void testToPageAlignedSize() {
+        long value = randomLongBetween(0, Long.MAX_VALUE - SharedBytes.PAGE_SIZE);
+        long expected = ((value - 1) / SharedBytes.PAGE_SIZE + 1) * SharedBytes.PAGE_SIZE;
+        assertThat(BlobCacheUtils.toPageAlignedSize(value), Matchers.equalTo(expected));
+    }
 }

From b331e8fcbd4476ebda7666223c9fda515542a5e2 Mon Sep 17 00:00:00 2001
From: Armin Braun <me@obrown.io>
Date: Thu, 25 Apr 2024 11:27:34 +0200
Subject: [PATCH 16/58] Fix random access reads from ByteArrayIndexInput
 (#107885)

Fix random access read behavior for ByteArrayIndexInput and enhance our
generic test to test random access reads a little if it's a random
access input. Also, this adjustment to how `pos` works should maybe come
with a mild performance gain.
---
 .../lucene/store/ByteArrayIndexInput.java     | 15 +++++-----
 .../lucene/store/ESIndexInputTestCase.java    | 29 ++++++++++++++++---
 2 files changed, 33 insertions(+), 11 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/common/lucene/store/ByteArrayIndexInput.java b/server/src/main/java/org/elasticsearch/common/lucene/store/ByteArrayIndexInput.java
index 1cf90d1741203..3c0b29f483479 100644
--- a/server/src/main/java/org/elasticsearch/common/lucene/store/ByteArrayIndexInput.java
+++ b/server/src/main/java/org/elasticsearch/common/lucene/store/ByteArrayIndexInput.java
@@ -35,6 +35,7 @@ public ByteArrayIndexInput(String resourceDesc, byte[] bytes, int offset, int le
         super(resourceDesc);
         this.bytes = bytes;
         this.offset = offset;
+        this.pos = offset;
         this.length = length;
     }
 
@@ -43,7 +44,7 @@ public void close() throws IOException {}
 
     @Override
     public long getFilePointer() {
-        return pos;
+        return pos - offset;
     }
 
     @Override
@@ -57,7 +58,7 @@ private int position(long p) throws EOFException {
         } else if (p > length) {
             throw new EOFException("seek past EOF");
         }
-        return (int) p;
+        return (int) p + offset;
     }
 
     @Override
@@ -108,7 +109,7 @@ public byte readByte() throws IOException {
         if (pos >= offset + length) {
             throw new EOFException("seek past EOF");
         }
-        return bytes[offset + pos++];
+        return bytes[pos++];
     }
 
     @Override
@@ -116,14 +117,14 @@ public void readBytes(final byte[] b, final int offset, int len) throws IOExcept
         if (pos + len > this.offset + length) {
             throw new EOFException("seek past EOF");
         }
-        System.arraycopy(bytes, this.offset + pos, b, offset, len);
+        System.arraycopy(bytes, pos, b, offset, len);
         pos += len;
     }
 
     @Override
     public short readShort() throws IOException {
         try {
-            return (short) BitUtil.VH_LE_SHORT.get(bytes, pos + offset);
+            return (short) BitUtil.VH_LE_SHORT.get(bytes, pos);
         } finally {
             pos += Short.BYTES;
         }
@@ -132,7 +133,7 @@ public short readShort() throws IOException {
     @Override
     public int readInt() throws IOException {
         try {
-            return (int) BitUtil.VH_LE_INT.get(bytes, pos + offset);
+            return (int) BitUtil.VH_LE_INT.get(bytes, pos);
         } finally {
             pos += Integer.BYTES;
         }
@@ -141,7 +142,7 @@ public int readInt() throws IOException {
     @Override
     public long readLong() throws IOException {
         try {
-            return (long) BitUtil.VH_LE_LONG.get(bytes, pos + offset);
+            return (long) BitUtil.VH_LE_LONG.get(bytes, pos);
         } finally {
             pos += Long.BYTES;
         }
diff --git a/test/framework/src/main/java/org/elasticsearch/common/lucene/store/ESIndexInputTestCase.java b/test/framework/src/main/java/org/elasticsearch/common/lucene/store/ESIndexInputTestCase.java
index 1ec3997553f77..2b4e7fd4c7517 100644
--- a/test/framework/src/main/java/org/elasticsearch/common/lucene/store/ESIndexInputTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/common/lucene/store/ESIndexInputTestCase.java
@@ -8,6 +8,7 @@
 package org.elasticsearch.common.lucene.store;
 
 import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.RandomAccessInput;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.common.settings.Settings;
@@ -73,16 +74,36 @@ protected byte[] randomReadAndSlice(IndexInput indexInput, int length) throws IO
             switch (readStrategy) {
                 case 0, 1, 2, 3:
                     if (length - readPos >= Long.BYTES && readStrategy <= 0) {
-                        ByteBuffer.wrap(output, readPos, Long.BYTES).order(ByteOrder.LITTLE_ENDIAN).putLong(indexInput.readLong());
+                        long read = indexInput.readLong();
+                        ByteBuffer.wrap(output, readPos, Long.BYTES).order(ByteOrder.LITTLE_ENDIAN).putLong(read);
                         readPos += Long.BYTES;
+                        if (indexInput instanceof RandomAccessInput randomAccessInput) {
+                            assertEquals(read, randomAccessInput.readLong(indexInput.getFilePointer() - Long.BYTES));
+                            indexInput.seek(readPos);
+                        }
                     } else if (length - readPos >= Integer.BYTES && readStrategy <= 1) {
-                        ByteBuffer.wrap(output, readPos, Integer.BYTES).order(ByteOrder.LITTLE_ENDIAN).putInt(indexInput.readInt());
+                        int read = indexInput.readInt();
+                        ByteBuffer.wrap(output, readPos, Integer.BYTES).order(ByteOrder.LITTLE_ENDIAN).putInt(read);
                         readPos += Integer.BYTES;
+                        if (indexInput instanceof RandomAccessInput randomAccessInput) {
+                            assertEquals(read, randomAccessInput.readInt(indexInput.getFilePointer() - Integer.BYTES));
+                            indexInput.seek(readPos);
+                        }
                     } else if (length - readPos >= Short.BYTES && readStrategy <= 2) {
-                        ByteBuffer.wrap(output, readPos, Short.BYTES).order(ByteOrder.LITTLE_ENDIAN).putShort(indexInput.readShort());
+                        short read = indexInput.readShort();
+                        ByteBuffer.wrap(output, readPos, Short.BYTES).order(ByteOrder.LITTLE_ENDIAN).putShort(read);
                         readPos += Short.BYTES;
+                        if (indexInput instanceof RandomAccessInput randomAccessInput) {
+                            assertEquals(read, randomAccessInput.readShort(indexInput.getFilePointer() - Short.BYTES));
+                            indexInput.seek(readPos);
+                        }
                     } else {
-                        output[readPos++] = indexInput.readByte();
+                        byte read = indexInput.readByte();
+                        output[readPos++] = read;
+                        if (indexInput instanceof RandomAccessInput randomAccessInput) {
+                            assertEquals(read, randomAccessInput.readByte(indexInput.getFilePointer() - 1));
+                            indexInput.seek(readPos);
+                        }
                     }
                     break;
                 case 4:

From b6b20a5d6f3580b13803ca5c660584ded829d6a8 Mon Sep 17 00:00:00 2001
From: Simon Cooper <simon.cooper@elastic.co>
Date: Thu, 25 Apr 2024 10:37:02 +0100
Subject: [PATCH 17/58] Update several references to IndexVersion.toString to
 use toReleaseVersion (#107828)

---
 docs/changelog/107828.yaml                            |  6 ++++++
 .../elasticsearch/upgrades/FullClusterRestartIT.java  |  9 ++++++---
 .../org/elasticsearch/upgrades/FeatureUpgradeIT.java  |  2 +-
 .../system/indices/FeatureUpgradeApiIT.java           |  2 +-
 .../elasticsearch/snapshots/RestoreSnapshotIT.java    |  4 ++--
 .../migration/GetFeatureUpgradeStatusResponse.java    |  4 ++--
 .../action/admin/cluster/stats/VersionStats.java      |  2 +-
 .../cluster/coordination/NodeJoinExecutor.java        |  8 ++++----
 .../elasticsearch/cluster/metadata/IndexMetadata.java |  6 +++---
 .../cluster/metadata/IndexMetadataVerifier.java       |  4 ++--
 .../elasticsearch/cluster/routing/RecoverySource.java |  2 +-
 .../java/org/elasticsearch/env/NodeEnvironment.java   |  4 ++--
 .../java/org/elasticsearch/index/IndexSettings.java   | 11 ++++++++---
 .../repositories/RepositoriesModule.java              |  4 ++--
 .../repositories/blobstore/BlobStoreRepository.java   |  2 +-
 .../org/elasticsearch/snapshots/SnapshotInfo.java     |  2 +-
 .../cluster/metadata/IndexMetadataVerifierTests.java  |  4 ++--
 .../org/elasticsearch/env/NodeEnvironmentTests.java   |  2 +-
 .../xpack/deprecation/IndexDeprecationChecks.java     |  2 +-
 .../deprecation/IndexDeprecationChecksTests.java      |  2 +-
 .../elasticsearch/oldrepos/OldRepositoryAccessIT.java |  4 ++--
 21 files changed, 50 insertions(+), 36 deletions(-)
 create mode 100644 docs/changelog/107828.yaml

diff --git a/docs/changelog/107828.yaml b/docs/changelog/107828.yaml
new file mode 100644
index 0000000000000..ba0d44029203d
--- /dev/null
+++ b/docs/changelog/107828.yaml
@@ -0,0 +1,6 @@
+pr: 107828
+summary: Update several references to `IndexVersion.toString` to use `toReleaseVersion`
+area: Infra/Core
+type: bug
+issues:
+ - 107821
diff --git a/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartIT.java b/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartIT.java
index e94638bb17791..1a86947acab95 100644
--- a/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartIT.java
+++ b/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartIT.java
@@ -48,7 +48,6 @@
 import org.elasticsearch.xcontent.XContentParser;
 import org.elasticsearch.xcontent.XContentType;
 import org.elasticsearch.xcontent.json.JsonXContent;
-import org.hamcrest.Matchers;
 import org.junit.Before;
 import org.junit.ClassRule;
 import org.junit.rules.RuleChain;
@@ -81,8 +80,8 @@
 import static org.elasticsearch.test.MapMatcher.matchesMap;
 import static org.elasticsearch.transport.RemoteClusterService.REMOTE_CLUSTER_COMPRESS;
 import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder;
+import static org.hamcrest.Matchers.anyOf;
 import static org.hamcrest.Matchers.containsString;
-import static org.hamcrest.Matchers.either;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
@@ -1289,7 +1288,11 @@ private void checkSnapshot(String snapshotName, int count, String tookOnVersion,
         // the format can change depending on the ES node version running & this test code running
         assertThat(
             XContentMapValues.extractValue("snapshots.version", snapResponse),
-            either(Matchers.<Object>equalTo(List.of(tookOnVersion))).or(equalTo(List.of(tookOnIndexVersion.toString())))
+            anyOf(
+                equalTo(List.of(tookOnVersion)),
+                equalTo(List.of(tookOnIndexVersion.toString())),
+                equalTo(List.of(tookOnIndexVersion.toReleaseVersion()))
+            )
         );
 
         // Remove the routing setting and template so we can test restoring them.
diff --git a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/FeatureUpgradeIT.java b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/FeatureUpgradeIT.java
index 47be0e5efff62..4fe45c05b157b 100644
--- a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/FeatureUpgradeIT.java
+++ b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/FeatureUpgradeIT.java
@@ -99,7 +99,7 @@ public void testGetFeatureUpgradeStatus() throws Exception {
                     .orElse(Collections.emptyMap());
 
                 assertThat(feature, aMapWithSize(4));
-                assertThat(feature.get("minimum_index_version"), equalTo(getOldClusterIndexVersion().toString()));
+                assertThat(feature.get("minimum_index_version"), equalTo(getOldClusterIndexVersion().toReleaseVersion()));
 
                 // Feature migration happens only across major versions; also, we usually begin to require migrations once we start testing
                 // for the next major version upgrade (see e.g. #93666). Trying to express this with features may be problematic, so we
diff --git a/qa/system-indices/src/javaRestTest/java/org/elasticsearch/system/indices/FeatureUpgradeApiIT.java b/qa/system-indices/src/javaRestTest/java/org/elasticsearch/system/indices/FeatureUpgradeApiIT.java
index 48f987d0359f0..8c790fe1cde49 100644
--- a/qa/system-indices/src/javaRestTest/java/org/elasticsearch/system/indices/FeatureUpgradeApiIT.java
+++ b/qa/system-indices/src/javaRestTest/java/org/elasticsearch/system/indices/FeatureUpgradeApiIT.java
@@ -61,7 +61,7 @@ public void testGetFeatureUpgradedStatuses() throws Exception {
             .orElse(Collections.emptyMap());
 
         assertThat(testFeature.size(), equalTo(4));
-        assertThat(testFeature.get("minimum_index_version"), equalTo(IndexVersion.current().toString()));
+        assertThat(testFeature.get("minimum_index_version"), equalTo(IndexVersion.current().toReleaseVersion()));
         assertThat(testFeature.get("migration_status"), equalTo("NO_MIGRATION_NEEDED"));
         assertThat(testFeature.get("indices"), instanceOf(List.class));
 
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/RestoreSnapshotIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/RestoreSnapshotIT.java
index 6c452103cc014..5f9ad28b561f8 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/RestoreSnapshotIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/RestoreSnapshotIT.java
@@ -890,9 +890,9 @@ public void testFailOnAncientVersion() throws Exception {
             snapshotRestoreException.getMessage(),
             containsString(
                 "the snapshot was created with Elasticsearch version ["
-                    + oldVersion
+                    + oldVersion.toReleaseVersion()
                     + "] which is below the current versions minimum index compatibility version ["
-                    + IndexVersions.MINIMUM_COMPATIBLE
+                    + IndexVersions.MINIMUM_COMPATIBLE.toReleaseVersion()
                     + "]"
             )
         );
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/GetFeatureUpgradeStatusResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/GetFeatureUpgradeStatusResponse.java
index 29846fab10977..ffbd6b70e9ed9 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/GetFeatureUpgradeStatusResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/GetFeatureUpgradeStatusResponse.java
@@ -185,7 +185,7 @@ public void writeTo(StreamOutput out) throws IOException {
         public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
             builder.startObject();
             builder.field("feature_name", this.featureName);
-            builder.field("minimum_index_version", this.minimumIndexVersion.toString());
+            builder.field("minimum_index_version", this.minimumIndexVersion.toReleaseVersion());
             builder.field("migration_status", this.upgradeStatus);
             builder.startArray("indices");
             for (IndexInfo version : this.indexInfos) {
@@ -300,7 +300,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
 
             builder.startObject();
             builder.field("index", this.indexName);
-            builder.field("version", this.version.toString());
+            builder.field("version", this.version.toReleaseVersion());
             if (exception != null) {
                 builder.startObject("failure_cause");
                 {
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/VersionStats.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/VersionStats.java
index 703a650489bbd..12fe846a9e68b 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/VersionStats.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/VersionStats.java
@@ -168,7 +168,7 @@ static class SingleVersionStats implements ToXContentObject, Writeable, Comparab
         @Override
         public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
             builder.startObject();
-            builder.field("version", version.toString());
+            builder.field("version", version.toReleaseVersion());
             builder.field("index_count", indexCount);
             builder.field("primary_shard_count", primaryShardCount);
             builder.humanReadableField("total_primary_bytes", "total_primary_size", ByteSizeValue.ofBytes(totalPrimaryByteCount));
diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/NodeJoinExecutor.java b/server/src/main/java/org/elasticsearch/cluster/coordination/NodeJoinExecutor.java
index 480f1d5503d61..c35ea1279ac99 100644
--- a/server/src/main/java/org/elasticsearch/cluster/coordination/NodeJoinExecutor.java
+++ b/server/src/main/java/org/elasticsearch/cluster/coordination/NodeJoinExecutor.java
@@ -362,9 +362,9 @@ public static void ensureIndexCompatibility(IndexVersion minSupportedVersion, In
                     "index "
                         + idxMetadata.getIndex()
                         + " version not supported: "
-                        + idxMetadata.getCompatibilityVersion()
+                        + idxMetadata.getCompatibilityVersion().toReleaseVersion()
                         + " maximum compatible index version is: "
-                        + maxSupportedVersion
+                        + maxSupportedVersion.toReleaseVersion()
                 );
             }
             if (idxMetadata.getCompatibilityVersion().before(minSupportedVersion)) {
@@ -372,9 +372,9 @@ public static void ensureIndexCompatibility(IndexVersion minSupportedVersion, In
                     "index "
                         + idxMetadata.getIndex()
                         + " version not supported: "
-                        + idxMetadata.getCompatibilityVersion()
+                        + idxMetadata.getCompatibilityVersion().toReleaseVersion()
                         + " minimum compatible index version is: "
-                        + minSupportedVersion
+                        + minSupportedVersion.toReleaseVersion()
                 );
             }
         }
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java
index 560c6815d2252..678655252248f 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java
@@ -383,11 +383,11 @@ public void validate(final IndexVersion compatibilityVersion, final Map<Setting<
                     throw new IllegalArgumentException(
                         SETTING_VERSION_COMPATIBILITY
                             + " ["
-                            + compatibilityVersion
+                            + compatibilityVersion.toReleaseVersion()
                             + "] must be >= "
                             + SETTING_VERSION_CREATED
                             + " ["
-                            + createdVersion
+                            + createdVersion.toReleaseVersion()
                             + "]"
                     );
                 }
@@ -2629,7 +2629,7 @@ public static IndexMetadata legacyFromXContent(XContentParser parser) throws IOE
                             throw new IllegalStateException(
                                 "this method should only be used to parse older incompatible index metadata versions "
                                     + "but got "
-                                    + SETTING_INDEX_VERSION_COMPATIBILITY.get(settings)
+                                    + SETTING_INDEX_VERSION_COMPATIBILITY.get(settings).toReleaseVersion()
                             );
                         }
                         builder.settings(settings);
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifier.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifier.java
index 201d4afc0494c..641fc0e76311f 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifier.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifier.java
@@ -110,9 +110,9 @@ private static void checkSupportedVersion(IndexMetadata indexMetadata, IndexVers
                 "The index "
                     + indexMetadata.getIndex()
                     + " has current compatibility version ["
-                    + indexMetadata.getCompatibilityVersion()
+                    + indexMetadata.getCompatibilityVersion().toReleaseVersion()
                     + "] but the minimum compatible version is ["
-                    + minimumIndexCompatibilityVersion
+                    + minimumIndexCompatibilityVersion.toReleaseVersion()
                     + "]. It should be re-indexed in Elasticsearch "
                     + (Version.CURRENT.major - 1)
                     + ".x before upgrading to "
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/RecoverySource.java b/server/src/main/java/org/elasticsearch/cluster/routing/RecoverySource.java
index bb4eef2bd422d..0d288cf331d75 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/RecoverySource.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/RecoverySource.java
@@ -261,7 +261,7 @@ public Type getType() {
         public void addAdditionalFields(XContentBuilder builder, ToXContent.Params params) throws IOException {
             builder.field("repository", snapshot.getRepository())
                 .field("snapshot", snapshot.getSnapshotId().getName())
-                .field("version", version.toString())
+                .field("version", version.toReleaseVersion())
                 .field("index", index.getName())
                 .field("restoreUUID", restoreUUID);
         }
diff --git a/server/src/main/java/org/elasticsearch/env/NodeEnvironment.java b/server/src/main/java/org/elasticsearch/env/NodeEnvironment.java
index 291e9697def4a..e0948c688a19c 100644
--- a/server/src/main/java/org/elasticsearch/env/NodeEnvironment.java
+++ b/server/src/main/java/org/elasticsearch/env/NodeEnvironment.java
@@ -528,13 +528,13 @@ static void checkForIndexCompatibility(Logger logger, DataPath... dataPaths) thr
             String bestDowngradeVersion = getBestDowngradeVersion(metadata.previousNodeVersion().toString());
             throw new IllegalStateException(
                 "Cannot start this node because it holds metadata for indices with version ["
-                    + metadata.oldestIndexVersion()
+                    + metadata.oldestIndexVersion().toReleaseVersion()
                     + "] with which this node of version ["
                     + Build.current().version()
                     + "] is incompatible. Revert this node to version ["
                     + bestDowngradeVersion
                     + "] and delete any indices with versions earlier than ["
-                    + IndexVersions.MINIMUM_COMPATIBLE
+                    + IndexVersions.MINIMUM_COMPATIBLE.toReleaseVersion()
                     + "] before upgrading to version ["
                     + Build.current().version()
                     + "]. If all such indices have already been deleted, revert this node to version ["
diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java
index 6decd20e0a41f..aa92025f32428 100644
--- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java
+++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java
@@ -1114,16 +1114,21 @@ public synchronized boolean updateIndexMetadata(IndexMetadata indexMetadata) {
         final Settings newSettings = indexMetadata.getSettings();
         IndexVersion newIndexVersion = SETTING_INDEX_VERSION_CREATED.get(newSettings);
         if (version.equals(newIndexVersion) == false) {
-            throw new IllegalArgumentException("version mismatch on settings update expected: " + version + " but was: " + newIndexVersion);
+            throw new IllegalArgumentException(
+                "version mismatch on settings update expected: "
+                    + version.toReleaseVersion()
+                    + " but was: "
+                    + newIndexVersion.toReleaseVersion()
+            );
         }
         IndexVersion newCompatibilityVersion = IndexMetadata.SETTING_INDEX_VERSION_COMPATIBILITY.get(newSettings);
         IndexVersion compatibilityVersion = IndexMetadata.SETTING_INDEX_VERSION_COMPATIBILITY.get(settings);
         if (compatibilityVersion.equals(newCompatibilityVersion) == false) {
             throw new IllegalArgumentException(
                 "compatibility version mismatch on settings update expected: "
-                    + compatibilityVersion
+                    + compatibilityVersion.toReleaseVersion()
                     + " but was: "
-                    + newCompatibilityVersion
+                    + newCompatibilityVersion.toReleaseVersion()
             );
         }
         final String newUUID = newSettings.get(IndexMetadata.SETTING_INDEX_UUID, IndexMetadata.INDEX_UUID_NA_VALUE);
diff --git a/server/src/main/java/org/elasticsearch/repositories/RepositoriesModule.java b/server/src/main/java/org/elasticsearch/repositories/RepositoriesModule.java
index 902dcd6078e7f..2ac804b0597f8 100644
--- a/server/src/main/java/org/elasticsearch/repositories/RepositoriesModule.java
+++ b/server/src/main/java/org/elasticsearch/repositories/RepositoriesModule.java
@@ -103,9 +103,9 @@ public RepositoriesModule(
                     throw new SnapshotRestoreException(
                         snapshot,
                         "the snapshot was created with Elasticsearch version ["
-                            + version
+                            + version.toReleaseVersion()
                             + "] which is below the current versions minimum index compatibility version ["
-                            + IndexVersions.MINIMUM_COMPATIBLE
+                            + IndexVersions.MINIMUM_COMPATIBLE.toReleaseVersion()
                             + "]"
                     );
                 }
diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
index d9937960577a4..8ccc100e31501 100644
--- a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
+++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
@@ -2346,7 +2346,7 @@ private void cacheRepositoryData(RepositoryData repositoryData, IndexVersion ver
             toCache = repositoryData.withoutShardGenerations();
             assert repositoryData.indexMetaDataGenerations().equals(IndexMetaDataGenerations.EMPTY)
                 : "repository data should not contain index generations at version ["
-                    + version
+                    + version.toReleaseVersion()
                     + "] but saw ["
                     + repositoryData.indexMetaDataGenerations()
                     + "]";
diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java
index 8a1f68c867943..1a022d08d3a24 100644
--- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java
+++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java
@@ -567,7 +567,7 @@ public XContentBuilder toXContentExternal(final XContentBuilder builder, final T
 
         if (version != null) {
             builder.field(VERSION_ID, version.id());
-            builder.field(VERSION, version.toString());
+            builder.field(VERSION, version.toReleaseVersion());
         }
 
         if (params.paramAsBoolean(INDEX_NAMES_XCONTENT_PARAM, true)) {
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifierTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifierTests.java
index eb9e26d08ed9c..388cbc83b7c6f 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifierTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifierTests.java
@@ -116,10 +116,10 @@ public void testIncompatibleVersion() {
                 "The index [foo/"
                     + metadata.getIndexUUID()
                     + "] has current compatibility version ["
-                    + indexCreated
+                    + indexCreated.toReleaseVersion()
                     + "] "
                     + "but the minimum compatible version is ["
-                    + minCompat
+                    + minCompat.toReleaseVersion()
                     + "]. It should be re-indexed in Elasticsearch "
                     + (Version.CURRENT.major - 1)
                     + ".x before upgrading to "
diff --git a/server/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java b/server/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java
index ce0597e7169a4..966d7f9b7fcc9 100644
--- a/server/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java
+++ b/server/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java
@@ -601,7 +601,7 @@ public void testIndexCompatibilityChecks() throws IOException {
                 ex.getMessage(),
                 allOf(
                     containsString("Cannot start this node"),
-                    containsString("it holds metadata for indices with version [" + oldIndexVersion + "]"),
+                    containsString("it holds metadata for indices with version [" + oldIndexVersion.toReleaseVersion() + "]"),
                     containsString(
                         "Revert this node to version ["
                             + (previousNodeVersion.major == Version.V_8_0_0.major ? Version.V_7_17_0 : previousNodeVersion)
diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecks.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecks.java
index 49bdfd58eafd8..3da32c7f5a4c2 100644
--- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecks.java
+++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecks.java
@@ -38,7 +38,7 @@ static DeprecationIssue oldIndicesCheck(IndexMetadata indexMetadata) {
                 DeprecationIssue.Level.CRITICAL,
                 "Old index with a compatibility version < 7.0",
                 "https://www.elastic.co/guide/en/elasticsearch/reference/master/" + "breaking-changes-8.0.html",
-                "This index has version: " + currentCompatibilityVersion,
+                "This index has version: " + currentCompatibilityVersion.toReleaseVersion(),
                 false,
                 null
             );
diff --git a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecksTests.java b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecksTests.java
index abff1499fceb7..62f89f650dec2 100644
--- a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecksTests.java
+++ b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecksTests.java
@@ -38,7 +38,7 @@ public void testOldIndicesCheck() {
             DeprecationIssue.Level.CRITICAL,
             "Old index with a compatibility version < 7.0",
             "https://www.elastic.co/guide/en/elasticsearch/reference/master/" + "breaking-changes-8.0.html",
-            "This index has version: " + createdWith,
+            "This index has version: " + createdWith.toReleaseVersion(),
             false,
             null
         );
diff --git a/x-pack/qa/repository-old-versions/src/test/java/org/elasticsearch/oldrepos/OldRepositoryAccessIT.java b/x-pack/qa/repository-old-versions/src/test/java/org/elasticsearch/oldrepos/OldRepositoryAccessIT.java
index cb3100257903b..a332bcb599e90 100644
--- a/x-pack/qa/repository-old-versions/src/test/java/org/elasticsearch/oldrepos/OldRepositoryAccessIT.java
+++ b/x-pack/qa/repository-old-versions/src/test/java/org/elasticsearch/oldrepos/OldRepositoryAccessIT.java
@@ -215,7 +215,7 @@ private void beforeRestart(
         assertEquals(numberOfShards, (int) getResp.evaluate("snapshots.0.shards.successful"));
         assertEquals(numberOfShards, (int) getResp.evaluate("snapshots.0.shards.total"));
         assertEquals(0, (int) getResp.evaluate("snapshots.0.shards.failed"));
-        assertEquals(indexVersion.toString(), getResp.evaluate("snapshots.0.version"));
+        assertEquals(indexVersion.toReleaseVersion(), getResp.evaluate("snapshots.0.version"));
 
         // list specific snapshot on new ES
         getSnaps = new Request("GET", "/_snapshot/" + repoName + "/" + snapshotName);
@@ -229,7 +229,7 @@ private void beforeRestart(
         assertEquals(numberOfShards, (int) getResp.evaluate("snapshots.0.shards.successful"));
         assertEquals(numberOfShards, (int) getResp.evaluate("snapshots.0.shards.total"));
         assertEquals(0, (int) getResp.evaluate("snapshots.0.shards.failed"));
-        assertEquals(indexVersion.toString(), getResp.evaluate("snapshots.0.version"));
+        assertEquals(indexVersion.toReleaseVersion(), getResp.evaluate("snapshots.0.version"));
 
         // list advanced snapshot info on new ES
         getSnaps = new Request("GET", "/_snapshot/" + repoName + "/" + snapshotName + "/_status");

From 6f1e5547a4bcf40cb82674a946a87d206894369d Mon Sep 17 00:00:00 2001
From: David Turner <david.turner@elastic.co>
Date: Thu, 25 Apr 2024 10:57:35 +0100
Subject: [PATCH 18/58] Remove `String`-based timeout support (#107747)

Today various APIs accept both a `TimeValue` and a `String` to specify
the same timeout, with the API implementation taking care of the
conversion from `String` to `TimeValue`. This was convenient for users
of the transport client, but these days it's not necessary. This commit
removes the unnecessary overloads and parsing code and migrates all
callers to creating a proper `TimeValue`.
---
 .../RestDataStreamLifecycleStatsAction.java   |  5 +--
 .../RestExplainDataStreamLifecycleAction.java |  5 +--
 .../ClientScrollableHitSourceTests.java       |  2 +-
 .../azure/classic/AzureSimpleTests.java       | 23 ++++++++++--
 .../classic/AzureTwoStartedNodesTests.java    | 23 ++++++++++--
 .../discovery/gce/GceDiscoverTests.java       |  5 +--
 ...ansportClusterStateActionDisruptionIT.java |  4 +--
 .../admin/indices/create/CreateIndexIT.java   | 16 +++++++--
 .../support/ActiveShardsObserverIT.java       |  7 ++--
 .../cluster/ClusterHealthIT.java              | 14 +++++---
 .../cluster/SimpleClusterStateIT.java         |  3 +-
 .../cluster/SpecificMasterNodesIT.java        | 31 ++++++++++++++--
 .../coordination/RareClusterStateIT.java      |  5 +--
 .../discovery/ClusterDisruptionIT.java        |  4 +--
 .../index/store/CorruptedFileIT.java          |  3 +-
 .../indices/settings/UpdateSettingsIT.java    |  2 +-
 .../indices/state/OpenCloseIndexIT.java       |  8 ++++-
 .../readiness/ReadinessClusterIT.java         |  6 +++-
 .../recovery/FullRollingRestartIT.java        |  2 +-
 .../recovery/RecoveryWhileUnderLoadIT.java    | 36 +++++++++++++++----
 .../elasticsearch/recovery/RelocationIT.java  |  2 +-
 .../SearchServiceCleanupOnLostMasterIT.java   |  6 +++-
 .../search/basic/SearchWhileRelocatingIT.java |  3 +-
 .../search/ccs/CrossClusterSearchIT.java      |  4 +--
 .../search/ccs/CrossClusterSearchLeakIT.java  |  2 +-
 .../search/nested/SimpleNestedIT.java         |  5 +--
 .../search/scroll/DuelScrollIT.java           |  9 ++---
 .../search/scroll/SearchScrollIT.java         | 10 +++---
 .../search/searchafter/SearchAfterIT.java     |  2 +-
 .../DedicatedClusterSnapshotRestoreIT.java    | 19 +++++++---
 .../snapshots/RepositoriesIT.java             |  7 ++--
 .../cluster/health/ClusterHealthRequest.java  |  4 ---
 .../health/ClusterHealthRequestBuilder.java   |  5 ---
 .../action/bulk/BulkRequest.java              |  7 ----
 .../action/bulk/BulkRequestBuilder.java       |  8 -----
 .../action/search/SearchRequest.java          |  7 ----
 .../action/search/SearchRequestBuilder.java   |  8 -----
 .../action/search/SearchScrollRequest.java    |  7 ----
 .../search/SearchScrollRequestBuilder.java    |  8 -----
 .../support/master/AcknowledgedRequest.java   | 11 ------
 .../master/AcknowledgedRequestBuilder.java    |  9 -----
 .../MasterNodeOperationRequestBuilder.java    |  9 -----
 .../support/master/MasterNodeRequest.java     |  8 -----
 .../support/nodes/BaseNodesRequest.java       |  6 ----
 .../nodes/NodesOperationRequestBuilder.java   |  5 ---
 .../replication/ReplicationRequest.java       |  7 ----
 .../ReplicationRequestBuilder.java            |  9 -----
 .../InstanceShardOperationRequest.java        |  7 ----
 .../InstanceShardOperationRequestBuilder.java |  9 -----
 .../support/tasks/BaseTasksRequest.java       |  6 ----
 .../reindex/AbstractBulkByScrollRequest.java  |  8 -----
 .../admin/cluster/RestClusterStatsAction.java |  2 +-
 .../cluster/RestNodesHotThreadsAction.java    |  5 ++-
 .../admin/cluster/RestNodesInfoAction.java    |  2 +-
 .../admin/cluster/RestNodesStatsAction.java   |  2 +-
 .../admin/cluster/RestNodesUsageAction.java   |  2 +-
 .../RestReloadSecureSettingsAction.java       |  2 +-
 .../create/CreateSnapshotRequestTests.java    |  3 +-
 .../state/ClusterStateRequestTests.java       |  2 +-
 .../search/SearchScrollRequestTests.java      |  2 +-
 .../search/TransportSearchActionTests.java    |  6 ++--
 .../TransportReplicationActionTests.java      | 33 +++++++++--------
 .../search/SearchServiceTests.java            | 12 +++----
 .../search/slice/SliceBuilderTests.java       |  3 +-
 .../elasticsearch/xpack/ccr/AutoFollowIT.java | 12 +++----
 .../xpack/ccr/rest/RestFollowStatsAction.java |  4 +--
 .../elasticsearch/xpack/CcrIntegTestCase.java |  4 +--
 .../xpack/CcrSingleNodeTestCase.java          |  4 +--
 .../sourceonly/SourceOnlySnapshotIT.java      |  6 ++--
 .../xpack/graph/GraphExploreRequest.java      |  5 ---
 .../xpack/core/ccr/action/CcrStatsAction.java |  4 ---
 .../action/GraphExploreRequestBuilder.java    |  9 -----
 .../core/ccr/action/CcrStatsActionTests.java  | 11 +++---
 .../action/RestExplainLifecycleAction.java    |  6 +---
 .../persistence/BatchedDocumentsIterator.java |  3 +-
 .../integration/SecurityClearScrollTests.java |  3 +-
 .../votingonly/VotingOnlyNodePluginTests.java |  8 ++++-
 77 files changed, 277 insertions(+), 299 deletions(-)

diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestDataStreamLifecycleStatsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestDataStreamLifecycleStatsAction.java
index ece16042706a7..e734c913fe9e8 100644
--- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestDataStreamLifecycleStatsAction.java
+++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestDataStreamLifecycleStatsAction.java
@@ -35,11 +35,8 @@ public List<Route> routes() {
 
     @Override
     protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) {
-        String masterNodeTimeout = restRequest.param("master_timeout");
         GetDataStreamLifecycleStatsAction.Request request = new GetDataStreamLifecycleStatsAction.Request();
-        if (masterNodeTimeout != null) {
-            request.masterNodeTimeout(masterNodeTimeout);
-        }
+        request.masterNodeTimeout(restRequest.paramAsTime("master_timeout", request.masterNodeTimeout()));
         return channel -> client.execute(
             GetDataStreamLifecycleStatsAction.INSTANCE,
             request,
diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestExplainDataStreamLifecycleAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestExplainDataStreamLifecycleAction.java
index d3115d6d3d3a3..522ce12d834a8 100644
--- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestExplainDataStreamLifecycleAction.java
+++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestExplainDataStreamLifecycleAction.java
@@ -41,10 +41,7 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient
         ExplainDataStreamLifecycleAction.Request explainRequest = new ExplainDataStreamLifecycleAction.Request(indices);
         explainRequest.includeDefaults(restRequest.paramAsBoolean("include_defaults", false));
         explainRequest.indicesOptions(IndicesOptions.fromRequest(restRequest, IndicesOptions.strictExpandOpen()));
-        String masterNodeTimeout = restRequest.param("master_timeout");
-        if (masterNodeTimeout != null) {
-            explainRequest.masterNodeTimeout(masterNodeTimeout);
-        }
+        explainRequest.masterNodeTimeout(restRequest.paramAsTime("master_timeout", explainRequest.masterNodeTimeout()));
         return channel -> client.execute(
             ExplainDataStreamLifecycleAction.INSTANCE,
             explainRequest,
diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/ClientScrollableHitSourceTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/ClientScrollableHitSourceTests.java
index 44e69d3a4cda8..b3558d4930ba3 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/reindex/ClientScrollableHitSourceTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/ClientScrollableHitSourceTests.java
@@ -96,7 +96,7 @@ private void dotestBasicsWithRetry(int retries, int minFailures, int maxFailures
             responses::add,
             failureHandler,
             new ParentTaskAssigningClient(client, parentTask),
-            new SearchRequest().scroll("1m")
+            new SearchRequest().scroll(TimeValue.timeValueMinutes(1))
         );
 
         hitSource.start();
diff --git a/plugins/discovery-azure-classic/src/internalClusterTest/java/org/elasticsearch/discovery/azure/classic/AzureSimpleTests.java b/plugins/discovery-azure-classic/src/internalClusterTest/java/org/elasticsearch/discovery/azure/classic/AzureSimpleTests.java
index 00d8b5980374c..9a55bfde38b3c 100644
--- a/plugins/discovery-azure-classic/src/internalClusterTest/java/org/elasticsearch/discovery/azure/classic/AzureSimpleTests.java
+++ b/plugins/discovery-azure-classic/src/internalClusterTest/java/org/elasticsearch/discovery/azure/classic/AzureSimpleTests.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.cloud.azure.classic.management.AzureComputeService.Discovery;
 import org.elasticsearch.cloud.azure.classic.management.AzureComputeService.Management;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.test.ESIntegTestCase;
 
 import static org.hamcrest.Matchers.containsString;
@@ -26,7 +27,16 @@ public void testOneNodeShouldRunUsingPrivateIp() {
 
         final String node1 = internalCluster().startNode(settings);
         registerAzureNode(node1);
-        assertNotNull(client().admin().cluster().prepareState().setMasterNodeTimeout("1s").get().getState().nodes().getMasterNodeId());
+        assertNotNull(
+            client().admin()
+                .cluster()
+                .prepareState()
+                .setMasterNodeTimeout(TimeValue.timeValueSeconds(1))
+                .get()
+                .getState()
+                .nodes()
+                .getMasterNodeId()
+        );
 
         // We expect having 1 node as part of the cluster, let's test that
         assertNumberOfNodes(1);
@@ -39,7 +49,16 @@ public void testOneNodeShouldRunUsingPublicIp() {
 
         final String node1 = internalCluster().startNode(settings);
         registerAzureNode(node1);
-        assertNotNull(client().admin().cluster().prepareState().setMasterNodeTimeout("1s").get().getState().nodes().getMasterNodeId());
+        assertNotNull(
+            client().admin()
+                .cluster()
+                .prepareState()
+                .setMasterNodeTimeout(TimeValue.timeValueSeconds(1))
+                .get()
+                .getState()
+                .nodes()
+                .getMasterNodeId()
+        );
 
         // We expect having 1 node as part of the cluster, let's test that
         assertNumberOfNodes(1);
diff --git a/plugins/discovery-azure-classic/src/internalClusterTest/java/org/elasticsearch/discovery/azure/classic/AzureTwoStartedNodesTests.java b/plugins/discovery-azure-classic/src/internalClusterTest/java/org/elasticsearch/discovery/azure/classic/AzureTwoStartedNodesTests.java
index dd85fa88b94db..b8d0a1ef7bdd5 100644
--- a/plugins/discovery-azure-classic/src/internalClusterTest/java/org/elasticsearch/discovery/azure/classic/AzureTwoStartedNodesTests.java
+++ b/plugins/discovery-azure-classic/src/internalClusterTest/java/org/elasticsearch/discovery/azure/classic/AzureTwoStartedNodesTests.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.cloud.azure.classic.management.AzureComputeService.Discovery;
 import org.elasticsearch.cloud.azure.classic.management.AzureComputeService.Management;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.test.ESIntegTestCase;
 
 @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, numClientNodes = 0)
@@ -29,12 +30,30 @@ public void testTwoNodesShouldRunUsingPrivateOrPublicIp() {
         logger.info("--> start first node");
         final String node1 = internalCluster().startNode(settings);
         registerAzureNode(node1);
-        assertNotNull(client().admin().cluster().prepareState().setMasterNodeTimeout("1s").get().getState().nodes().getMasterNodeId());
+        assertNotNull(
+            client().admin()
+                .cluster()
+                .prepareState()
+                .setMasterNodeTimeout(TimeValue.timeValueSeconds(1))
+                .get()
+                .getState()
+                .nodes()
+                .getMasterNodeId()
+        );
 
         logger.info("--> start another node");
         final String node2 = internalCluster().startNode(settings);
         registerAzureNode(node2);
-        assertNotNull(client().admin().cluster().prepareState().setMasterNodeTimeout("1s").get().getState().nodes().getMasterNodeId());
+        assertNotNull(
+            client().admin()
+                .cluster()
+                .prepareState()
+                .setMasterNodeTimeout(TimeValue.timeValueSeconds(1))
+                .get()
+                .getState()
+                .nodes()
+                .getMasterNodeId()
+        );
 
         // We expect having 2 nodes as part of the cluster, let's test that
         assertNumberOfNodes(2);
diff --git a/plugins/discovery-gce/src/internalClusterTest/java/org/elasticsearch/discovery/gce/GceDiscoverTests.java b/plugins/discovery-gce/src/internalClusterTest/java/org/elasticsearch/discovery/gce/GceDiscoverTests.java
index c655b8ca0a17e..32be38ac7f813 100644
--- a/plugins/discovery-gce/src/internalClusterTest/java/org/elasticsearch/discovery/gce/GceDiscoverTests.java
+++ b/plugins/discovery-gce/src/internalClusterTest/java/org/elasticsearch/discovery/gce/GceDiscoverTests.java
@@ -16,6 +16,7 @@
 import org.elasticsearch.cloud.gce.util.Access;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.plugin.discovery.gce.GceDiscoveryPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
@@ -67,7 +68,7 @@ public void testJoin() {
         ClusterStateResponse clusterStateResponse = client(masterNode).admin()
             .cluster()
             .prepareState()
-            .setMasterNodeTimeout("1s")
+            .setMasterNodeTimeout(TimeValue.timeValueSeconds(1))
             .clear()
             .setNodes(true)
             .get();
@@ -79,7 +80,7 @@ public void testJoin() {
         clusterStateResponse = client(secondNode).admin()
             .cluster()
             .prepareState()
-            .setMasterNodeTimeout("1s")
+            .setMasterNodeTimeout(TimeValue.timeValueSeconds(1))
             .clear()
             .setNodes(true)
             .setLocal(true)
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateActionDisruptionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateActionDisruptionIT.java
index 8750389480071..7679d9f5b9c0c 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateActionDisruptionIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateActionDisruptionIT.java
@@ -48,7 +48,7 @@ public void testNonLocalRequestAlwaysFindsMaster() throws Exception {
             final ClusterStateRequestBuilder clusterStateRequestBuilder = clusterAdmin().prepareState()
                 .clear()
                 .setNodes(true)
-                .setMasterNodeTimeout("100ms");
+                .setMasterNodeTimeout(TimeValue.timeValueMillis(100));
             final ClusterStateResponse clusterStateResponse;
             try {
                 clusterStateResponse = clusterStateRequestBuilder.get();
@@ -68,7 +68,7 @@ public void testLocalRequestAlwaysSucceeds() throws Exception {
                 .clear()
                 .setLocal(true)
                 .setNodes(true)
-                .setMasterNodeTimeout("100ms")
+                .setMasterNodeTimeout(TimeValue.timeValueMillis(100))
                 .get()
                 .getState()
                 .nodes();
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java
index 7574cd0271c46..e0be40aeab18c 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java
@@ -317,11 +317,23 @@ public void testDefaultWaitForActiveShardsUsesIndexSetting() throws Exception {
 
         // all should fail
         settings = Settings.builder().put(settings).put(SETTING_WAIT_FOR_ACTIVE_SHARDS.getKey(), "all").build();
-        assertFalse(indicesAdmin().prepareCreate("test-idx-2").setSettings(settings).setTimeout("100ms").get().isShardsAcknowledged());
+        assertFalse(
+            indicesAdmin().prepareCreate("test-idx-2")
+                .setSettings(settings)
+                .setTimeout(TimeValue.timeValueMillis(100))
+                .get()
+                .isShardsAcknowledged()
+        );
 
         // the numeric equivalent of all should also fail
         settings = Settings.builder().put(settings).put(SETTING_WAIT_FOR_ACTIVE_SHARDS.getKey(), Integer.toString(numReplicas + 1)).build();
-        assertFalse(indicesAdmin().prepareCreate("test-idx-3").setSettings(settings).setTimeout("100ms").get().isShardsAcknowledged());
+        assertFalse(
+            indicesAdmin().prepareCreate("test-idx-3")
+                .setSettings(settings)
+                .setTimeout(TimeValue.timeValueMillis(100))
+                .get()
+                .isShardsAcknowledged()
+        );
     }
 
     public void testInvalidPartitionSize() {
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/support/ActiveShardsObserverIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/support/ActiveShardsObserverIT.java
index a377ba9eb94ad..39273e9d1712b 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/action/support/ActiveShardsObserverIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/action/support/ActiveShardsObserverIT.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.test.ESIntegTestCase;
 
 import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING;
@@ -38,7 +39,7 @@ public void testCreateIndexNoActiveShardsTimesOut() throws Exception {
         assertFalse(
             prepareCreate(indexName).setSettings(settings)
                 .setWaitForActiveShards(randomBoolean() ? ActiveShardCount.from(1) : ActiveShardCount.ALL)
-                .setTimeout("100ms")
+                .setTimeout(TimeValue.timeValueMillis(100))
                 .get()
                 .isShardsAcknowledged()
         );
@@ -70,7 +71,7 @@ public void testCreateIndexNotEnoughActiveShardsTimesOut() throws Exception {
         assertFalse(
             prepareCreate(indexName).setSettings(settings)
                 .setWaitForActiveShards(randomIntBetween(numDataNodes + 1, numReplicas + 1))
-                .setTimeout("100ms")
+                .setTimeout(TimeValue.timeValueMillis(100))
                 .get()
                 .isShardsAcknowledged()
         );
@@ -101,7 +102,7 @@ public void testCreateIndexWaitsForAllActiveShards() throws Exception {
         assertFalse(
             prepareCreate(indexName).setSettings(settings)
                 .setWaitForActiveShards(ActiveShardCount.ALL)
-                .setTimeout("100ms")
+                .setTimeout(TimeValue.timeValueMillis(100))
                 .get()
                 .isShardsAcknowledged()
         );
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/ClusterHealthIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/ClusterHealthIT.java
index 7a8accf8cc7ce..65c0aa6548182 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/ClusterHealthIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/ClusterHealthIT.java
@@ -56,13 +56,16 @@ public void testSimpleLocalHealth() {
 
     public void testHealth() {
         logger.info("--> running cluster health on an index that does not exists");
-        ClusterHealthResponse healthResponse = clusterAdmin().prepareHealth("test1").setWaitForYellowStatus().setTimeout("1s").get();
+        ClusterHealthResponse healthResponse = clusterAdmin().prepareHealth("test1")
+            .setWaitForYellowStatus()
+            .setTimeout(TimeValue.timeValueSeconds(1))
+            .get();
         assertThat(healthResponse.isTimedOut(), equalTo(true));
         assertThat(healthResponse.getStatus(), equalTo(ClusterHealthStatus.RED));
         assertThat(healthResponse.getIndices().isEmpty(), equalTo(true));
 
         logger.info("--> running cluster wide health");
-        healthResponse = clusterAdmin().prepareHealth().setWaitForGreenStatus().setTimeout("10s").get();
+        healthResponse = clusterAdmin().prepareHealth().setWaitForGreenStatus().setTimeout(TimeValue.timeValueSeconds(10)).get();
         assertThat(healthResponse.isTimedOut(), equalTo(false));
         assertThat(healthResponse.getStatus(), equalTo(ClusterHealthStatus.GREEN));
         assertThat(healthResponse.getIndices().isEmpty(), equalTo(true));
@@ -71,13 +74,16 @@ public void testHealth() {
         createIndex("test1");
 
         logger.info("--> running cluster health on an index that does exists");
-        healthResponse = clusterAdmin().prepareHealth("test1").setWaitForGreenStatus().setTimeout("10s").get();
+        healthResponse = clusterAdmin().prepareHealth("test1").setWaitForGreenStatus().setTimeout(TimeValue.timeValueSeconds(10)).get();
         assertThat(healthResponse.isTimedOut(), equalTo(false));
         assertThat(healthResponse.getStatus(), equalTo(ClusterHealthStatus.GREEN));
         assertThat(healthResponse.getIndices().get("test1").getStatus(), equalTo(ClusterHealthStatus.GREEN));
 
         logger.info("--> running cluster health on an index that does exists and an index that doesn't exists");
-        healthResponse = clusterAdmin().prepareHealth("test1", "test2").setWaitForYellowStatus().setTimeout("1s").get();
+        healthResponse = clusterAdmin().prepareHealth("test1", "test2")
+            .setWaitForYellowStatus()
+            .setTimeout(TimeValue.timeValueSeconds(1))
+            .get();
         assertThat(healthResponse.isTimedOut(), equalTo(true));
         assertThat(healthResponse.getStatus(), equalTo(ClusterHealthStatus.RED));
         assertThat(healthResponse.getIndices().get("test1").getStatus(), equalTo(ClusterHealthStatus.GREEN));
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/SimpleClusterStateIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/SimpleClusterStateIT.java
index f4457a7db8b7c..3dba41adec08b 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/SimpleClusterStateIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/SimpleClusterStateIT.java
@@ -26,6 +26,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.plugins.ClusterPlugin;
@@ -243,7 +244,7 @@ public void testLargeClusterStatePublishing() throws Exception {
                     indexSettings(numberOfShards, 0).put(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey(), Long.MAX_VALUE)
                 )
                 .setMapping(mapping)
-                .setTimeout("60s")
+                .setTimeout(TimeValue.timeValueMinutes(1))
         );
         ensureGreen(); // wait for green state, so its both green, and there are no more pending events
         MappingMetadata masterMappingMetadata = indicesAdmin().prepareGetMappings("test").get().getMappings().get("test");
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/SpecificMasterNodesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/SpecificMasterNodesIT.java
index cd0bf5c428118..538f5e7a1640d 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/SpecificMasterNodesIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/SpecificMasterNodesIT.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.action.admin.cluster.configuration.AddVotingConfigExclusionsRequest;
 import org.elasticsearch.action.admin.cluster.configuration.TransportAddVotingConfigExclusionsAction;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.discovery.MasterNotDiscoveredException;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.test.ESIntegTestCase;
@@ -35,7 +36,15 @@ public void testSimpleOnlyMasterNodeElection() throws IOException {
         logger.info("--> start data node / non master node");
         internalCluster().startNode(Settings.builder().put(dataOnlyNode()).put("discovery.initial_state_timeout", "1s"));
         try {
-            assertThat(clusterAdmin().prepareState().setMasterNodeTimeout("100ms").get().getState().nodes().getMasterNodeId(), nullValue());
+            assertThat(
+                clusterAdmin().prepareState()
+                    .setMasterNodeTimeout(TimeValue.timeValueMillis(100))
+                    .get()
+                    .getState()
+                    .nodes()
+                    .getMasterNodeId(),
+                nullValue()
+            );
             fail("should not be able to find master");
         } catch (MasterNotDiscoveredException e) {
             // all is well, no master elected
@@ -56,7 +65,15 @@ public void testSimpleOnlyMasterNodeElection() throws IOException {
         internalCluster().stopCurrentMasterNode();
 
         try {
-            assertThat(clusterAdmin().prepareState().setMasterNodeTimeout("100ms").get().getState().nodes().getMasterNodeId(), nullValue());
+            assertThat(
+                clusterAdmin().prepareState()
+                    .setMasterNodeTimeout(TimeValue.timeValueMillis(100))
+                    .get()
+                    .getState()
+                    .nodes()
+                    .getMasterNodeId(),
+                nullValue()
+            );
             fail("should not be able to find master");
         } catch (MasterNotDiscoveredException e) {
             // all is well, no master elected
@@ -81,7 +98,15 @@ public void testElectOnlyBetweenMasterNodes() throws Exception {
         logger.info("--> start data node / non master node");
         internalCluster().startNode(Settings.builder().put(dataOnlyNode()).put("discovery.initial_state_timeout", "1s"));
         try {
-            assertThat(clusterAdmin().prepareState().setMasterNodeTimeout("100ms").get().getState().nodes().getMasterNodeId(), nullValue());
+            assertThat(
+                clusterAdmin().prepareState()
+                    .setMasterNodeTimeout(TimeValue.timeValueMillis(100))
+                    .get()
+                    .getState()
+                    .nodes()
+                    .getMasterNodeId(),
+                nullValue()
+            );
             fail("should not be able to find master");
         } catch (MasterNotDiscoveredException e) {
             // all is well, no master elected
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/RareClusterStateIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/RareClusterStateIT.java
index 334f2cfccbe0a..a208656179339 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/RareClusterStateIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/RareClusterStateIT.java
@@ -28,6 +28,7 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.CollectionUtils;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.plugins.Plugin;
@@ -158,14 +159,14 @@ public void testDeleteCreateInOneBulk() throws Exception {
         );
 
         logger.info("--> delete index");
-        assertFalse(indicesAdmin().prepareDelete("test").setTimeout("0s").get().isAcknowledged());
+        assertFalse(indicesAdmin().prepareDelete("test").setTimeout(TimeValue.ZERO).get().isAcknowledged());
         logger.info("--> and recreate it");
         assertFalse(
             prepareCreate("test").setSettings(
                 Settings.builder()
                     .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
                     .put(IndexMetadata.SETTING_WAIT_FOR_ACTIVE_SHARDS.getKey(), "0")
-            ).setTimeout("0s").get().isAcknowledged()
+            ).setTimeout(TimeValue.ZERO).get().isAcknowledged()
         );
 
         // unblock publications & do a trivial cluster state update to bring data node up to date
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/discovery/ClusterDisruptionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/discovery/ClusterDisruptionIT.java
index c661894840261..e36d7a4e56eab 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/discovery/ClusterDisruptionIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/discovery/ClusterDisruptionIT.java
@@ -428,7 +428,7 @@ public boolean validateClusterForming() {
                     .cluster()
                     .prepareHealth()
                     .setWaitForNodes("2")
-                    .setTimeout("2s")
+                    .setTimeout(TimeValue.timeValueSeconds(2))
                     .get()
                     .isTimedOut()
             );
@@ -456,7 +456,7 @@ public void testIndicesDeleted() throws Exception {
         networkDisruption.startDisrupting();
         // We know this will time out due to the partition, we check manually below to not proceed until
         // the delete has been applied to the master node and the master eligible node.
-        internalCluster().client(masterNode1).admin().indices().prepareDelete(idxName).setTimeout("0s").get();
+        internalCluster().client(masterNode1).admin().indices().prepareDelete(idxName).setTimeout(TimeValue.ZERO).get();
         // Don't restart the master node until we know the index deletion has taken effect on master and the master eligible node.
         assertBusy(() -> {
             for (String masterNode : allMasterEligibleNodes) {
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/store/CorruptedFileIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/store/CorruptedFileIT.java
index a6e82f982b576..58b63eb77d2bd 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/index/store/CorruptedFileIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/index/store/CorruptedFileIT.java
@@ -180,7 +180,8 @@ public void testCorruptFileAndRecover() throws InterruptedException, IOException
         setReplicaCount(2, "test");
         ClusterHealthResponse health = clusterAdmin().health(
             new ClusterHealthRequest("test").waitForGreenStatus()
-                .timeout("5m") // sometimes due to cluster rebalacing and random settings default timeout is just not enough.
+                // sometimes due to cluster rebalancing and random settings default timeout is just not enough.
+                .timeout(TimeValue.timeValueMinutes(5))
                 .waitForNoRelocatingShards(true)
         ).actionGet();
         if (health.isTimedOut()) {
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/settings/UpdateSettingsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/settings/UpdateSettingsIT.java
index 5bbedd8dc5870..6e58d275e578f 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/indices/settings/UpdateSettingsIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/settings/UpdateSettingsIT.java
@@ -328,7 +328,7 @@ public void testOpenCloseUpdateSettings() throws Exception {
 
         // Wait for the index to turn green before attempting to close it
         ClusterHealthResponse health = clusterAdmin().prepareHealth()
-            .setTimeout("30s")
+            .setTimeout(TimeValue.timeValueSeconds(30))
             .setWaitForEvents(Priority.LANGUID)
             .setWaitForGreenStatus()
             .get();
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/state/OpenCloseIndexIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/state/OpenCloseIndexIT.java
index 61bb48b7f7583..5201e4ab3d812 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/indices/state/OpenCloseIndexIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/state/OpenCloseIndexIT.java
@@ -20,6 +20,7 @@
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.rest.RestStatus;
@@ -233,7 +234,12 @@ public void testOpenWaitingForActiveShardsFailed() throws Exception {
         assertAcked(client.admin().indices().prepareCreate("test").setSettings(settings).get());
         assertAcked(client.admin().indices().prepareClose("test").get());
 
-        OpenIndexResponse response = client.admin().indices().prepareOpen("test").setTimeout("100ms").setWaitForActiveShards(2).get();
+        OpenIndexResponse response = client.admin()
+            .indices()
+            .prepareOpen("test")
+            .setTimeout(TimeValue.timeValueMillis(100))
+            .setWaitForActiveShards(2)
+            .get();
         assertThat(response.isShardsAcknowledged(), equalTo(false));
         assertBusy(
             () -> assertThat(
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/readiness/ReadinessClusterIT.java b/server/src/internalClusterTest/java/org/elasticsearch/readiness/ReadinessClusterIT.java
index 4ecc508c066ac..2ecdd06f379d2 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/readiness/ReadinessClusterIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/readiness/ReadinessClusterIT.java
@@ -18,6 +18,7 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.Strings;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.discovery.MasterNotDiscoveredException;
 import org.elasticsearch.plugins.Plugin;
@@ -115,7 +116,10 @@ private void assertMasterNode(Client client, String node) {
     }
 
     private void expectMasterNotFound() {
-        expectThrows(MasterNotDiscoveredException.class, clusterAdmin().prepareState().setMasterNodeTimeout("100ms"));
+        expectThrows(
+            MasterNotDiscoveredException.class,
+            clusterAdmin().prepareState().setMasterNodeTimeout(TimeValue.timeValueMillis(100))
+        );
     }
 
     public void testReadinessDuringRestarts() throws Exception {
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/recovery/FullRollingRestartIT.java b/server/src/internalClusterTest/java/org/elasticsearch/recovery/FullRollingRestartIT.java
index 1e67a38c76017..da59d306d4119 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/recovery/FullRollingRestartIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/recovery/FullRollingRestartIT.java
@@ -46,7 +46,7 @@ public void testFullRollingRestart() throws Exception {
         internalCluster().startNode();
         createIndex("test");
 
-        final String healthTimeout = "1m";
+        final var healthTimeout = TimeValue.timeValueMinutes(1);
 
         for (int i = 0; i < 1000; i++) {
             prepareIndex("test").setId(Long.toString(i)).setSource(Map.<String, Object>of("test", "value" + i)).get();
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/recovery/RecoveryWhileUnderLoadIT.java b/server/src/internalClusterTest/java/org/elasticsearch/recovery/RecoveryWhileUnderLoadIT.java
index 782aafece4399..70aabbc8c30d5 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/recovery/RecoveryWhileUnderLoadIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/recovery/RecoveryWhileUnderLoadIT.java
@@ -101,7 +101,12 @@ public void testRecoverWhileUnderLoadAllocateReplicasTest() throws Exception {
 
             logger.info("--> waiting for GREEN health status ...");
             // make sure the cluster state is green, and all has been recovered
-            assertNoTimeout(clusterAdmin().prepareHealth().setWaitForEvents(Priority.LANGUID).setTimeout("5m").setWaitForGreenStatus());
+            assertNoTimeout(
+                clusterAdmin().prepareHealth()
+                    .setWaitForEvents(Priority.LANGUID)
+                    .setTimeout(TimeValue.timeValueMinutes(5))
+                    .setWaitForGreenStatus()
+            );
 
             logger.info("--> waiting for {} docs to be indexed ...", totalNumDocs);
             waitForDocs(totalNumDocs, indexer);
@@ -157,7 +162,12 @@ public void testRecoverWhileUnderLoadAllocateReplicasRelocatePrimariesTest() thr
             allowNodes("test", 4);
 
             logger.info("--> waiting for GREEN health status ...");
-            assertNoTimeout(clusterAdmin().prepareHealth().setWaitForEvents(Priority.LANGUID).setTimeout("5m").setWaitForGreenStatus());
+            assertNoTimeout(
+                clusterAdmin().prepareHealth()
+                    .setWaitForEvents(Priority.LANGUID)
+                    .setTimeout(TimeValue.timeValueMinutes(5))
+                    .setWaitForGreenStatus()
+            );
 
             logger.info("--> waiting for {} docs to be indexed ...", totalNumDocs);
             waitForDocs(totalNumDocs, indexer);
@@ -217,7 +227,7 @@ public void testRecoverWhileUnderLoadWithReducedAllowedNodes() throws Exception
             assertNoTimeout(
                 clusterAdmin().prepareHealth()
                     .setWaitForEvents(Priority.LANGUID)
-                    .setTimeout("5m")
+                    .setTimeout(TimeValue.timeValueMinutes(5))
                     .setWaitForGreenStatus()
                     .setWaitForNoRelocatingShards(true)
             );
@@ -232,21 +242,30 @@ public void testRecoverWhileUnderLoadWithReducedAllowedNodes() throws Exception
             allowNodes("test", 3);
             logger.info("--> waiting for relocations ...");
             assertNoTimeout(
-                clusterAdmin().prepareHealth().setWaitForEvents(Priority.LANGUID).setTimeout("5m").setWaitForNoRelocatingShards(true)
+                clusterAdmin().prepareHealth()
+                    .setWaitForEvents(Priority.LANGUID)
+                    .setTimeout(TimeValue.timeValueMinutes(5))
+                    .setWaitForNoRelocatingShards(true)
             );
 
             logger.info("--> allow 2 nodes for index [test] ...");
             allowNodes("test", 2);
             logger.info("--> waiting for relocations ...");
             assertNoTimeout(
-                clusterAdmin().prepareHealth().setWaitForEvents(Priority.LANGUID).setTimeout("5m").setWaitForNoRelocatingShards(true)
+                clusterAdmin().prepareHealth()
+                    .setWaitForEvents(Priority.LANGUID)
+                    .setTimeout(TimeValue.timeValueMinutes(5))
+                    .setWaitForNoRelocatingShards(true)
             );
 
             logger.info("--> allow 1 nodes for index [test] ...");
             allowNodes("test", 1);
             logger.info("--> waiting for relocations ...");
             assertNoTimeout(
-                clusterAdmin().prepareHealth().setWaitForEvents(Priority.LANGUID).setTimeout("5m").setWaitForNoRelocatingShards(true)
+                clusterAdmin().prepareHealth()
+                    .setWaitForEvents(Priority.LANGUID)
+                    .setTimeout(TimeValue.timeValueMinutes(5))
+                    .setWaitForNoRelocatingShards(true)
             );
 
             logger.info("--> marking and waiting for indexing threads to stop ...");
@@ -254,7 +273,10 @@ public void testRecoverWhileUnderLoadWithReducedAllowedNodes() throws Exception
             logger.info("--> indexing threads stopped");
 
             assertNoTimeout(
-                clusterAdmin().prepareHealth().setWaitForEvents(Priority.LANGUID).setTimeout("5m").setWaitForNoRelocatingShards(true)
+                clusterAdmin().prepareHealth()
+                    .setWaitForEvents(Priority.LANGUID)
+                    .setTimeout(TimeValue.timeValueMinutes(5))
+                    .setWaitForNoRelocatingShards(true)
             );
 
             logger.info("--> refreshing the index");
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/recovery/RelocationIT.java b/server/src/internalClusterTest/java/org/elasticsearch/recovery/RelocationIT.java
index 0e14d80aaa0cd..4c3c05992a449 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/recovery/RelocationIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/recovery/RelocationIT.java
@@ -349,7 +349,7 @@ public void indexShardStateChanged(
                 clusterAdmin().prepareHealth()
                     .setWaitForNoRelocatingShards(true)
                     .setWaitForEvents(Priority.LANGUID)
-                    .setTimeout("30s")
+                    .setTimeout(TimeValue.timeValueSeconds(30))
                     .get()
                     .isTimedOut()
             );
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/SearchServiceCleanupOnLostMasterIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/SearchServiceCleanupOnLostMasterIT.java
index 398226e868d47..5625299890b7e 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/SearchServiceCleanupOnLostMasterIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/SearchServiceCleanupOnLostMasterIT.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.common.CheckedBiConsumer;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.transport.MockTransportService;
@@ -70,7 +71,10 @@ private void testLostMaster(CheckedBiConsumer<String, String, Exception> loseMas
 
         index("test", "test", "{}");
 
-        assertResponse(prepareSearch("test").setScroll("30m"), response -> assertThat(response.getScrollId(), is(notNullValue())));
+        assertResponse(
+            prepareSearch("test").setScroll(TimeValue.timeValueMinutes(30)),
+            response -> assertThat(response.getScrollId(), is(notNullValue()))
+        );
         loseMaster.accept(master, dataNode);
         // in the past, this failed because the search context for the scroll would prevent the shard lock from being released.
         ensureYellow();
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/basic/SearchWhileRelocatingIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/basic/SearchWhileRelocatingIT.java
index 26d81f672d650..03a21210630b7 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/basic/SearchWhileRelocatingIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/basic/SearchWhileRelocatingIT.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchPhaseExecutionException;
 import org.elasticsearch.common.Priority;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.search.SearchHits;
 import org.elasticsearch.test.ESIntegTestCase;
 
@@ -127,7 +128,7 @@ public void run() {
                 .setWaitForYellowStatus()
                 .setWaitForNoRelocatingShards(true)
                 .setWaitForEvents(Priority.LANGUID)
-                .setTimeout("5m")
+                .setTimeout(TimeValue.timeValueMinutes(5))
                 .get();
             assertNoTimeout(resp);
             // if we hit only non-critical exceptions we make sure that the post search works
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CrossClusterSearchIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CrossClusterSearchIT.java
index d21619f4e6f89..89bc0e83351ad 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CrossClusterSearchIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CrossClusterSearchIT.java
@@ -105,7 +105,7 @@ public void testClusterDetailsAfterSuccessfulCCS() throws Exception {
 
         SearchRequest searchRequest = new SearchRequest(localIndex, REMOTE_CLUSTER + ":" + remoteIndex);
         if (randomBoolean()) {
-            searchRequest = searchRequest.scroll("1m");
+            searchRequest = searchRequest.scroll(TimeValue.timeValueMinutes(1));
         }
         searchRequest.allowPartialSearchResults(false);
         if (randomBoolean()) {
@@ -286,7 +286,7 @@ public void testClusterDetailsAfterCCSWhereRemoteClusterHasNoShardsToSearch() th
 
         SearchRequest searchRequest = new SearchRequest(localIndex, REMOTE_CLUSTER + ":" + "no_such_index*");
         if (randomBoolean()) {
-            searchRequest = searchRequest.scroll("1m");
+            searchRequest = searchRequest.scroll(TimeValue.timeValueMinutes(1));
         }
         searchRequest.allowPartialSearchResults(false);
         if (randomBoolean()) {
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CrossClusterSearchLeakIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CrossClusterSearchLeakIT.java
index 8b6f4112cfc17..a9ae215c1ab79 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CrossClusterSearchLeakIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CrossClusterSearchLeakIT.java
@@ -130,7 +130,7 @@ public void testSearch() throws Exception {
                     .size(between(scroll ? 1 : 0, 1000))
             );
             if (scroll) {
-                searchRequest.scroll("30s");
+                searchRequest.scroll(TimeValue.timeValueSeconds(30));
             }
             searchRequest.setCcsMinimizeRoundtrips(rarely());
             futures.add(client(LOCAL_CLUSTER).search(searchRequest));
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/nested/SimpleNestedIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/nested/SimpleNestedIT.java
index 29a3e589e7923..a9d33b268e73f 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/nested/SimpleNestedIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/nested/SimpleNestedIT.java
@@ -19,6 +19,7 @@
 import org.elasticsearch.action.search.SearchType;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.search.sort.NestedSortBuilder;
 import org.elasticsearch.search.sort.SortBuilders;
@@ -619,7 +620,7 @@ public void testSimpleNestedSortingWithNestedFilterMissing() throws Exception {
             );
 
         if (randomBoolean()) {
-            searchRequestBuilder.setScroll("10m");
+            searchRequestBuilder.setScroll(TimeValue.timeValueMinutes(10));
         }
 
         assertResponse(searchRequestBuilder, response -> {
@@ -640,7 +641,7 @@ public void testSimpleNestedSortingWithNestedFilterMissing() throws Exception {
             );
 
         if (randomBoolean()) {
-            searchRequestBuilder.setScroll("10m");
+            searchRequestBuilder.setScroll(TimeValue.timeValueMinutes(10));
         }
 
         assertResponse(searchRequestBuilder, response -> {
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/scroll/DuelScrollIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/scroll/DuelScrollIT.java
index 036467b8d0774..89d4c0cc74852 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/scroll/DuelScrollIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/scroll/DuelScrollIT.java
@@ -16,6 +16,7 @@
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.SearchHits;
@@ -48,7 +49,7 @@ public void testDuelQueryThenFetch() throws Exception {
                 SearchResponse searchScrollResponse = prepareSearch("index").setSearchType(context.searchType)
                     .addSort(context.sort)
                     .setSize(context.scrollRequestSize)
-                    .setScroll("10m")
+                    .setScroll(TimeValue.timeValueMinutes(10))
                     .get();
                 try {
 
@@ -65,7 +66,7 @@ public void testDuelQueryThenFetch() throws Exception {
                     String scrollId = searchScrollResponse.getScrollId();
                     while (true) {
                         searchScrollResponse.decRef();
-                        searchScrollResponse = client().prepareSearchScroll(scrollId).setScroll("10m").get();
+                        searchScrollResponse = client().prepareSearchScroll(scrollId).setScroll(TimeValue.timeValueMinutes(10)).get();
                         assertNoFailures(searchScrollResponse);
                         assertThat(searchScrollResponse.getHits().getTotalHits().value, equalTo((long) context.numDocs));
                         if (searchScrollResponse.getHits().getHits().length == 0) {
@@ -232,7 +233,7 @@ private void testDuelIndexOrder(SearchType searchType, boolean trackScores, int
                     .setQuery(QueryBuilders.matchQuery("foo", "true"))
                     .addSort(SortBuilders.fieldSort("_doc"))
                     .setTrackScores(trackScores)
-                    .setScroll("10m")
+                    .setScroll(TimeValue.timeValueMinutes(10))
                     .get();
 
                 int scrollDocs = 0;
@@ -251,7 +252,7 @@ private void testDuelIndexOrder(SearchType searchType, boolean trackScores, int
                         }
                         scrollDocs += scroll.getHits().getHits().length;
                         scroll.decRef();
-                        scroll = client().prepareSearchScroll(scroll.getScrollId()).setScroll("10m").get();
+                        scroll = client().prepareSearchScroll(scroll.getScrollId()).setScroll(TimeValue.timeValueMinutes(10)).get();
                     }
                     assertEquals(control.getHits().getTotalHits().value, scrollDocs);
                 } catch (AssertionError e) {
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/scroll/SearchScrollIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/scroll/SearchScrollIT.java
index 7dcdb92ec5680..03c217266d527 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/scroll/SearchScrollIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/scroll/SearchScrollIT.java
@@ -458,7 +458,7 @@ public void testDeepScrollingDoesNotBlowUp() throws Exception {
             SearchRequestBuilder builder = prepareSearch("index").setSearchType(searchType)
                 .setQuery(QueryBuilders.matchAllQuery())
                 .setSize(Integer.MAX_VALUE)
-                .setScroll("1m");
+                .setScroll(TimeValue.timeValueMinutes(1));
 
             SearchResponse response = builder.get();
             try {
@@ -477,7 +477,7 @@ public void testThatNonExistingScrollIdReturnsCorrectException() throws Exceptio
         prepareIndex("index").setId("1").setSource("field", "value").execute().get();
         refresh();
 
-        SearchResponse searchResponse = prepareSearch("index").setSize(1).setScroll("1m").get();
+        SearchResponse searchResponse = prepareSearch("index").setSize(1).setScroll(TimeValue.timeValueMinutes(1)).get();
         try {
             assertThat(searchResponse.getScrollId(), is(notNullValue()));
 
@@ -498,7 +498,8 @@ public void testStringSortMissingAscTerminates() throws Exception {
         refresh();
 
         assertResponse(
-            prepareSearch("test").addSort(new FieldSortBuilder("no_field").order(SortOrder.ASC).missing("_last")).setScroll("1m"),
+            prepareSearch("test").addSort(new FieldSortBuilder("no_field").order(SortOrder.ASC).missing("_last"))
+                .setScroll(TimeValue.timeValueMinutes(1)),
             response -> {
                 assertHitCount(response, 1);
                 assertSearchHits(response, "1");
@@ -510,7 +511,8 @@ public void testStringSortMissingAscTerminates() throws Exception {
         );
 
         assertResponse(
-            prepareSearch("test").addSort(new FieldSortBuilder("no_field").order(SortOrder.ASC).missing("_first")).setScroll("1m"),
+            prepareSearch("test").addSort(new FieldSortBuilder("no_field").order(SortOrder.ASC).missing("_first"))
+                .setScroll(TimeValue.timeValueMinutes(1)),
             response -> {
                 assertHitCount(response, 1);
                 assertSearchHits(response, "1");
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/searchafter/SearchAfterIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/searchafter/SearchAfterIT.java
index 776dd789d8d9e..a526e721da1ec 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/searchafter/SearchAfterIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/searchafter/SearchAfterIT.java
@@ -73,7 +73,7 @@ public void testsShouldFail() throws Exception {
                 prepareSearch("test").addSort("field1", SortOrder.ASC)
                     .setQuery(matchAllQuery())
                     .searchAfter(new Object[] { 0 })
-                    .setScroll("1m")
+                    .setScroll(TimeValue.timeValueMinutes(1))
             );
             assertThat(e.getMessage(), containsString("[search_after] cannot be used in a scroll context"));
         }
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java
index 8fbcfe9caafdd..44b0a22f352ac 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java
@@ -37,6 +37,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsFilter;
 import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.seqno.RetentionLeaseActions;
@@ -237,7 +238,12 @@ public void testRestoreIndexWithMissingShards() throws Exception {
         logger.info("--> shutdown one of the nodes");
         internalCluster().stopRandomDataNode();
         assertThat(
-            clusterAdmin().prepareHealth().setWaitForEvents(Priority.LANGUID).setTimeout("1m").setWaitForNodes("<2").get().isTimedOut(),
+            clusterAdmin().prepareHealth()
+                .setWaitForEvents(Priority.LANGUID)
+                .setTimeout(TimeValue.timeValueMinutes(1))
+                .setWaitForNodes("<2")
+                .get()
+                .isTimedOut(),
             equalTo(false)
         );
 
@@ -415,7 +421,12 @@ public boolean clearData(String nodeName) {
         });
 
         assertThat(
-            clusterAdmin().prepareHealth().setWaitForEvents(Priority.LANGUID).setTimeout("1m").setWaitForNodes("2").get().isTimedOut(),
+            clusterAdmin().prepareHealth()
+                .setWaitForEvents(Priority.LANGUID)
+                .setTimeout(TimeValue.timeValueMinutes(1))
+                .setWaitForNodes("2")
+                .get()
+                .isTimedOut(),
             equalTo(false)
         );
 
@@ -644,7 +655,7 @@ public void testRestoreShrinkIndex() throws Exception {
         assertAcked(indicesAdmin().prepareDelete(sourceIdx).get());
         assertAcked(indicesAdmin().prepareDelete(shrunkIdx).get());
         internalCluster().stopRandomDataNode();
-        clusterAdmin().prepareHealth().setTimeout("30s").setWaitForNodes("1");
+        clusterAdmin().prepareHealth().setTimeout(TimeValue.timeValueSeconds(30)).setWaitForNodes("1");
 
         logger.info("--> start a new data node");
         final Settings dataSettings = Settings.builder()
@@ -652,7 +663,7 @@ public void testRestoreShrinkIndex() throws Exception {
             .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir()) // to get a new node id
             .build();
         internalCluster().startDataOnlyNode(dataSettings);
-        clusterAdmin().prepareHealth().setTimeout("30s").setWaitForNodes("2");
+        clusterAdmin().prepareHealth().setTimeout(TimeValue.timeValueSeconds(30)).setWaitForNodes("2");
 
         logger.info("--> restore the shrunk index and ensure all shards are allocated");
         RestoreSnapshotResponse restoreResponse = clusterAdmin().prepareRestoreSnapshot(repo, snapshot)
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/RepositoriesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/RepositoriesIT.java
index 6d36ce6924826..313c395b6bc24 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/RepositoriesIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/RepositoriesIT.java
@@ -29,6 +29,7 @@
 import org.elasticsearch.common.io.FileSystemUtils;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeUnit;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.repositories.RepositoriesService;
 import org.elasticsearch.repositories.RepositoryConflictException;
@@ -171,7 +172,7 @@ public void testRepositoryAckTimeout() {
                     .put("compress", randomBoolean())
                     .put("chunk_size", randomIntBetween(5, 100), ByteSizeUnit.BYTES)
             )
-            .setTimeout("0s")
+            .setTimeout(TimeValue.ZERO)
             .get();
         assertThat(putRepositoryResponse.isAcknowledged(), equalTo(false));
 
@@ -188,7 +189,9 @@ public void testRepositoryAckTimeout() {
         assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true));
 
         logger.info("-->  deleting repository test-repo-2 with 0s timeout - shouldn't ack");
-        AcknowledgedResponse deleteRepositoryResponse = clusterAdmin().prepareDeleteRepository("test-repo-2").setTimeout("0s").get();
+        AcknowledgedResponse deleteRepositoryResponse = clusterAdmin().prepareDeleteRepository("test-repo-2")
+            .setTimeout(TimeValue.ZERO)
+            .get();
         assertThat(deleteRepositoryResponse.isAcknowledged(), equalTo(false));
 
         logger.info("-->  deleting repository test-repo-1 with standard timeout - should ack");
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthRequest.java
index 02b41b01cef68..7bf0c976d52a5 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthRequest.java
@@ -127,10 +127,6 @@ public ClusterHealthRequest timeout(TimeValue timeout) {
         return this;
     }
 
-    public ClusterHealthRequest timeout(String timeout) {
-        return this.timeout(TimeValue.parseTimeValue(timeout, null, getClass().getSimpleName() + ".timeout"));
-    }
-
     public ClusterHealthStatus waitForStatus() {
         return waitForStatus;
     }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthRequestBuilder.java
index cfd1fd71b612e..ac70049162e5c 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthRequestBuilder.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthRequestBuilder.java
@@ -40,11 +40,6 @@ public ClusterHealthRequestBuilder setTimeout(TimeValue timeout) {
         return this;
     }
 
-    public ClusterHealthRequestBuilder setTimeout(String timeout) {
-        request.timeout(timeout);
-        return this;
-    }
-
     public ClusterHealthRequestBuilder setWaitForStatus(ClusterHealthStatus waitForStatus) {
         request.waitForStatus(waitForStatus);
         return this;
diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java
index 6998ca4150ad5..b73c853421e71 100644
--- a/server/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java
@@ -355,13 +355,6 @@ public final BulkRequest routing(String globalRouting) {
         return this;
     }
 
-    /**
-     * A timeout to wait if the index operation can't be performed immediately. Defaults to {@code 1m}.
-     */
-    public final BulkRequest timeout(String timeout) {
-        return timeout(TimeValue.parseTimeValue(timeout, null, getClass().getSimpleName() + ".timeout"));
-    }
-
     public TimeValue timeout() {
         return timeout;
     }
diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkRequestBuilder.java
index 6a90c46fc7fab..31e54cdb872ca 100644
--- a/server/src/main/java/org/elasticsearch/action/bulk/BulkRequestBuilder.java
+++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkRequestBuilder.java
@@ -164,14 +164,6 @@ public final BulkRequestBuilder setTimeout(TimeValue timeout) {
         return this;
     }
 
-    /**
-     * A timeout to wait if the index operation can't be performed immediately. Defaults to {@code 1m}.
-     */
-    public final BulkRequestBuilder setTimeout(String timeout) {
-        this.timeout = TimeValue.parseTimeValue(timeout, null, getClass().getSimpleName() + ".timeout");
-        return this;
-    }
-
     /**
      * The number of actions currently in the bulk.
      */
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java
index 5c0db65868dbc..a20846ab98a4d 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java
@@ -692,13 +692,6 @@ public SearchRequest scroll(TimeValue keepAlive) {
         return scroll(new Scroll(keepAlive));
     }
 
-    /**
-     * If set, will enable scrolling of the search request for the specified timeout.
-     */
-    public SearchRequest scroll(String keepAlive) {
-        return scroll(new Scroll(TimeValue.parseTimeValue(keepAlive, null, getClass().getSimpleName() + ".Scroll.keepAlive")));
-    }
-
     /**
      * Sets if this request should use the request cache or not, assuming that it can (for
      * example, if "now" is used, it will never be cached). By default (not set, or null,
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java
index 60f92cacba963..5be696da5c6b1 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java
@@ -77,14 +77,6 @@ public SearchRequestBuilder setScroll(TimeValue keepAlive) {
         return this;
     }
 
-    /**
-     * If set, will enable scrolling of the search request for the specified timeout.
-     */
-    public SearchRequestBuilder setScroll(String keepAlive) {
-        request.scroll(keepAlive);
-        return this;
-    }
-
     /**
      * An optional timeout to control how long search is allowed to take.
      */
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchScrollRequest.java b/server/src/main/java/org/elasticsearch/action/search/SearchScrollRequest.java
index af225c8f5d8ef..c49518b0523eb 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchScrollRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchScrollRequest.java
@@ -97,13 +97,6 @@ public SearchScrollRequest scroll(TimeValue keepAlive) {
         return scroll(new Scroll(keepAlive));
     }
 
-    /**
-     * If set, will enable scrolling of the search request for the specified timeout.
-     */
-    public SearchScrollRequest scroll(String keepAlive) {
-        return scroll(new Scroll(TimeValue.parseTimeValue(keepAlive, null, getClass().getSimpleName() + ".keepAlive")));
-    }
-
     @Override
     public Task createTask(long id, String type, String action, TaskId parentTaskId, Map<String, String> headers) {
         return new SearchTask(id, type, action, this::getDescription, parentTaskId, headers);
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchScrollRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/search/SearchScrollRequestBuilder.java
index d5c55280b1917..f75dc52b5cd7c 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchScrollRequestBuilder.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchScrollRequestBuilder.java
@@ -49,12 +49,4 @@ public SearchScrollRequestBuilder setScroll(TimeValue keepAlive) {
         request.scroll(keepAlive);
         return this;
     }
-
-    /**
-     * If set, will enable scrolling of the search request for the specified timeout.
-     */
-    public SearchScrollRequestBuilder setScroll(String keepAlive) {
-        request.scroll(keepAlive);
-        return this;
-    }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/support/master/AcknowledgedRequest.java b/server/src/main/java/org/elasticsearch/action/support/master/AcknowledgedRequest.java
index ff39f1e83ba5f..7e271536be9fe 100644
--- a/server/src/main/java/org/elasticsearch/action/support/master/AcknowledgedRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/support/master/AcknowledgedRequest.java
@@ -53,17 +53,6 @@ protected AcknowledgedRequest(StreamInput in) throws IOException {
         this.ackTimeout = in.readTimeValue();
     }
 
-    /**
-     * Sets the {@link #ackTimeout}, which specifies how long to wait for all relevant nodes to apply a cluster state update and acknowledge
-     * this to the elected master.
-     *
-     * @param ackTimeout timeout as a string
-     * @return this request, for method chaining.
-     */
-    public final Request ackTimeout(String ackTimeout) {
-        return ackTimeout(TimeValue.parseTimeValue(ackTimeout, this.ackTimeout, getClass().getSimpleName() + ".ackTimeout"));
-    }
-
     /**
      * Sets the {@link #ackTimeout}, which specifies how long to wait for all relevant nodes to apply a cluster state update and acknowledge
      * this to the elected master.
diff --git a/server/src/main/java/org/elasticsearch/action/support/master/AcknowledgedRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/support/master/AcknowledgedRequestBuilder.java
index 65892094b4c46..a749be99e2808 100644
--- a/server/src/main/java/org/elasticsearch/action/support/master/AcknowledgedRequestBuilder.java
+++ b/server/src/main/java/org/elasticsearch/action/support/master/AcknowledgedRequestBuilder.java
@@ -36,13 +36,4 @@ public RequestBuilder setTimeout(TimeValue timeout) {
         return (RequestBuilder) this;
     }
 
-    /**
-     * Timeout to wait for the operation to be acknowledged by current cluster nodes. Defaults
-     * to {@code 10s}.
-     */
-    @SuppressWarnings("unchecked")
-    public RequestBuilder setTimeout(String timeout) {
-        request.ackTimeout(timeout);
-        return (RequestBuilder) this;
-    }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/support/master/MasterNodeOperationRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/support/master/MasterNodeOperationRequestBuilder.java
index a4aea92973363..3f5e89eb3b18c 100644
--- a/server/src/main/java/org/elasticsearch/action/support/master/MasterNodeOperationRequestBuilder.java
+++ b/server/src/main/java/org/elasticsearch/action/support/master/MasterNodeOperationRequestBuilder.java
@@ -37,13 +37,4 @@ public final RequestBuilder setMasterNodeTimeout(TimeValue timeout) {
         return (RequestBuilder) this;
     }
 
-    /**
-     * Sets the master node timeout in case the master has not yet been discovered.
-     */
-    @SuppressWarnings("unchecked")
-    public final RequestBuilder setMasterNodeTimeout(String timeout) {
-        request.masterNodeTimeout(timeout);
-        return (RequestBuilder) this;
-    }
-
 }
diff --git a/server/src/main/java/org/elasticsearch/action/support/master/MasterNodeRequest.java b/server/src/main/java/org/elasticsearch/action/support/master/MasterNodeRequest.java
index 8edf28083a0da..6459f6c1b458a 100644
--- a/server/src/main/java/org/elasticsearch/action/support/master/MasterNodeRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/support/master/MasterNodeRequest.java
@@ -48,14 +48,6 @@ public final Request masterNodeTimeout(TimeValue timeout) {
         return (Request) this;
     }
 
-    /**
-     * Specifies how long to wait when the master has not been discovered yet, or is disconnected, or is busy processing other tasks. The
-     * value {@link TimeValue#MINUS_ONE} means to wait forever.
-     */
-    public final Request masterNodeTimeout(String timeout) {
-        return masterNodeTimeout(TimeValue.parseTimeValue(timeout, null, getClass().getSimpleName() + ".masterNodeTimeout"));
-    }
-
     /**
      * @return how long to wait when the master has not been discovered yet, or is disconnected, or is busy processing other tasks. The
      * value {@link TimeValue#MINUS_ONE} means to wait forever.
diff --git a/server/src/main/java/org/elasticsearch/action/support/nodes/BaseNodesRequest.java b/server/src/main/java/org/elasticsearch/action/support/nodes/BaseNodesRequest.java
index 5d6eca2ef005e..8a2e7684cadfa 100644
--- a/server/src/main/java/org/elasticsearch/action/support/nodes/BaseNodesRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/support/nodes/BaseNodesRequest.java
@@ -77,12 +77,6 @@ public final Request timeout(TimeValue timeout) {
         return (Request) this;
     }
 
-    @SuppressWarnings("unchecked")
-    public final Request timeout(String timeout) {
-        this.timeout = TimeValue.parseTimeValue(timeout, null, getClass().getSimpleName() + ".timeout");
-        return (Request) this;
-    }
-
     public DiscoveryNode[] concreteNodes() {
         return concreteNodes;
     }
diff --git a/server/src/main/java/org/elasticsearch/action/support/nodes/NodesOperationRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/support/nodes/NodesOperationRequestBuilder.java
index 5c606fb2b432c..bcbbe64a03d5e 100644
--- a/server/src/main/java/org/elasticsearch/action/support/nodes/NodesOperationRequestBuilder.java
+++ b/server/src/main/java/org/elasticsearch/action/support/nodes/NodesOperationRequestBuilder.java
@@ -36,9 +36,4 @@ public RequestBuilder setTimeout(TimeValue timeout) {
         return (RequestBuilder) this;
     }
 
-    @SuppressWarnings("unchecked")
-    public final RequestBuilder setTimeout(String timeout) {
-        request.timeout(timeout);
-        return (RequestBuilder) this;
-    }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/support/replication/ReplicationRequest.java b/server/src/main/java/org/elasticsearch/action/support/replication/ReplicationRequest.java
index bff8a8b0f66b6..1da69d76ebc82 100644
--- a/server/src/main/java/org/elasticsearch/action/support/replication/ReplicationRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/support/replication/ReplicationRequest.java
@@ -98,13 +98,6 @@ public final Request timeout(TimeValue timeout) {
         return (Request) this;
     }
 
-    /**
-     * A timeout to wait if the index operation can't be performed immediately. Defaults to {@code 1m}.
-     */
-    public final Request timeout(String timeout) {
-        return timeout(TimeValue.parseTimeValue(timeout, null, getClass().getSimpleName() + ".timeout"));
-    }
-
     public TimeValue timeout() {
         return timeout;
     }
diff --git a/server/src/main/java/org/elasticsearch/action/support/replication/ReplicationRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/support/replication/ReplicationRequestBuilder.java
index 8eb82af2091cd..a918618fc78ec 100644
--- a/server/src/main/java/org/elasticsearch/action/support/replication/ReplicationRequestBuilder.java
+++ b/server/src/main/java/org/elasticsearch/action/support/replication/ReplicationRequestBuilder.java
@@ -38,15 +38,6 @@ public RequestBuilder setTimeout(TimeValue timeout) {
         return (RequestBuilder) this;
     }
 
-    /**
-     * A timeout to wait if the index operation can't be performed immediately. Defaults to {@code 1m}.
-     */
-    @SuppressWarnings("unchecked")
-    public RequestBuilder setTimeout(String timeout) {
-        this.timeout = TimeValue.parseTimeValue(timeout, null, getClass().getSimpleName() + ".timeout");
-        return (RequestBuilder) this;
-    }
-
     @SuppressWarnings("unchecked")
     public RequestBuilder setIndex(String index) {
         this.index = index;
diff --git a/server/src/main/java/org/elasticsearch/action/support/single/instance/InstanceShardOperationRequest.java b/server/src/main/java/org/elasticsearch/action/support/single/instance/InstanceShardOperationRequest.java
index 7fab9fea12fdc..e689492523838 100644
--- a/server/src/main/java/org/elasticsearch/action/support/single/instance/InstanceShardOperationRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/support/single/instance/InstanceShardOperationRequest.java
@@ -106,13 +106,6 @@ public final Request timeout(TimeValue timeout) {
         return (Request) this;
     }
 
-    /**
-     * A timeout to wait if the index operation can't be performed immediately. Defaults to {@code 1m}.
-     */
-    public final Request timeout(String timeout) {
-        return timeout(TimeValue.parseTimeValue(timeout, null, getClass().getSimpleName() + ".timeout"));
-    }
-
     public String concreteIndex() {
         return concreteIndex;
     }
diff --git a/server/src/main/java/org/elasticsearch/action/support/single/instance/InstanceShardOperationRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/support/single/instance/InstanceShardOperationRequestBuilder.java
index 64efcda2f14db..25278c71e1fe2 100644
--- a/server/src/main/java/org/elasticsearch/action/support/single/instance/InstanceShardOperationRequestBuilder.java
+++ b/server/src/main/java/org/elasticsearch/action/support/single/instance/InstanceShardOperationRequestBuilder.java
@@ -46,15 +46,6 @@ public RequestBuilder setTimeout(TimeValue timeout) {
         return (RequestBuilder) this;
     }
 
-    /**
-     * A timeout to wait if the index operation can't be performed immediately. Defaults to {@code 1m}.
-     */
-    @SuppressWarnings("unchecked")
-    public RequestBuilder setTimeout(String timeout) {
-        this.timeout = TimeValue.parseTimeValue(timeout, null, getClass().getSimpleName() + ".timeout");
-        return (RequestBuilder) this;
-    }
-
     protected void apply(Request request) {
         if (index != null) {
             request.index(index);
diff --git a/server/src/main/java/org/elasticsearch/action/support/tasks/BaseTasksRequest.java b/server/src/main/java/org/elasticsearch/action/support/tasks/BaseTasksRequest.java
index 1f71e4d1f6ff6..56f2f1f16ed1b 100644
--- a/server/src/main/java/org/elasticsearch/action/support/tasks/BaseTasksRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/support/tasks/BaseTasksRequest.java
@@ -170,12 +170,6 @@ public final Request setTimeout(TimeValue timeout) {
         return (Request) this;
     }
 
-    @SuppressWarnings("unchecked")
-    public final Request setTimeout(String timeout) {
-        this.timeout = TimeValue.parseTimeValue(timeout, null, getClass().getSimpleName() + ".timeout");
-        return (Request) this;
-    }
-
     public boolean match(Task task) {
         if (CollectionUtils.isEmpty(getActions()) == false && Regex.simpleMatch(getActions(), task.getAction()) == false) {
             return false;
diff --git a/server/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java b/server/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java
index 417b881ed91ae..4e708d58a8a63 100644
--- a/server/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java
+++ b/server/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java
@@ -252,14 +252,6 @@ public Self setTimeout(TimeValue timeout) {
         return self();
     }
 
-    /**
-     * Timeout to wait for the shards on to be available for each bulk request?
-     */
-    public Self setTimeout(String timeout) {
-        this.timeout = TimeValue.parseTimeValue(timeout, this.timeout, getClass().getSimpleName() + ".timeout");
-        return self();
-    }
-
     /**
      * The number of shard copies that must be active before proceeding with the write.
      */
diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterStatsAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterStatsAction.java
index 78d20c07dd3ff..ad8f5330d9780 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterStatsAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterStatsAction.java
@@ -38,7 +38,7 @@ public String getName() {
     @Override
     public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         ClusterStatsRequest clusterStatsRequest = new ClusterStatsRequest().nodesIds(request.paramAsStringArray("nodeId", null));
-        clusterStatsRequest.timeout(request.param("timeout"));
+        clusterStatsRequest.timeout(request.paramAsTime("timeout", null));
         return channel -> new RestCancellableNodeClient(client, request.getHttpChannel()).admin()
             .cluster()
             .clusterStats(clusterStatsRequest, new NodesResponseRestListener<>(channel));
diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesHotThreadsAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesHotThreadsAction.java
index bc0750f16e0e7..d866140844926 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesHotThreadsAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesHotThreadsAction.java
@@ -14,7 +14,6 @@
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.core.RestApiVersion;
-import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.monitor.jvm.HotThreads;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
@@ -110,9 +109,9 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC
         nodesHotThreadsRequest.sortOrder(
             HotThreads.SortOrder.of(request.param("sort", nodesHotThreadsRequest.sortOrder().getOrderValue()))
         );
-        nodesHotThreadsRequest.interval(TimeValue.parseTimeValue(request.param("interval"), nodesHotThreadsRequest.interval(), "interval"));
+        nodesHotThreadsRequest.interval(request.paramAsTime("interval", nodesHotThreadsRequest.interval()));
         nodesHotThreadsRequest.snapshots(request.paramAsInt("snapshots", nodesHotThreadsRequest.snapshots()));
-        nodesHotThreadsRequest.timeout(request.param("timeout"));
+        nodesHotThreadsRequest.timeout(request.paramAsTime("timeout", nodesHotThreadsRequest.timeout()));
         return channel -> client.execute(TransportNodesHotThreadsAction.TYPE, nodesHotThreadsRequest, new RestResponseListener<>(channel) {
             @Override
             public RestResponse buildResponse(NodesHotThreadsResponse response) {
diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesInfoAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesInfoAction.java
index fc04374411536..0834f83b3cf98 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesInfoAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesInfoAction.java
@@ -86,7 +86,7 @@ static NodesInfoRequest prepareRequest(final RestRequest request) {
         }
 
         final NodesInfoRequest nodesInfoRequest = new NodesInfoRequest(nodeIds);
-        nodesInfoRequest.timeout(request.param("timeout"));
+        nodesInfoRequest.timeout(request.paramAsTime("timeout", null));
         // shortcut, don't do checks if only all is specified
         if (metrics.size() == 1 && metrics.contains("_all")) {
             nodesInfoRequest.all();
diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesStatsAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesStatsAction.java
index d311f39f42f7a..d9039749cd46b 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesStatsAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesStatsAction.java
@@ -90,7 +90,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC
         Set<String> metrics = Strings.tokenizeByCommaToSet(request.param("metric", "_all"));
 
         NodesStatsRequest nodesStatsRequest = new NodesStatsRequest(nodesIds);
-        nodesStatsRequest.timeout(request.param("timeout"));
+        nodesStatsRequest.timeout(request.paramAsTime("timeout", null));
         // level parameter validation
         nodesStatsRequest.setIncludeShardsStats(NodeStatsLevel.of(request, NodeStatsLevel.NODE) != NodeStatsLevel.NODE);
 
diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesUsageAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesUsageAction.java
index da66cdf898d6d..45eb046b21731 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesUsageAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesUsageAction.java
@@ -48,7 +48,7 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli
         Set<String> metrics = Strings.tokenizeByCommaToSet(request.param("metric", "_all"));
 
         NodesUsageRequest nodesUsageRequest = new NodesUsageRequest(nodesIds);
-        nodesUsageRequest.timeout(request.param("timeout"));
+        nodesUsageRequest.timeout(request.paramAsTime("timeout", null));
 
         if (metrics.size() == 1 && metrics.contains("_all")) {
             nodesUsageRequest.all();
diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestReloadSecureSettingsAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestReloadSecureSettingsAction.java
index fce50eec6fc01..4f2ad461ff046 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestReloadSecureSettingsAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestReloadSecureSettingsAction.java
@@ -63,7 +63,7 @@ public List<Route> routes() {
     public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
         final NodesReloadSecureSettingsRequest reloadSecureSettingsRequest = new NodesReloadSecureSettingsRequest();
         reloadSecureSettingsRequest.nodesIds(Strings.splitStringByCommaToArray(request.param("nodeId")));
-        reloadSecureSettingsRequest.timeout(request.param("timeout"));
+        reloadSecureSettingsRequest.timeout(request.paramAsTime("timeout", null));
         request.withContentOrSourceParamParserOrNull(parser -> {
             if (parser != null) {
                 final ParsedRequestBody parsedRequestBody = PARSER.parse(parser, null);
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequestTests.java
index dc4ad8ac45866..794be6e463548 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequestTests.java
@@ -11,6 +11,7 @@
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
 import org.elasticsearch.xcontent.ToXContent.MapParams;
@@ -96,7 +97,7 @@ public void testToXContent() throws IOException {
         }
 
         if (randomBoolean()) {
-            original.masterNodeTimeout("60s");
+            original.masterNodeTimeout(TimeValue.timeValueMinutes(1));
         }
 
         XContentBuilder builder = original.toXContent(XContentFactory.jsonBuilder(), new MapParams(Collections.emptyMap()));
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/state/ClusterStateRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/state/ClusterStateRequestTests.java
index 2f757b9dc2905..1b1535185bef7 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/state/ClusterStateRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/state/ClusterStateRequestTests.java
@@ -86,7 +86,7 @@ private static void assertOptionsMatch(IndicesOptions in, IndicesOptions out) {
     public void testDescription() {
         assertThat(new ClusterStateRequest().clear().getDescription(), equalTo("cluster state [master timeout [30s]]"));
         assertThat(
-            new ClusterStateRequest().masterNodeTimeout("5m").getDescription(),
+            new ClusterStateRequest().masterNodeTimeout(TimeValue.timeValueMinutes(5)).getDescription(),
             equalTo("cluster state [routing table, nodes, metadata, blocks, customs, master timeout [5m]]")
         );
         assertThat(new ClusterStateRequest().clear().routingTable(true).getDescription(), containsString("routing table"));
diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchScrollRequestTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchScrollRequestTests.java
index 73c8b35706b3e..f41bc7fa30968 100644
--- a/server/src/test/java/org/elasticsearch/action/search/SearchScrollRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/SearchScrollRequestTests.java
@@ -93,7 +93,7 @@ public void testFromXContentWithUnknownParamThrowsException() throws Exception {
     public void testToXContent() throws IOException {
         SearchScrollRequest searchScrollRequest = new SearchScrollRequest();
         searchScrollRequest.scrollId("SCROLL_ID");
-        searchScrollRequest.scroll("1m");
+        searchScrollRequest.scroll(TimeValue.timeValueMinutes(1));
         try (XContentBuilder builder = JsonXContent.contentBuilder()) {
             searchScrollRequest.toXContent(builder, ToXContent.EMPTY_PARAMS);
             assertEquals("""
diff --git a/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java b/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java
index 2613ce7e5a65d..fea391e8205f5 100644
--- a/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java
@@ -1312,7 +1312,7 @@ public void testShouldMinimizeRoundtrips() throws Exception {
         }
         {
             SearchRequest searchRequest = new SearchRequest();
-            searchRequest.scroll("5s");
+            searchRequest.scroll(TimeValue.timeValueSeconds(5));
             assertFalse(TransportSearchAction.shouldMinimizeRoundtrips(searchRequest));
         }
         {
@@ -1467,7 +1467,7 @@ public void testShouldPreFilterSearchShards() {
         }
         {
             SearchRequest searchRequest = new SearchRequest().source(new SearchSourceBuilder().sort(SortBuilders.fieldSort("timestamp")))
-                .scroll("5m");
+                .scroll(TimeValue.timeValueMinutes(5));
             assertTrue(
                 TransportSearchAction.shouldPreFilterSearchShards(
                     clusterState,
@@ -1553,7 +1553,7 @@ public void testShouldPreFilterSearchShardsWithReadOnly() {
             SearchRequest searchRequest = new SearchRequest().source(
                 new SearchSourceBuilder().query(QueryBuilders.rangeQuery("timestamp"))
             );
-            searchRequest.scroll("5s");
+            searchRequest.scroll(TimeValue.timeValueSeconds(5));
             assertTrue(
                 TransportSearchAction.shouldPreFilterSearchShards(
                     clusterState,
diff --git a/server/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java b/server/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java
index 9b8b501912bac..fcbddb581946b 100644
--- a/server/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java
@@ -49,6 +49,7 @@
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.Releasable;
 import org.elasticsearch.core.Strings;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.IndexService;
@@ -277,7 +278,7 @@ public ClusterBlockLevel indexBlockLevel() {
         {
             setStateWithBlock(clusterService, retryableBlock, globalBlock);
 
-            Request requestWithTimeout = (globalBlock ? new Request(shardId) : new Request(shardId)).timeout("5ms");
+            Request requestWithTimeout = new Request(shardId).timeout(TimeValue.timeValueMillis(5));
             PlainActionFuture<TestResponse> listener = new PlainActionFuture<>();
             ReplicationTask task = maybeTask();
 
@@ -318,7 +319,8 @@ public ClusterBlockLevel indexBlockLevel() {
             assertIndexShardUninitialized();
         }
         {
-            Request requestWithTimeout = new Request(new ShardId("unknown", "_na_", 0)).index("unknown").timeout("5ms");
+            Request requestWithTimeout = new Request(new ShardId("unknown", "_na_", 0)).index("unknown")
+                .timeout(TimeValue.timeValueMillis(5));
             PlainActionFuture<TestResponse> listener = new PlainActionFuture<>();
             ReplicationTask task = maybeTask();
 
@@ -425,7 +427,7 @@ public void testNotStartedPrimary() {
 
         logger.debug("--> using initial state:\n{}", clusterService.state());
 
-        Request request = new Request(shardId).timeout("1ms");
+        Request request = new Request(shardId).timeout(TimeValue.timeValueMillis(1));
         PlainActionFuture<TestResponse> listener = new PlainActionFuture<>();
         TestAction.ReroutePhase reroutePhase = action.new ReroutePhase(task, request, listener);
         reroutePhase.run();
@@ -513,7 +515,8 @@ public void testNoRerouteOnStaleClusterState() {
         setState(clusterService, state);
         logger.debug("--> relocation ongoing state:\n{}", clusterService.state());
 
-        Request request = new Request(shardId).timeout("1ms").routedBasedOnClusterVersion(clusterService.state().version() + 1);
+        Request request = new Request(shardId).timeout(TimeValue.timeValueMillis(1))
+            .routedBasedOnClusterVersion(clusterService.state().version() + 1);
         PlainActionFuture<TestResponse> listener = new PlainActionFuture<>();
         TestAction.ReroutePhase reroutePhase = action.new ReroutePhase(null, request, listener);
         reroutePhase.run();
@@ -555,7 +558,7 @@ public void testUnknownIndexOrShardOnReroute() {
         setState(clusterService, state(index, true, randomBoolean() ? ShardRoutingState.INITIALIZING : ShardRoutingState.UNASSIGNED));
         logger.debug("--> using initial state:\n{}", clusterService.state());
 
-        Request request = new Request(new ShardId("unknown_index", "_na_", 0)).timeout("1ms");
+        Request request = new Request(new ShardId("unknown_index", "_na_", 0)).timeout(TimeValue.timeValueMillis(1));
         PlainActionFuture<TestResponse> listener = new PlainActionFuture<>();
         ReplicationTask task = maybeTask();
 
@@ -567,7 +570,7 @@ public void testUnknownIndexOrShardOnReroute() {
 
         // try again with a request that is based on a newer cluster state, make sure we waited until that
         // cluster state for the index to appear
-        request = new Request(new ShardId("unknown_index", "_na_", 0)).timeout("1ms");
+        request = new Request(new ShardId("unknown_index", "_na_", 0)).timeout(TimeValue.timeValueMillis(1));
         request.routedBasedOnClusterVersion(clusterService.state().version() + 1);
         listener = new PlainActionFuture<>();
         task = maybeTask();
@@ -578,7 +581,7 @@ public void testUnknownIndexOrShardOnReroute() {
         assertPhase(task, "failed");
         assertTrue(request.isRetrySet.get());
 
-        request = new Request(new ShardId(index, "_na_", 10)).timeout("1ms");
+        request = new Request(new ShardId(index, "_na_", 10)).timeout(TimeValue.timeValueMillis(1));
         listener = new PlainActionFuture<>();
         reroutePhase = action.new ReroutePhase(null, request, listener);
         reroutePhase.run();
@@ -602,7 +605,9 @@ public void testClosedIndexOnReroute() {
         );
         assertThat(clusterService.state().metadata().indices().get(index).getState(), equalTo(IndexMetadata.State.CLOSE));
         logger.debug("--> using initial state:\n{}", clusterService.state());
-        Request request = new Request(new ShardId(clusterService.state().metadata().indices().get(index).getIndex(), 0)).timeout("1ms");
+        Request request = new Request(new ShardId(clusterService.state().metadata().indices().get(index).getIndex(), 0)).timeout(
+            TimeValue.timeValueMillis(1)
+        );
         PlainActionFuture<TestResponse> listener = new PlainActionFuture<>();
         ReplicationTask task = maybeTask();
 
@@ -631,9 +636,9 @@ public void testStalePrimaryShardOnReroute() {
         Request request = new Request(shardId);
         boolean timeout = randomBoolean();
         if (timeout) {
-            request.timeout("0s");
+            request.timeout(TimeValue.ZERO);
         } else {
-            request.timeout("1h");
+            request.timeout(TimeValue.timeValueHours(1));
         }
         PlainActionFuture<TestResponse> listener = new PlainActionFuture<>();
         ReplicationTask task = maybeTask();
@@ -714,7 +719,7 @@ public void testPrimaryPhaseExecutesOrDelegatesRequestToRelocationTarget() throw
         final ShardId shardId = new ShardId(index, "_na_", 0);
         ClusterState state = stateWithActivePrimary(index, true, randomInt(5));
         setState(clusterService, state);
-        Request request = new Request(shardId).timeout("1ms");
+        Request request = new Request(shardId).timeout(TimeValue.timeValueMillis(1));
         PlainActionFuture<TestResponse> listener = new PlainActionFuture<>();
         ReplicationTask task = maybeTask();
         AtomicBoolean executed = new AtomicBoolean();
@@ -784,7 +789,7 @@ public void testPrimaryPhaseExecutesDelegatedRequestOnRelocationTarget() {
         // simulate execution of the primary phase on the relocation target node
         state = ClusterState.builder(state).nodes(DiscoveryNodes.builder(state.nodes()).localNodeId(primaryTargetNodeId)).build();
         setState(clusterService, state);
-        Request request = new Request(shardId).timeout("1ms");
+        Request request = new Request(shardId).timeout(TimeValue.timeValueMillis(1));
         PlainActionFuture<TestResponse> listener = new PlainActionFuture<>();
         ReplicationTask task = maybeTask();
         AtomicBoolean executed = new AtomicBoolean();
@@ -1121,7 +1126,7 @@ public void testPrimaryActionRejectsWrongAidOrWrongTerm() throws Exception {
         PlainActionFuture<TransportResponse> listener = new PlainActionFuture<>();
         final boolean wrongAllocationId = randomBoolean();
         final long requestTerm = wrongAllocationId && randomBoolean() ? primaryTerm : primaryTerm + randomIntBetween(1, 10);
-        Request request = new Request(shardId).timeout("1ms");
+        Request request = new Request(shardId).timeout(TimeValue.timeValueMillis(1));
         action.handlePrimaryRequest(
             new TransportReplicationAction.ConcreteShardRequest<>(
                 request,
@@ -1171,7 +1176,7 @@ public void testReplicaActionRejectsWrongAid() throws Exception {
         setState(clusterService, state);
 
         PlainActionFuture<TransportResponse> listener = new PlainActionFuture<>();
-        Request request = new Request(shardId).timeout("1ms");
+        Request request = new Request(shardId).timeout(TimeValue.timeValueMillis(1));
         action.handleReplicaRequest(
             new TransportReplicationAction.ConcreteReplicaRequest<>(
                 request,
diff --git a/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java b/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java
index b0c4ef00230d5..3c81fd60fe25c 100644
--- a/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java
@@ -246,7 +246,7 @@ public void testClearOnClose() {
         createIndex("index");
         prepareIndex("index").setId("1").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get();
         assertResponse(
-            client().prepareSearch("index").setSize(1).setScroll("1m"),
+            client().prepareSearch("index").setSize(1).setScroll(TimeValue.timeValueMinutes(1)),
             searchResponse -> assertThat(searchResponse.getScrollId(), is(notNullValue()))
         );
         SearchService service = getInstanceFromNode(SearchService.class);
@@ -260,7 +260,7 @@ public void testClearOnStop() {
         createIndex("index");
         prepareIndex("index").setId("1").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get();
         assertResponse(
-            client().prepareSearch("index").setSize(1).setScroll("1m"),
+            client().prepareSearch("index").setSize(1).setScroll(TimeValue.timeValueMinutes(1)),
             searchResponse -> assertThat(searchResponse.getScrollId(), is(notNullValue()))
         );
         SearchService service = getInstanceFromNode(SearchService.class);
@@ -274,7 +274,7 @@ public void testClearIndexDelete() {
         createIndex("index");
         prepareIndex("index").setId("1").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get();
         assertResponse(
-            client().prepareSearch("index").setSize(1).setScroll("1m"),
+            client().prepareSearch("index").setSize(1).setScroll(TimeValue.timeValueMinutes(1)),
             searchResponse -> assertThat(searchResponse.getScrollId(), is(notNullValue()))
         );
         SearchService service = getInstanceFromNode(SearchService.class);
@@ -478,7 +478,7 @@ public void testBeforeShardLockDuringShardCreate() {
         IndexService indexService = createIndex("index", Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).build());
         prepareIndex("index").setId("1").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get();
         assertResponse(
-            client().prepareSearch("index").setSize(1).setScroll("1m"),
+            client().prepareSearch("index").setSize(1).setScroll(TimeValue.timeValueMinutes(1)),
             searchResponse -> assertThat(searchResponse.getScrollId(), is(notNullValue()))
         );
         SearchService service = getInstanceFromNode(SearchService.class);
@@ -787,7 +787,7 @@ public void testMaxOpenScrollContexts() throws Exception {
         LinkedList<String> clearScrollIds = new LinkedList<>();
 
         for (int i = 0; i < SearchService.MAX_OPEN_SCROLL_CONTEXT.get(Settings.EMPTY); i++) {
-            assertResponse(client().prepareSearch("index").setSize(1).setScroll("1m"), searchResponse -> {
+            assertResponse(client().prepareSearch("index").setSize(1).setScroll(TimeValue.timeValueMinutes(1)), searchResponse -> {
                 if (randomInt(4) == 0) clearScrollIds.addLast(searchResponse.getScrollId());
             });
         }
@@ -797,7 +797,7 @@ public void testMaxOpenScrollContexts() throws Exception {
         client().clearScroll(clearScrollRequest);
 
         for (int i = 0; i < clearScrollIds.size(); i++) {
-            client().prepareSearch("index").setSize(1).setScroll("1m").get().decRef();
+            client().prepareSearch("index").setSize(1).setScroll(TimeValue.timeValueMinutes(1)).get().decRef();
         }
 
         final ShardScrollRequestTest request = new ShardScrollRequestTest(indexShard.shardId());
diff --git a/server/src/test/java/org/elasticsearch/search/slice/SliceBuilderTests.java b/server/src/test/java/org/elasticsearch/search/slice/SliceBuilderTests.java
index 35cac166f64c7..ba1453e464c64 100644
--- a/server/src/test/java/org/elasticsearch/search/slice/SliceBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/search/slice/SliceBuilderTests.java
@@ -24,6 +24,7 @@
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.Nullable;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.fielddata.IndexNumericFieldData;
@@ -111,7 +112,7 @@ private ShardSearchRequest createPointInTimeRequest(int shardIndex, int numShard
     }
 
     private ShardSearchRequest createScrollRequest(int shardIndex, int numShards) {
-        SearchRequest searchRequest = new SearchRequest().allowPartialSearchResults(true).scroll("1m");
+        SearchRequest searchRequest = new SearchRequest().allowPartialSearchResults(true).scroll(TimeValue.timeValueMinutes(1));
         return new ShardSearchRequest(
             OriginalIndices.NONE,
             searchRequest,
diff --git a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java
index dc899989eb042..8842d9ef35fec 100644
--- a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java
+++ b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java
@@ -735,7 +735,7 @@ private void putAutoFollowPatterns(String name, String[] patterns, List<String>
         // Need to set this, because following an index in the same cluster
         request.setFollowIndexNamePattern("copy-{{leader_index}}");
         if (randomBoolean()) {
-            request.masterNodeTimeout(randomFrom("10s", "20s", "30s"));
+            request.masterNodeTimeout(TimeValue.timeValueSeconds(randomFrom(10, 20, 30)));
         }
 
         assertTrue(followerClient().execute(PutAutoFollowPatternAction.INSTANCE, request).actionGet().isAcknowledged());
@@ -744,7 +744,7 @@ private void putAutoFollowPatterns(String name, String[] patterns, List<String>
     private void deleteAutoFollowPattern(final String name) {
         DeleteAutoFollowPatternAction.Request request = new DeleteAutoFollowPatternAction.Request(name);
         if (randomBoolean()) {
-            request.masterNodeTimeout(randomFrom("10s", "20s", "30s"));
+            request.masterNodeTimeout(TimeValue.timeValueSeconds(randomFrom(10, 20, 30)));
         }
         assertTrue(followerClient().execute(DeleteAutoFollowPatternAction.INSTANCE, request).actionGet().isAcknowledged());
     }
@@ -752,7 +752,7 @@ private void deleteAutoFollowPattern(final String name) {
     private AutoFollowStats getAutoFollowStats() {
         CcrStatsAction.Request request = new CcrStatsAction.Request();
         if (randomBoolean()) {
-            request.masterNodeTimeout(randomFrom("10s", "20s", "30s"));
+            request.masterNodeTimeout(TimeValue.timeValueSeconds(randomFrom(10, 20, 30)));
         }
         return followerClient().execute(CcrStatsAction.INSTANCE, request).actionGet().getAutoFollowStats();
     }
@@ -766,7 +766,7 @@ private void createLeaderIndex(String index, Settings settings) {
     private void pauseAutoFollowPattern(final String name) {
         ActivateAutoFollowPatternAction.Request request = new ActivateAutoFollowPatternAction.Request(name, false);
         if (randomBoolean()) {
-            request.masterNodeTimeout(randomFrom("10s", "20s", "30s"));
+            request.masterNodeTimeout(TimeValue.timeValueSeconds(randomFrom(10, 20, 30)));
         }
         assertAcked(followerClient().execute(ActivateAutoFollowPatternAction.INSTANCE, request).actionGet());
     }
@@ -774,7 +774,7 @@ private void pauseAutoFollowPattern(final String name) {
     private void resumeAutoFollowPattern(final String name) {
         ActivateAutoFollowPatternAction.Request request = new ActivateAutoFollowPatternAction.Request(name, true);
         if (randomBoolean()) {
-            request.masterNodeTimeout(randomFrom("10s", "20s", "30s"));
+            request.masterNodeTimeout(TimeValue.timeValueSeconds(randomFrom(10, 20, 30)));
         }
         assertAcked(followerClient().execute(ActivateAutoFollowPatternAction.INSTANCE, request).actionGet());
     }
@@ -783,7 +783,7 @@ private AutoFollowMetadata.AutoFollowPattern getAutoFollowPattern(final String n
         GetAutoFollowPatternAction.Request request = new GetAutoFollowPatternAction.Request();
         request.setName(name);
         if (randomBoolean()) {
-            request.masterNodeTimeout(randomFrom("10s", "20s", "30s"));
+            request.masterNodeTimeout(TimeValue.timeValueSeconds(randomFrom(10, 20, 30)));
         }
         GetAutoFollowPatternAction.Response response = followerClient().execute(GetAutoFollowPatternAction.INSTANCE, request).actionGet();
         assertTrue(response.getAutoFollowPatterns().containsKey(name));
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestFollowStatsAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestFollowStatsAction.java
index e7e4d34a82425..997698e3578e6 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestFollowStatsAction.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestFollowStatsAction.java
@@ -40,9 +40,7 @@ public String getName() {
     protected RestChannelConsumer prepareRequest(final RestRequest restRequest, final NodeClient client) {
         final FollowStatsAction.StatsRequest request = new FollowStatsAction.StatsRequest();
         request.setIndices(Strings.splitStringByCommaToArray(restRequest.param("index")));
-        if (restRequest.hasParam("timeout")) {
-            request.setTimeout(restRequest.param("timeout"));
-        }
+        request.setTimeout(restRequest.paramAsTime("timeout", request.getTimeout()));
         return channel -> client.execute(
             FollowStatsAction.INSTANCE,
             request,
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java
index 4f453a2ad66f4..e67372516688f 100644
--- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java
@@ -595,7 +595,7 @@ public static PutFollowAction.Request putFollow(String leaderIndex, String follo
         request.getParameters().setMaxReadRequestOperationCount(between(1, 10000));
         request.waitForActiveShards(waitForActiveShards);
         if (randomBoolean()) {
-            request.masterNodeTimeout(randomFrom("10s", "20s", "30s"));
+            request.masterNodeTimeout(TimeValue.timeValueSeconds(randomFrom(10, 20, 30)));
         }
         return request;
     }
@@ -606,7 +606,7 @@ public static ResumeFollowAction.Request resumeFollow(String followerIndex) {
         request.getParameters().setMaxRetryDelay(TimeValue.timeValueMillis(10));
         request.getParameters().setReadPollTimeout(TimeValue.timeValueMillis(10));
         if (randomBoolean()) {
-            request.masterNodeTimeout(randomFrom("10s", "20s", "30s"));
+            request.masterNodeTimeout(TimeValue.timeValueSeconds(randomFrom(10, 20, 30)));
         }
         return request;
     }
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrSingleNodeTestCase.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrSingleNodeTestCase.java
index da3f29fcef8d3..efcaee96c008a 100644
--- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrSingleNodeTestCase.java
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrSingleNodeTestCase.java
@@ -104,7 +104,7 @@ protected ResumeFollowAction.Request getResumeFollowRequest(String followerIndex
         request.getParameters().setMaxRetryDelay(TimeValue.timeValueMillis(1));
         request.getParameters().setReadPollTimeout(TimeValue.timeValueMillis(1));
         if (randomBoolean()) {
-            request.masterNodeTimeout(randomFrom("10s", "20s", "30s"));
+            request.masterNodeTimeout(TimeValue.timeValueSeconds(randomFrom(10, 20, 30)));
         }
         return request;
     }
@@ -118,7 +118,7 @@ protected PutFollowAction.Request getPutFollowRequest(String leaderIndex, String
         request.getParameters().setReadPollTimeout(TimeValue.timeValueMillis(1));
         request.waitForActiveShards(ActiveShardCount.ONE);
         if (randomBoolean()) {
-            request.masterNodeTimeout(randomFrom("10s", "20s", "30s"));
+            request.masterNodeTimeout(TimeValue.timeValueSeconds(randomFrom(10, 20, 30)));
         }
         return request;
     }
diff --git a/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshotIT.java b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshotIT.java
index 9d3821d64626f..c2b5080aa16c1 100644
--- a/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshotIT.java
+++ b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshotIT.java
@@ -277,7 +277,7 @@ private void assertHits(String index, int numDocsExpected, boolean sourceHadDele
             assertEquals(numDocsExpected, searchResponse.getHits().getTotalHits().value);
         });
         SearchResponse searchResponse = prepareSearch(index).addSort(SeqNoFieldMapper.NAME, SortOrder.ASC)
-            .setScroll("1m")
+            .setScroll(TimeValue.timeValueMinutes(1))
             .slice(new SliceBuilder(SeqNoFieldMapper.NAME, randomIntBetween(0, 1), 2))
             .setSize(randomIntBetween(1, 10))
             .get();
@@ -349,11 +349,11 @@ private IndexRequestBuilder[] snapshotAndRestore(final String sourceIdx, final b
         logger.info("--> delete index and stop the data node");
         assertAcked(client().admin().indices().prepareDelete(sourceIdx).get());
         internalCluster().stopRandomDataNode();
-        assertFalse(clusterAdmin().prepareHealth().setTimeout("30s").setWaitForNodes("1").get().isTimedOut());
+        assertFalse(clusterAdmin().prepareHealth().setTimeout(TimeValue.timeValueSeconds(30)).setWaitForNodes("1").get().isTimedOut());
 
         final String newDataNode = internalCluster().startDataOnlyNode();
         logger.info("--> start a new data node " + newDataNode);
-        assertFalse(clusterAdmin().prepareHealth().setTimeout("30s").setWaitForNodes("2").get().isTimedOut());
+        assertFalse(clusterAdmin().prepareHealth().setTimeout(TimeValue.timeValueSeconds(30)).setWaitForNodes("2").get().isTimedOut());
 
         logger.info("--> restore the index and ensure all shards are allocated");
         RestoreSnapshotResponse restoreResponse = clusterAdmin().prepareRestoreSnapshot(repo, snapshot)
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/graph/GraphExploreRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/graph/GraphExploreRequest.java
index 2603002a2348b..f2e045b08328b 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/graph/GraphExploreRequest.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/graph/GraphExploreRequest.java
@@ -170,11 +170,6 @@ public GraphExploreRequest timeout(TimeValue timeout) {
         return this;
     }
 
-    public GraphExploreRequest timeout(String timeout) {
-        timeout(TimeValue.parseTimeValue(timeout, null, getClass().getSimpleName() + ".timeout"));
-        return this;
-    }
-
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/CcrStatsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/CcrStatsAction.java
index a9954005b4486..771071c6e1029 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/CcrStatsAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/CcrStatsAction.java
@@ -68,10 +68,6 @@ public void setTimeout(TimeValue timeout) {
             this.timeout = timeout;
         }
 
-        public void setTimeout(String timeout) {
-            this.timeout = TimeValue.parseTimeValue(timeout, null, getClass().getSimpleName() + ".timeout");
-        }
-
         @Override
         public boolean equals(Object o) {
             if (this == o) {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/graph/action/GraphExploreRequestBuilder.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/graph/action/GraphExploreRequestBuilder.java
index b8322d4fe0779..6b0be9ba33919 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/graph/action/GraphExploreRequestBuilder.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/graph/action/GraphExploreRequestBuilder.java
@@ -96,15 +96,6 @@ public GraphExploreRequestBuilder setTimeout(TimeValue timeout) {
         return this;
     }
 
-    /**
-     * An optional timeout to control how long the graph exploration is allowed
-     * to take.
-     */
-    public GraphExploreRequestBuilder setTimeout(String timeout) {
-        request.timeout(timeout);
-        return this;
-    }
-
     /**
      * Add a stage in the graph exploration. Each hop represents a stage of
      * querying elasticsearch to identify terms which can then be connnected
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ccr/action/CcrStatsActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ccr/action/CcrStatsActionTests.java
index e40954b361ef6..fab9f8de5339f 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ccr/action/CcrStatsActionTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ccr/action/CcrStatsActionTests.java
@@ -9,6 +9,7 @@
 
 import org.elasticsearch.TransportVersions;
 import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
 import org.elasticsearch.test.TransportVersionUtils;
 
@@ -24,8 +25,8 @@ protected Writeable.Reader<CcrStatsAction.Request> instanceReader() {
     @Override
     protected CcrStatsAction.Request createTestInstance() {
         var request = new CcrStatsAction.Request();
-        request.setTimeout(randomFrom("1s", "5s", "10s", "15s"));
-        request.masterNodeTimeout(randomFrom("1s", "5s", "10s", "15s"));
+        request.setTimeout(TimeValue.timeValueSeconds(randomFrom(1, 5, 10, 15)));
+        request.masterNodeTimeout(TimeValue.timeValueSeconds(randomFrom(1, 5, 10, 15)));
         return request;
     }
 
@@ -35,12 +36,12 @@ protected CcrStatsAction.Request mutateInstance(CcrStatsAction.Request instance)
             case 0 -> {
                 var mutatedInstance = new CcrStatsAction.Request();
                 mutatedInstance.setTimeout(instance.getTimeout());
-                mutatedInstance.masterNodeTimeout(randomFrom("20s", "25s", "30s"));
+                mutatedInstance.masterNodeTimeout(TimeValue.timeValueSeconds(randomFrom(20, 25, 30)));
                 yield mutatedInstance;
             }
             case 1 -> {
                 var mutatedInstance = new CcrStatsAction.Request();
-                mutatedInstance.setTimeout(randomFrom("20s", "25s", "30s"));
+                mutatedInstance.setTimeout(TimeValue.timeValueSeconds(randomFrom(20, 25, 30)));
                 mutatedInstance.masterNodeTimeout(instance.masterNodeTimeout());
                 yield mutatedInstance;
             }
@@ -52,7 +53,7 @@ public void testSerializationBwc() throws IOException {
         // In previous version `timeout` is not set
         var request = new CcrStatsAction.Request();
         if (randomBoolean()) {
-            request.masterNodeTimeout(randomFrom("20s", "25s", "30s"));
+            request.masterNodeTimeout(TimeValue.timeValueSeconds(randomFrom(20, 25, 30)));
         }
         assertSerialization(request, TransportVersionUtils.getPreviousVersion(TransportVersions.CCR_STATS_API_TIMEOUT_PARAM));
         assertSerialization(request, TransportVersions.MINIMUM_CCS_VERSION);
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/RestExplainLifecycleAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/RestExplainLifecycleAction.java
index 3090368c981d4..beae3f4d18194 100644
--- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/RestExplainLifecycleAction.java
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/RestExplainLifecycleAction.java
@@ -40,11 +40,7 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient
         explainLifecycleRequest.indicesOptions(IndicesOptions.fromRequest(restRequest, IndicesOptions.strictExpandOpen()));
         explainLifecycleRequest.onlyManaged(restRequest.paramAsBoolean("only_managed", false));
         explainLifecycleRequest.onlyErrors(restRequest.paramAsBoolean("only_errors", false));
-        String masterNodeTimeout = restRequest.param("master_timeout");
-        if (masterNodeTimeout != null) {
-            explainLifecycleRequest.masterNodeTimeout(masterNodeTimeout);
-        }
-
+        explainLifecycleRequest.masterNodeTimeout(restRequest.paramAsTime("master_timeout", explainLifecycleRequest.masterNodeTimeout()));
         return channel -> client.execute(ExplainLifecycleAction.INSTANCE, explainLifecycleRequest, new RestToXContentListener<>(channel));
     }
 }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/utils/persistence/BatchedDocumentsIterator.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/utils/persistence/BatchedDocumentsIterator.java
index c1e600aa66ba5..86488a647baa1 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/utils/persistence/BatchedDocumentsIterator.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/utils/persistence/BatchedDocumentsIterator.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.search.SearchScrollRequest;
 import org.elasticsearch.client.internal.OriginSettingClient;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
@@ -32,7 +33,7 @@
 public abstract class BatchedDocumentsIterator<T> implements BatchedIterator<T> {
     private static final Logger LOGGER = LogManager.getLogger(BatchedDocumentsIterator.class);
 
-    private static final String CONTEXT_ALIVE_DURATION = "5m";
+    private static final TimeValue CONTEXT_ALIVE_DURATION = TimeValue.timeValueMinutes(5);
     private static final int BATCH_SIZE = 10000;
 
     private final OriginSettingClient client;
diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/SecurityClearScrollTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/SecurityClearScrollTests.java
index 434e478545ffe..c656241b1280d 100644
--- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/SecurityClearScrollTests.java
+++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/SecurityClearScrollTests.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.action.search.MultiSearchResponse;
 import org.elasticsearch.action.search.SearchPhaseExecutionException;
 import org.elasticsearch.core.Strings;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.test.SecurityIntegTestCase;
 import org.elasticsearch.test.SecuritySettingsSourceField;
 import org.elasticsearch.xcontent.XContentType;
@@ -77,7 +78,7 @@ public void indexRandomDocuments() {
         MultiSearchRequestBuilder multiSearchRequestBuilder = client().prepareMultiSearch();
         int count = randomIntBetween(5, 15);
         for (int i = 0; i < count; i++) {
-            multiSearchRequestBuilder.add(prepareSearch("index").setScroll("10m").setSize(1));
+            multiSearchRequestBuilder.add(prepareSearch("index").setScroll(TimeValue.timeValueMinutes(10)).setSize(1));
         }
         scrollIds = new ArrayList<>();
         assertResponse(multiSearchRequestBuilder, multiSearchResponse -> scrollIds.addAll(getScrollIds(multiSearchResponse)));
diff --git a/x-pack/plugin/voting-only-node/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/votingonly/VotingOnlyNodePluginTests.java b/x-pack/plugin/voting-only-node/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/votingonly/VotingOnlyNodePluginTests.java
index 983ca4e741d83..2876fc50e036f 100644
--- a/x-pack/plugin/voting-only-node/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/votingonly/VotingOnlyNodePluginTests.java
+++ b/x-pack/plugin/voting-only-node/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/votingonly/VotingOnlyNodePluginTests.java
@@ -18,6 +18,7 @@
 import org.elasticsearch.common.blobstore.BlobStore;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.discovery.MasterNotDiscoveredException;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.IndexVersion;
@@ -153,7 +154,12 @@ public void testVotingOnlyNodesCannotBeMasterWithoutFullMasterNodes() throws Exc
         expectThrows(
             MasterNotDiscoveredException.class,
             () -> assertThat(
-                clusterAdmin().prepareState().setMasterNodeTimeout("100ms").get().getState().nodes().getMasterNodeId(),
+                clusterAdmin().prepareState()
+                    .setMasterNodeTimeout(TimeValue.timeValueMillis(100))
+                    .get()
+                    .getState()
+                    .nodes()
+                    .getMasterNodeId(),
                 nullValue()
             )
         );

From 9482673fbe2ac5302e272b29f21fbd34fc2497d0 Mon Sep 17 00:00:00 2001
From: Bogdan Pintea <bogdan.pintea@elastic.co>
Date: Thu, 25 Apr 2024 13:57:45 +0200
Subject: [PATCH 19/58] Docs: move base64 functions under string functions
 (#107866)

This moves the TO_BASE64 and FROM_BASE64 from the type conversion
functions under string functions (they take a string as input and output
another string).
---
 docs/reference/esql/functions/string-functions.asciidoc       | 4 ++++
 .../esql/functions/type-conversion-functions.asciidoc         | 4 ----
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/docs/reference/esql/functions/string-functions.asciidoc b/docs/reference/esql/functions/string-functions.asciidoc
index 353b7cd01ab6c..fbc61c79a6bc6 100644
--- a/docs/reference/esql/functions/string-functions.asciidoc
+++ b/docs/reference/esql/functions/string-functions.asciidoc
@@ -9,6 +9,7 @@
 
 // tag::string_list[]
 * <<esql-concat>>
+* <<esql-from_base64>>
 * <<esql-left>>
 * <<esql-length>>
 * <<esql-locate>>
@@ -18,12 +19,14 @@
 * <<esql-rtrim>>
 * <<esql-split>>
 * <<esql-substring>>
+* <<esql-to_base64>>
 * <<esql-to_lower>>
 * <<esql-to_upper>>
 * <<esql-trim>>
 // end::string_list[]
 
 include::layout/concat.asciidoc[]
+include::layout/from_base64.asciidoc[]
 include::layout/left.asciidoc[]
 include::layout/length.asciidoc[]
 include::layout/locate.asciidoc[]
@@ -33,6 +36,7 @@ include::layout/right.asciidoc[]
 include::layout/rtrim.asciidoc[]
 include::layout/split.asciidoc[]
 include::layout/substring.asciidoc[]
+include::layout/to_base64.asciidoc[]
 include::layout/to_lower.asciidoc[]
 include::layout/to_upper.asciidoc[]
 include::layout/trim.asciidoc[]
diff --git a/docs/reference/esql/functions/type-conversion-functions.asciidoc b/docs/reference/esql/functions/type-conversion-functions.asciidoc
index 207df3d271cf6..2fec7f40bde8b 100644
--- a/docs/reference/esql/functions/type-conversion-functions.asciidoc
+++ b/docs/reference/esql/functions/type-conversion-functions.asciidoc
@@ -8,8 +8,6 @@
 {esql} supports these type conversion functions:
 
 // tag::type_list[]
-* <<esql-from_base64>>
-* <<esql-to_base64>>
 * <<esql-to_boolean>>
 * <<esql-to_cartesianpoint>>
 * <<esql-to_cartesianshape>>
@@ -27,8 +25,6 @@
 * <<esql-to_version>>
 // end::type_list[]
 
-include::layout/from_base64.asciidoc[]
-include::layout/to_base64.asciidoc[]
 include::layout/to_boolean.asciidoc[]
 include::layout/to_cartesianpoint.asciidoc[]
 include::layout/to_cartesianshape.asciidoc[]

From 164fcf091b380379dcd6900eb32c07931d3ff19a Mon Sep 17 00:00:00 2001
From: shainaraskas <58563081+shainaraskas@users.noreply.github.com>
Date: Thu, 25 Apr 2024 08:26:40 -0400
Subject: [PATCH 20/58]  unhide setting (#107019)

---
 docs/reference/security/authorization/privileges.asciidoc | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/docs/reference/security/authorization/privileges.asciidoc b/docs/reference/security/authorization/privileges.asciidoc
index 5ff713a4b58fc..f85ff6bc92845 100644
--- a/docs/reference/security/authorization/privileges.asciidoc
+++ b/docs/reference/security/authorization/privileges.asciidoc
@@ -360,11 +360,9 @@ All {Ilm} operations relating to managing the execution of policies of an index
 or data stream. This includes operations such as retrying policies and removing
 a policy from an index or data stream.
 
-ifeval::["{release-state}"!="released"]
 `manage_data_stream_lifecycle`::
-All data stream lifecycle operations relating to reading and managing the built-in lifecycle of a data stream.
+All <<data-stream-lifecycle, Data stream lifecycle>> operations relating to reading and managing the built-in lifecycle of a data stream.
 This includes operations such as adding and removing a lifecycle from a data stream.
-endif::[]
 
 `manage_leader_index`::
 All actions that are required to manage the lifecycle of a leader index, which

From 3e6df2630e40f0083b4ac68bbd932de2ce7e272f Mon Sep 17 00:00:00 2001
From: Pat Whelan <pat.whelan@elastic.co>
Date: Thu, 25 Apr 2024 08:40:53 -0400
Subject: [PATCH 21/58] [Transform] Halt Indexer on Stop/Abort API (#107792)

When `_stop?wait_for_checkpoint=false` and
`_stop?force=true&wait_for_checkpoint=false` are called, there is a
small chance that the Transform Indexer thread will run if it is
scheduled before the stop API is called but before the threadpool runs
the executable. The `onStart` method now checks the state of the
indexer before executing. This will mitigate errors caused by reading
from Transform internal indices while the Task is stopped or deleted.

This does not impact when `wait_for_checkpoint=true`, because the
indexer state will remain `INDEXING` until the checkpoint is finished.

Relate #107266
---
 docs/changelog/107792.yaml                    |   5 +
 .../transforms/TransformIndexer.java          |  14 +-
 .../TransformIndexerStateTests.java           | 203 +++++++++++++++---
 3 files changed, 185 insertions(+), 37 deletions(-)
 create mode 100644 docs/changelog/107792.yaml

diff --git a/docs/changelog/107792.yaml b/docs/changelog/107792.yaml
new file mode 100644
index 0000000000000..bd9730d49d5d6
--- /dev/null
+++ b/docs/changelog/107792.yaml
@@ -0,0 +1,5 @@
+pr: 107792
+summary: Halt Indexer on Stop/Abort API
+area: Transform
+type: bug
+issues: []
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformIndexer.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformIndexer.java
index 636ed3cc02706..36d10653aae63 100644
--- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformIndexer.java
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformIndexer.java
@@ -268,11 +268,19 @@ protected void createCheckpoint(ActionListener<TransformCheckpoint> listener) {
     @Override
     protected void onStart(long now, ActionListener<Boolean> listener) {
         if (context.getTaskState() == TransformTaskState.FAILED) {
-            logger.debug("[{}] attempted to start while failed.", getJobId());
+            logger.debug("[{}] attempted to start while in state [{}].", getJobId(), TransformTaskState.FAILED.value());
             listener.onFailure(new ElasticsearchException("Attempted to start a failed transform [{}].", getJobId()));
             return;
         }
 
+        switch (getState()) {
+            case ABORTING, STOPPING, STOPPED -> {
+                logger.debug("[{}] attempted to start while in state [{}].", getJobId(), getState().value());
+                listener.onResponse(false);
+                return;
+            }
+        }
+
         if (context.getAuthState() != null && HealthStatus.RED.equals(context.getAuthState().getStatus())) {
             // AuthorizationState status is RED which means there was permission check error during PUT or _update.
             listener.onFailure(
@@ -543,7 +551,9 @@ private void executeRetentionPolicy(ActionListener<Void> listener) {
     private void finalizeCheckpoint(ActionListener<Void> listener) {
         try {
             // reset the page size, so we do not memorize a low page size forever
-            context.setPageSize(function.getInitialPageSize());
+            if (function != null) {
+                context.setPageSize(function.getInitialPageSize());
+            }
             // reset the changed bucket to free memory
             if (changeCollector != null) {
                 changeCollector.clear();
diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerStateTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerStateTests.java
index c4b35181ecf67..b9c4067da6b91 100644
--- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerStateTests.java
+++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerStateTests.java
@@ -109,7 +109,9 @@ class MockedTransformIndexer extends TransformIndexer {
 
         private TransformState persistedState;
         private AtomicInteger saveStateListenerCallCount = new AtomicInteger(0);
+        private SearchResponse searchResponse = ONE_HIT_SEARCH_RESPONSE;
         // used for synchronizing with the test
+        private CountDownLatch startLatch;
         private CountDownLatch searchLatch;
         private CountDownLatch doProcessLatch;
 
@@ -163,6 +165,10 @@ public CountDownLatch createCountDownOnResponseLatch(int count) {
             return doProcessLatch = new CountDownLatch(count);
         }
 
+        public CountDownLatch createAwaitForStartLatch(int count) {
+            return startLatch = new CountDownLatch(count);
+        }
+
         @Override
         void doGetInitialProgress(SearchRequest request, ActionListener<SearchResponse> responseListener) {
             responseListener.onResponse(ONE_HIT_SEARCH_RESPONSE);
@@ -188,14 +194,24 @@ void refreshDestinationIndex(ActionListener<Void> responseListener) {
 
         @Override
         protected void doNextSearch(long waitTimeInNanos, ActionListener<SearchResponse> nextPhase) {
-            if (searchLatch != null) {
+            maybeWaitOnLatch(searchLatch);
+            threadPool.generic().execute(() -> nextPhase.onResponse(searchResponse));
+        }
+
+        private static void maybeWaitOnLatch(CountDownLatch countDownLatch) {
+            if (countDownLatch != null) {
                 try {
-                    searchLatch.await();
+                    countDownLatch.await();
                 } catch (InterruptedException e) {
                     throw new IllegalStateException(e);
                 }
             }
-            threadPool.generic().execute(() -> nextPhase.onResponse(ONE_HIT_SEARCH_RESPONSE));
+        }
+
+        @Override
+        protected void onStart(long now, ActionListener<Boolean> listener) {
+            maybeWaitOnLatch(startLatch);
+            super.onStart(now, listener);
         }
 
         @Override
@@ -259,6 +275,10 @@ void persistState(TransformState state, ActionListener<Void> listener) {
         void validate(ActionListener<ValidateTransformAction.Response> listener) {
             listener.onResponse(null);
         }
+
+        void finishCheckpoint() {
+            searchResponse = null;
+        }
     }
 
     class MockedTransformIndexerForStatePersistenceTesting extends TransformIndexer {
@@ -371,22 +391,7 @@ public void tearDownClient() {
     }
 
     public void testTriggerStatePersistence() {
-        TransformConfig config = new TransformConfig(
-            randomAlphaOfLength(10),
-            randomSourceConfig(),
-            randomDestConfig(),
-            null,
-            new TimeSyncConfig("timestamp", TimeValue.timeValueSeconds(1)),
-            null,
-            randomPivotConfig(),
-            null,
-            randomBoolean() ? null : randomAlphaOfLengthBetween(1, 1000),
-            null,
-            null,
-            null,
-            null,
-            null
-        );
+        TransformConfig config = createTransformConfig();
         AtomicReference<IndexerState> state = new AtomicReference<>(IndexerState.INDEXING);
 
         TransformContext context = new TransformContext(TransformTaskState.STARTED, "", 0, mock(TransformContext.Listener.class));
@@ -452,22 +457,7 @@ public void testTriggerStatePersistence() {
     }
 
     public void testStopAtCheckpoint() throws Exception {
-        TransformConfig config = new TransformConfig(
-            randomAlphaOfLength(10),
-            randomSourceConfig(),
-            randomDestConfig(),
-            null,
-            new TimeSyncConfig("timestamp", TimeValue.timeValueSeconds(1)),
-            null,
-            randomPivotConfig(),
-            null,
-            randomBoolean() ? null : randomAlphaOfLengthBetween(1, 1000),
-            null,
-            null,
-            null,
-            null,
-            null
-        );
+        TransformConfig config = createTransformConfig();
 
         for (IndexerState state : IndexerState.values()) {
             // skip indexing case, tested below
@@ -684,6 +674,130 @@ public void testStopAtCheckpoint() throws Exception {
         }
     }
 
+    /**
+     * Given a started transform
+     * And the indexer thread has not started yet
+     * When a user calls _stop?force=false
+     * Then the indexer thread should exit early
+     */
+    public void testStopBeforeIndexingThreadStarts() throws Exception {
+        var indexer = createMockIndexer(
+            createTransformConfig(),
+            new AtomicReference<>(IndexerState.STARTED),
+            null,
+            threadPool,
+            auditor,
+            null,
+            new TransformIndexerStats(),
+            new TransformContext(TransformTaskState.STARTED, "", 0, mock(TransformContext.Listener.class))
+        );
+
+        // stop the indexer thread once it kicks off
+        var startLatch = indexer.createAwaitForStartLatch(1);
+        assertEquals(IndexerState.STARTED, indexer.start());
+        assertTrue(indexer.maybeTriggerAsyncJob(System.currentTimeMillis()));
+        assertEquals(IndexerState.INDEXING, indexer.getState());
+
+        // stop the indexer, equivalent to _stop?force=false
+        assertEquals(IndexerState.STOPPING, indexer.stop());
+        assertEquals(IndexerState.STOPPING, indexer.getState());
+
+        // now let the indexer thread run
+        startLatch.countDown();
+
+        assertBusy(() -> {
+            assertThat(indexer.getState(), equalTo(IndexerState.STOPPED));
+            assertThat(indexer.getLastCheckpoint().getCheckpoint(), equalTo(-1L));
+        });
+    }
+
+    /**
+     * Given a started transform
+     * And the indexer thread has not started yet
+     * When a user calls _stop?force=true
+     * Then the indexer thread should exit early
+     */
+    public void testForceStopBeforeIndexingThreadStarts() throws Exception {
+        var indexer = createMockIndexer(
+            createTransformConfig(),
+            new AtomicReference<>(IndexerState.STARTED),
+            null,
+            threadPool,
+            auditor,
+            null,
+            new TransformIndexerStats(),
+            new TransformContext(TransformTaskState.STARTED, "", 0, mock(TransformContext.Listener.class))
+        );
+
+        // stop the indexer thread once it kicks off
+        var startLatch = indexer.createAwaitForStartLatch(1);
+        assertEquals(IndexerState.STARTED, indexer.start());
+        assertTrue(indexer.maybeTriggerAsyncJob(System.currentTimeMillis()));
+        assertEquals(IndexerState.INDEXING, indexer.getState());
+
+        // stop the indexer, equivalent to _stop?force=true
+        assertFalse("Transform Indexer thread should still be running", indexer.abort());
+        assertEquals(IndexerState.ABORTING, indexer.getState());
+
+        // now let the indexer thread run
+        startLatch.countDown();
+
+        assertBusy(() -> {
+            assertThat(indexer.getState(), equalTo(IndexerState.ABORTING));
+            assertThat(indexer.getLastCheckpoint().getCheckpoint(), equalTo(-1L));
+        });
+    }
+
+    /**
+     * Given a started transform
+     * And the indexer thread has not started yet
+     * When a user calls _stop?wait_for_checkpoint=true
+     * Then the indexer thread should not exit early
+     */
+    public void testStopWaitForCheckpointBeforeIndexingThreadStarts() throws Exception {
+        var context = new TransformContext(TransformTaskState.STARTED, "", 0, mock(TransformContext.Listener.class));
+        var indexer = createMockIndexer(
+            createTransformConfig(),
+            new AtomicReference<>(IndexerState.STARTED),
+            null,
+            threadPool,
+            auditor,
+            null,
+            new TransformIndexerStats(),
+            context
+        );
+
+        // stop the indexer thread once it kicks off
+        var startLatch = indexer.createAwaitForStartLatch(1);
+        assertEquals(IndexerState.STARTED, indexer.start());
+        assertTrue(indexer.maybeTriggerAsyncJob(System.currentTimeMillis()));
+        assertEquals(IndexerState.INDEXING, indexer.getState());
+
+        // stop the indexer, equivalent to _stop?wait_for_checkpoint=true
+        context.setShouldStopAtCheckpoint(true);
+        CountDownLatch stopLatch = new CountDownLatch(1);
+        countResponse(listener -> setStopAtCheckpoint(indexer, true, listener), stopLatch);
+
+        // now let the indexer thread run
+        indexer.finishCheckpoint();
+        startLatch.countDown();
+
+        // wait for all listeners
+        assertTrue("timed out after 5s", stopLatch.await(5, TimeUnit.SECONDS));
+
+        // there should be no listeners waiting
+        assertEquals(0, indexer.getSaveStateListenerCount());
+
+        // listener must have been called by the indexing thread between timesStopAtCheckpointChanged and 6 times
+        // this is not exact, because we do not know _when_ the other thread persisted the flag
+        assertThat(indexer.getSaveStateListenerCallCount(), lessThanOrEqualTo(1));
+
+        assertBusy(() -> {
+            assertThat(indexer.getState(), equalTo(IndexerState.STOPPED));
+            assertThat(indexer.getLastCheckpoint().getCheckpoint(), equalTo(1L));
+        });
+    }
+
     @TestIssueLogging(
         value = "org.elasticsearch.xpack.transform.transforms:DEBUG",
         issueUrl = "https://github.com/elastic/elasticsearch/issues/92069"
@@ -868,4 +982,23 @@ private MockedTransformIndexerForStatePersistenceTesting createMockIndexerForSta
         indexer.initialize();
         return indexer;
     }
+
+    private static TransformConfig createTransformConfig() {
+        return new TransformConfig(
+            randomAlphaOfLength(10),
+            randomSourceConfig(),
+            randomDestConfig(),
+            null,
+            new TimeSyncConfig("timestamp", TimeValue.timeValueSeconds(1)),
+            null,
+            randomPivotConfig(),
+            null,
+            randomBoolean() ? null : randomAlphaOfLengthBetween(1, 1000),
+            null,
+            null,
+            null,
+            null,
+            null
+        );
+    }
 }

From cd508957cc8e7c5193e1f34e5971ac5cb4e3a935 Mon Sep 17 00:00:00 2001
From: David Kyle <david.kyle@elastic.co>
Date: Thu, 25 Apr 2024 14:29:56 +0100
Subject: [PATCH 22/58] Mute upgrade tests (#107898)

For #107887
---
 .../xpack/application/CohereServiceUpgradeIT.java          | 7 +++----
 .../xpack/application/HuggingFaceServiceUpgradeIT.java     | 7 +++----
 .../xpack/application/OpenAiServiceUpgradeIT.java          | 7 +++----
 3 files changed, 9 insertions(+), 12 deletions(-)

diff --git a/x-pack/plugin/inference/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/application/CohereServiceUpgradeIT.java b/x-pack/plugin/inference/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/application/CohereServiceUpgradeIT.java
index 9f009ef32f3aa..73676aa730883 100644
--- a/x-pack/plugin/inference/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/application/CohereServiceUpgradeIT.java
+++ b/x-pack/plugin/inference/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/application/CohereServiceUpgradeIT.java
@@ -15,8 +15,6 @@
 import org.elasticsearch.test.http.MockWebServer;
 import org.elasticsearch.xpack.inference.services.cohere.embeddings.CohereEmbeddingType;
 import org.hamcrest.Matchers;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
 
 import java.io.IOException;
 import java.util.List;
@@ -41,7 +39,7 @@ public CohereServiceUpgradeIT(@Name("upgradedNodes") int upgradedNodes) {
         super(upgradedNodes);
     }
 
-    @BeforeClass
+    // @BeforeClass
     public static void startWebServer() throws IOException {
         cohereEmbeddingsServer = new MockWebServer();
         cohereEmbeddingsServer.start();
@@ -50,13 +48,14 @@ public static void startWebServer() throws IOException {
         cohereRerankServer.start();
     }
 
-    @AfterClass
+    // @AfterClass // for the awaitsfix
     public static void shutdown() {
         cohereEmbeddingsServer.close();
         cohereRerankServer.close();
     }
 
     @SuppressWarnings("unchecked")
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/107887")
     public void testCohereEmbeddings() throws IOException {
         var embeddingsSupported = getOldClusterTestVersion().onOrAfter(COHERE_EMBEDDINGS_ADDED);
         assumeTrue("Cohere embedding service added in " + COHERE_EMBEDDINGS_ADDED, embeddingsSupported);
diff --git a/x-pack/plugin/inference/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/application/HuggingFaceServiceUpgradeIT.java b/x-pack/plugin/inference/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/application/HuggingFaceServiceUpgradeIT.java
index 28ecbc2847ad4..7e78b83223acf 100644
--- a/x-pack/plugin/inference/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/application/HuggingFaceServiceUpgradeIT.java
+++ b/x-pack/plugin/inference/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/application/HuggingFaceServiceUpgradeIT.java
@@ -13,8 +13,6 @@
 import org.elasticsearch.inference.TaskType;
 import org.elasticsearch.test.http.MockResponse;
 import org.elasticsearch.test.http.MockWebServer;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
 
 import java.io.IOException;
 import java.util.List;
@@ -36,7 +34,7 @@ public HuggingFaceServiceUpgradeIT(@Name("upgradedNodes") int upgradedNodes) {
         super(upgradedNodes);
     }
 
-    @BeforeClass
+    // @BeforeClass
     public static void startWebServer() throws IOException {
         embeddingsServer = new MockWebServer();
         embeddingsServer.start();
@@ -45,13 +43,14 @@ public static void startWebServer() throws IOException {
         elserServer.start();
     }
 
-    @AfterClass
+    // @AfterClass for the awaits fix
     public static void shutdown() {
         embeddingsServer.close();
         elserServer.close();
     }
 
     @SuppressWarnings("unchecked")
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/107887")
     public void testHFEmbeddings() throws IOException {
         var embeddingsSupported = getOldClusterTestVersion().onOrAfter(HF_EMBEDDINGS_ADDED);
         assumeTrue("Hugging Face embedding service added in " + HF_EMBEDDINGS_ADDED, embeddingsSupported);
diff --git a/x-pack/plugin/inference/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/application/OpenAiServiceUpgradeIT.java b/x-pack/plugin/inference/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/application/OpenAiServiceUpgradeIT.java
index 97d6176f18eed..ac0a71ebb2c82 100644
--- a/x-pack/plugin/inference/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/application/OpenAiServiceUpgradeIT.java
+++ b/x-pack/plugin/inference/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/application/OpenAiServiceUpgradeIT.java
@@ -12,8 +12,6 @@
 import org.elasticsearch.inference.TaskType;
 import org.elasticsearch.test.http.MockResponse;
 import org.elasticsearch.test.http.MockWebServer;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
 
 import java.io.IOException;
 import java.util.List;
@@ -37,7 +35,7 @@ public OpenAiServiceUpgradeIT(@Name("upgradedNodes") int upgradedNodes) {
         super(upgradedNodes);
     }
 
-    @BeforeClass
+    // @BeforeClass
     public static void startWebServer() throws IOException {
         openAiEmbeddingsServer = new MockWebServer();
         openAiEmbeddingsServer.start();
@@ -46,13 +44,14 @@ public static void startWebServer() throws IOException {
         openAiChatCompletionsServer.start();
     }
 
-    @AfterClass
+    // @AfterClass for the awaits fix
     public static void shutdown() {
         openAiEmbeddingsServer.close();
         openAiChatCompletionsServer.close();
     }
 
     @SuppressWarnings("unchecked")
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/107887")
     public void testOpenAiEmbeddings() throws IOException {
         var openAiEmbeddingsSupported = getOldClusterTestVersion().onOrAfter(OPEN_AI_EMBEDDINGS_ADDED);
         assumeTrue("OpenAI embedding service added in " + OPEN_AI_EMBEDDINGS_ADDED, openAiEmbeddingsSupported);

From 31f2fb85dfe0a32ac6f9d8c5a972317195a19806 Mon Sep 17 00:00:00 2001
From: Bogdan Pintea <bogdan.pintea@elastic.co>
Date: Thu, 25 Apr 2024 15:41:11 +0200
Subject: [PATCH 23/58] Docs: move STARTS/ENDS_WITH under string functions in
 the docs (#107867)

This moves the STARTS_WITH and ENDS_with under the strings functions
section (as they're not operators).
---
 docs/reference/esql/functions/operators.asciidoc        | 4 ----
 docs/reference/esql/functions/string-functions.asciidoc | 4 ++++
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/docs/reference/esql/functions/operators.asciidoc b/docs/reference/esql/functions/operators.asciidoc
index 96cdd5a3778be..55227570b7664 100644
--- a/docs/reference/esql/functions/operators.asciidoc
+++ b/docs/reference/esql/functions/operators.asciidoc
@@ -13,11 +13,9 @@ Boolean operators for comparing against one or multiple expressions.
 * <<esql-logical-operators>>
 * <<esql-predicates>>
 * <<esql-cidr_match>>
-* <<esql-ends_with>>
 * <<esql-in-operator>>
 * <<esql-like-operator>>
 * <<esql-rlike-operator>>
-* <<esql-starts_with>>
 // end::op_list[]
 
 include::binary.asciidoc[]
@@ -25,8 +23,6 @@ include::unary.asciidoc[]
 include::logical.asciidoc[]
 include::predicates.asciidoc[]
 include::cidr_match.asciidoc[]
-include::ends_with.asciidoc[]
 include::in.asciidoc[]
 include::like.asciidoc[]
 include::rlike.asciidoc[]
-include::starts_with.asciidoc[]
diff --git a/docs/reference/esql/functions/string-functions.asciidoc b/docs/reference/esql/functions/string-functions.asciidoc
index fbc61c79a6bc6..423af69dae67b 100644
--- a/docs/reference/esql/functions/string-functions.asciidoc
+++ b/docs/reference/esql/functions/string-functions.asciidoc
@@ -9,6 +9,7 @@
 
 // tag::string_list[]
 * <<esql-concat>>
+* <<esql-ends_with>>
 * <<esql-from_base64>>
 * <<esql-left>>
 * <<esql-length>>
@@ -18,6 +19,7 @@
 * <<esql-right>>
 * <<esql-rtrim>>
 * <<esql-split>>
+* <<esql-starts_with>>
 * <<esql-substring>>
 * <<esql-to_base64>>
 * <<esql-to_lower>>
@@ -26,6 +28,7 @@
 // end::string_list[]
 
 include::layout/concat.asciidoc[]
+include::ends_with.asciidoc[]
 include::layout/from_base64.asciidoc[]
 include::layout/left.asciidoc[]
 include::layout/length.asciidoc[]
@@ -35,6 +38,7 @@ include::layout/replace.asciidoc[]
 include::layout/right.asciidoc[]
 include::layout/rtrim.asciidoc[]
 include::layout/split.asciidoc[]
+include::starts_with.asciidoc[]
 include::layout/substring.asciidoc[]
 include::layout/to_base64.asciidoc[]
 include::layout/to_lower.asciidoc[]

From 76a1f3d0d65c740aa784dcc337c41b5b28b1313b Mon Sep 17 00:00:00 2001
From: David Turner <david.turner@elastic.co>
Date: Thu, 25 Apr 2024 14:57:19 +0100
Subject: [PATCH 24/58] Remove unused afterPrimariesBeforeReplicas overload
 (#107854)

Completes the change started in #103971.
---
 .../routing/allocation/ExistingShardsAllocator.java    | 10 +---------
 1 file changed, 1 insertion(+), 9 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/ExistingShardsAllocator.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/ExistingShardsAllocator.java
index 31b5e5a7cad41..421184abb8f99 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/ExistingShardsAllocator.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/ExistingShardsAllocator.java
@@ -44,15 +44,7 @@ public interface ExistingShardsAllocator {
      * Called during a round of allocation after attempting to allocate all the primaries but before any replicas, allowing the allocator
      * to prepare for replica allocation.
      */
-    @Deprecated(forRemoval = true)
-    default void afterPrimariesBeforeReplicas(@SuppressWarnings("unused") RoutingAllocation allocation) {
-        assert false : "must be overridden";
-        throw new UnsupportedOperationException();
-    }
-
-    default void afterPrimariesBeforeReplicas(RoutingAllocation allocation, Predicate<ShardRouting> isRelevantShardPredicate) {
-        afterPrimariesBeforeReplicas(allocation);
-    }
+    void afterPrimariesBeforeReplicas(RoutingAllocation allocation, Predicate<ShardRouting> isRelevantShardPredicate);
 
     /**
      * Allocate any unassigned shards in the given {@link RoutingAllocation} for which this {@link ExistingShardsAllocator} is responsible.

From 266807226221b94b81303b9cf9bc07d2f070f5af Mon Sep 17 00:00:00 2001
From: Benjamin Trent <ben.w.trent@gmail.com>
Date: Thu, 25 Apr 2024 09:57:54 -0400
Subject: [PATCH 25/58] Fix FunctionScoreQueryBuilderTests testToQuery
 (#107782) (#107869)

Fixes the test value generation for FunctionScoreQueryBuilderTests.

closes #107782
---
 .../functionscore/FunctionScoreQueryBuilderTests.java     | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/server/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreQueryBuilderTests.java
index c4f2b3e8596ef..dea1e301ed425 100644
--- a/server/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreQueryBuilderTests.java
@@ -253,7 +253,7 @@ private static DecayFunctionBuilder<?> createRandomDecayFunction() {
                     ZoneOffset.UTC
                 ).toString();
                 scale = between(1, 1000) + randomFrom("d", "h", "ms", "s", "m");
-                offset = randomPositiveTimeValue();
+                offset = between(1, 1000) + randomFrom("d", "h", "ms", "s", "m", "micros", "nanos");
             }
             default -> {
                 origin = randomBoolean() ? randomInt() : randomFloat();
@@ -282,12 +282,6 @@ protected void doAssertLuceneQuery(FunctionScoreQueryBuilder queryBuilder, Query
         }
     }
 
-    @Override
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/107782")
-    public void testToQuery() throws IOException {
-        super.testToQuery();
-    }
-
     public void testIllegalArguments() {
         expectThrows(IllegalArgumentException.class, () -> new FunctionScoreQueryBuilder((QueryBuilder) null));
         expectThrows(IllegalArgumentException.class, () -> new FunctionScoreQueryBuilder((ScoreFunctionBuilder<?>) null));

From 7af45cc52ec3492321c42f9cf1dc0efb5fb4d608 Mon Sep 17 00:00:00 2001
From: Bogdan Pintea <bogdan.pintea@elastic.co>
Date: Thu, 25 Apr 2024 16:10:59 +0200
Subject: [PATCH 26/58] ESQL: Document the cast operator (::) (#107871)

This documents the cast operator, `::`.
---
 docs/reference/esql/functions/cast.asciidoc     | 17 +++++++++++++++++
 .../reference/esql/functions/operators.asciidoc |  2 ++
 .../src/main/resources/convert.csv-spec         | 13 +++++++++++++
 3 files changed, 32 insertions(+)
 create mode 100644 docs/reference/esql/functions/cast.asciidoc

diff --git a/docs/reference/esql/functions/cast.asciidoc b/docs/reference/esql/functions/cast.asciidoc
new file mode 100644
index 0000000000000..a392f82c0d9b7
--- /dev/null
+++ b/docs/reference/esql/functions/cast.asciidoc
@@ -0,0 +1,17 @@
+[discete]
+[[esql-cast-operator]]
+==== `Cast (::)`
+
+// tag::body[]
+The `::` operator provides a convenient alternative syntax to the TO_<type>
+<<esql-type-conversion-functions,conversion functions>>.
+
+[source.merge.styled,esql]
+----
+include::{esql-specs}/convert.csv-spec[tag=docsCastOperator]
+----
+[%header.monospaced.styled,format=dsv,separator=|]
+|===
+include::{esql-specs}/convert.csv-spec[tag=docsCastOperator-result]
+|===
+// end::body[]
diff --git a/docs/reference/esql/functions/operators.asciidoc b/docs/reference/esql/functions/operators.asciidoc
index 55227570b7664..47f71aef1fa34 100644
--- a/docs/reference/esql/functions/operators.asciidoc
+++ b/docs/reference/esql/functions/operators.asciidoc
@@ -12,6 +12,7 @@ Boolean operators for comparing against one or multiple expressions.
 * <<esql-unary-operators>>
 * <<esql-logical-operators>>
 * <<esql-predicates>>
+* <<esql-cast-operator>>
 * <<esql-cidr_match>>
 * <<esql-in-operator>>
 * <<esql-like-operator>>
@@ -22,6 +23,7 @@ include::binary.asciidoc[]
 include::unary.asciidoc[]
 include::logical.asciidoc[]
 include::predicates.asciidoc[]
+include::cast.asciidoc[]
 include::cidr_match.asciidoc[]
 include::in.asciidoc[]
 include::like.asciidoc[]
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/convert.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/convert.csv-spec
index dd495f1f9bd12..43e683e165e29 100644
--- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/convert.csv-spec
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/convert.csv-spec
@@ -167,3 +167,16 @@ required_feature: esql.casting_operator
 10002          |true                 |false          |2.08
 10004          |true                 |false          |1.78
 ;
+
+docsCastOperator
+required_feature: esql.casting_operator
+//tag::docsCastOperator[]
+ROW ver = CONCAT(("0"::INT + 1)::STRING, ".2.3")::VERSION
+//end::docsCastOperator[]
+;
+
+//tag::docsCastOperator-result[]
+ver:version
+1.2.3
+//end::docsCastOperator-result[]
+;

From a91564c677d9a8468d0026452c23d4c62ef06892 Mon Sep 17 00:00:00 2001
From: Laurent Ploix <laurent.ploix@gmail.com>
Date: Thu, 25 Apr 2024 16:28:10 +0200
Subject: [PATCH 27/58] [build-tools] licensingDir should be relative to enable
 caching between builds in different directories (#107657)

* licensingDir should be relative for ml plugin
---------

Co-authored-by: Rene Groeschke <rene@breskeby.com>
---
 x-pack/plugin/ml/build.gradle | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/x-pack/plugin/ml/build.gradle b/x-pack/plugin/ml/build.gradle
index 74600a072ea0d..26f5ea053771c 100644
--- a/x-pack/plugin/ml/build.gradle
+++ b/x-pack/plugin/ml/build.gradle
@@ -134,6 +134,8 @@ tasks.register("extractNativeLicenses", Copy) {
 tasks.named('generateNotice').configure {
   dependsOn "extractNativeLicenses"
   inputs.dir("${project.buildDir}/extractedNativeLicenses/platform/licenses")
+    .withPropertyName('licensingDir')
+    .withPathSensitivity(PathSensitivity.RELATIVE)
   licenseDirs.add(tasks.named("extractNativeLicenses").map {new File(it.destinationDir, "platform/licenses") })
 }
 

From f1475b1e76de10b59902283a33592e4956ce4a2f Mon Sep 17 00:00:00 2001
From: Oleksandr Kolomiiets <oleksandr.kolomiiets@elastic.co>
Date: Thu, 25 Apr 2024 08:09:08 -0700
Subject: [PATCH 28/58] Increase wait time SearchIdleIT#testSearchIdleStats
 (#107881)

---
 .../java/org/elasticsearch/index/shard/SearchIdleIT.java        | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/shard/SearchIdleIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/shard/SearchIdleIT.java
index 199a397f52ad2..d9100e1e631db 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/index/shard/SearchIdleIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/index/shard/SearchIdleIT.java
@@ -224,7 +224,7 @@ public void testSearchIdleStats() throws InterruptedException {
             .get();
         waitUntil(
             () -> Arrays.stream(indicesAdmin().prepareStats(indexName).get().getShards()).allMatch(ShardStats::isSearchIdle),
-            searchIdleAfter,
+            searchIdleAfter + 1,
             TimeUnit.SECONDS
         );
 

From c03dfb1ddd3bd4409e23f08d44b473dcb7c44c08 Mon Sep 17 00:00:00 2001
From: David Kyle <david.kyle@elastic.co>
Date: Thu, 25 Apr 2024 16:18:51 +0100
Subject: [PATCH 29/58] finish muting (#107906)

Follow on from #107898 which left some tests unmuted
---
 .../elasticsearch/xpack/application/CohereServiceUpgradeIT.java  | 1 +
 .../xpack/application/HuggingFaceServiceUpgradeIT.java           | 1 +
 .../elasticsearch/xpack/application/OpenAiServiceUpgradeIT.java  | 1 +
 3 files changed, 3 insertions(+)

diff --git a/x-pack/plugin/inference/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/application/CohereServiceUpgradeIT.java b/x-pack/plugin/inference/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/application/CohereServiceUpgradeIT.java
index 73676aa730883..c73827dba2cbb 100644
--- a/x-pack/plugin/inference/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/application/CohereServiceUpgradeIT.java
+++ b/x-pack/plugin/inference/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/application/CohereServiceUpgradeIT.java
@@ -169,6 +169,7 @@ void assertEmbeddingInference(String inferenceId, CohereEmbeddingType type) thro
     }
 
     @SuppressWarnings("unchecked")
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/107887")
     public void testRerank() throws IOException {
         var rerankSupported = getOldClusterTestVersion().onOrAfter(COHERE_RERANK_ADDED);
         assumeTrue("Cohere rerank service added in " + COHERE_RERANK_ADDED, rerankSupported);
diff --git a/x-pack/plugin/inference/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/application/HuggingFaceServiceUpgradeIT.java b/x-pack/plugin/inference/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/application/HuggingFaceServiceUpgradeIT.java
index 7e78b83223acf..718678f97f37f 100644
--- a/x-pack/plugin/inference/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/application/HuggingFaceServiceUpgradeIT.java
+++ b/x-pack/plugin/inference/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/application/HuggingFaceServiceUpgradeIT.java
@@ -100,6 +100,7 @@ void assertEmbeddingInference(String inferenceId) throws IOException {
     }
 
     @SuppressWarnings("unchecked")
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/107887")
     public void testElser() throws IOException {
         var supported = getOldClusterTestVersion().onOrAfter(HF_ELSER_ADDED);
         assumeTrue("HF elser service added in " + HF_ELSER_ADDED, supported);
diff --git a/x-pack/plugin/inference/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/application/OpenAiServiceUpgradeIT.java b/x-pack/plugin/inference/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/application/OpenAiServiceUpgradeIT.java
index ac0a71ebb2c82..4e8e1c845b070 100644
--- a/x-pack/plugin/inference/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/application/OpenAiServiceUpgradeIT.java
+++ b/x-pack/plugin/inference/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/application/OpenAiServiceUpgradeIT.java
@@ -111,6 +111,7 @@ void assertEmbeddingInference(String inferenceId) throws IOException {
     }
 
     @SuppressWarnings("unchecked")
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/107887")
     public void testOpenAiCompletions() throws IOException {
         var openAiEmbeddingsSupported = getOldClusterTestVersion().onOrAfter(OPEN_AI_COMPLETIONS_ADDED);
         assumeTrue("OpenAI completions service added in " + OPEN_AI_COMPLETIONS_ADDED, openAiEmbeddingsSupported);

From 4e7a833418f73c830ac583c67bdb2674ca5b7e22 Mon Sep 17 00:00:00 2001
From: Mark Vieira <portugee@gmail.com>
Date: Thu, 25 Apr 2024 08:42:39 -0700
Subject: [PATCH 30/58] Reduce inference rolling upgrade test parallelism

---
 x-pack/plugin/inference/qa/rolling-upgrade/build.gradle | 1 +
 1 file changed, 1 insertion(+)

diff --git a/x-pack/plugin/inference/qa/rolling-upgrade/build.gradle b/x-pack/plugin/inference/qa/rolling-upgrade/build.gradle
index 328444dacaf53..381f46cdf22bd 100644
--- a/x-pack/plugin/inference/qa/rolling-upgrade/build.gradle
+++ b/x-pack/plugin/inference/qa/rolling-upgrade/build.gradle
@@ -28,6 +28,7 @@ BuildParams.bwcVersions.withWireCompatible(v -> v.after("8.11.0")) { bwcVersion,
   tasks.register(bwcTaskName(bwcVersion), StandaloneRestIntegTestTask) {
     usesBwcDistribution(bwcVersion)
     systemProperty("tests.old_cluster_version", bwcVersion)
+    maxParallelForks = 1
   }
 }
 

From a21242054b87e456f6e301dfa017e612b83ebbd6 Mon Sep 17 00:00:00 2001
From: Bogdan Pintea <bogdan.pintea@elastic.co>
Date: Thu, 25 Apr 2024 18:38:12 +0200
Subject: [PATCH 31/58] ESQL: Document BUCKET as a grouping function (#107864)

This adds the documentation for BUCKET as a grouping function and the
addition of the "direct" invocation mode providing a span (in addition
to the auto mode).
---
 .../esql/esql-functions-operators.asciidoc    |  7 ++
 docs/reference/esql/esql-get-started.asciidoc |  7 --
 .../functions/aggregation-functions.asciidoc  |  2 +-
 .../functions/date-time-functions.asciidoc    |  2 -
 .../esql/functions/examples/bucket.asciidoc   | 48 +++++++++-
 .../functions/grouping-functions.asciidoc     | 14 +++
 .../functions/kibana/definition/bucket.json   |  2 +
 .../src/main/resources/bucket.csv-spec        | 90 ++++++++++++-------
 .../expression/function/grouping/Bucket.java  | 18 +++-
 9 files changed, 145 insertions(+), 45 deletions(-)
 create mode 100644 docs/reference/esql/functions/grouping-functions.asciidoc

diff --git a/docs/reference/esql/esql-functions-operators.asciidoc b/docs/reference/esql/esql-functions-operators.asciidoc
index ddc077f3b8ff8..3ad61a8d56455 100644
--- a/docs/reference/esql/esql-functions-operators.asciidoc
+++ b/docs/reference/esql/esql-functions-operators.asciidoc
@@ -16,6 +16,12 @@ The reference documentation is divided into the following categories:
 include::functions/aggregation-functions.asciidoc[tag=agg_list]
 ====
 
+.*Grouping functions*
+[%collapsible]
+====
+include::functions/grouping-functions.asciidoc[tag=group_list]
+====
+
 .*Math functions*
 [%collapsible]
 ====
@@ -68,6 +74,7 @@ include::functions/operators.asciidoc[tag=op_list]
 ====
 
 include::functions/aggregation-functions.asciidoc[]
+include::functions/grouping-functions.asciidoc[]
 include::functions/math-functions.asciidoc[]
 include::functions/string-functions.asciidoc[]
 include::functions/date-time-functions.asciidoc[]
diff --git a/docs/reference/esql/esql-get-started.asciidoc b/docs/reference/esql/esql-get-started.asciidoc
index 0e23c0d97e61b..663b2f8ecd249 100644
--- a/docs/reference/esql/esql-get-started.asciidoc
+++ b/docs/reference/esql/esql-get-started.asciidoc
@@ -244,13 +244,6 @@ To track statistics over time, {esql} enables you to create histograms using the
 and returns a value for each row that corresponds to the resulting bucket the
 row falls into.
 
-For example, to create hourly buckets for the data on October 23rd:
-
-[source,esql]
-----
-include::{esql-specs}/bucket.csv-spec[tag=gs-bucket]
-----
-
 Combine `BUCKET` with <<esql-stats-by>> to create a histogram. For example,
 to count the number of events per hour:
 
diff --git a/docs/reference/esql/functions/aggregation-functions.asciidoc b/docs/reference/esql/functions/aggregation-functions.asciidoc
index 2fdc8582d6bfb..074fcce9ad43d 100644
--- a/docs/reference/esql/functions/aggregation-functions.asciidoc
+++ b/docs/reference/esql/functions/aggregation-functions.asciidoc
@@ -5,7 +5,7 @@
 <titleabbrev>Aggregate functions</titleabbrev>
 ++++
 
-The <<esql-stats-by>> function supports these aggregate functions:
+The <<esql-stats-by>> command supports these aggregate functions:
 
 // tag::agg_list[]
 * <<esql-agg-avg>>
diff --git a/docs/reference/esql/functions/date-time-functions.asciidoc b/docs/reference/esql/functions/date-time-functions.asciidoc
index 75aeea40bc608..8ce26eaabe381 100644
--- a/docs/reference/esql/functions/date-time-functions.asciidoc
+++ b/docs/reference/esql/functions/date-time-functions.asciidoc
@@ -8,7 +8,6 @@
 {esql} supports these date-time functions:
 
 // tag::date_list[]
-* <<esql-bucket>>
 * <<esql-date_diff>>
 * <<esql-date_extract>>
 * <<esql-date_format>>
@@ -17,7 +16,6 @@
 * <<esql-now>>
 // end::date_list[]
 
-include::layout/bucket.asciidoc[]
 include::layout/date_diff.asciidoc[]
 include::layout/date_extract.asciidoc[]
 include::layout/date_format.asciidoc[]
diff --git a/docs/reference/esql/functions/examples/bucket.asciidoc b/docs/reference/esql/functions/examples/bucket.asciidoc
index 0854840ffda34..f66f737b7d4b5 100644
--- a/docs/reference/esql/functions/examples/bucket.asciidoc
+++ b/docs/reference/esql/functions/examples/bucket.asciidoc
@@ -2,6 +2,10 @@
 
 *Examples*
 
+`BUCKET` can work in two modes: one in which the size of the bucket is computed
+based on a buckets count recommendation (four parameters) and a range, and
+another in which the bucket size is provided directly (two parameters).
+
 Using a target number of buckets, a start of a range, and an end of a range,
 `BUCKET` picks an appropriate bucket size to generate the target number of buckets or fewer.
 For example, asking for at most 20 buckets over a year results in monthly buckets:
@@ -17,7 +21,7 @@ include::{esql-specs}/bucket.csv-spec[tag=docsBucketMonth-result]
 The goal isn't to provide *exactly* the target number of buckets,
 it's to pick a range that people are comfortable with that provides at most the target number of buckets.
 
-Combine `BUCKET` with <<esql-stats-by>> to create a histogram:
+Combine `BUCKET` with an <<esql-agg-functions,aggregation>> to create a histogram:
 [source.merge.styled,esql]
 ----
 include::{esql-specs}/bucket.csv-spec[tag=docsBucketMonthlyHistogram]
@@ -28,7 +32,7 @@ include::{esql-specs}/bucket.csv-spec[tag=docsBucketMonthlyHistogram-result]
 |===
 
 NOTE: `BUCKET` does not create buckets that don't match any documents.
-+ "That's why this example is missing `1985-03-01` and other dates.
+That's why this example is missing `1985-03-01` and other dates.
 
 Asking for more buckets can result in a smaller range.
 For example, asking for at most 100 buckets in a year results in weekly buckets:
@@ -45,6 +49,20 @@ NOTE: `BUCKET` does not filter any rows. It only uses the provided range to pick
 For rows with a value outside of the range, it returns a bucket value that corresponds to a bucket outside the range.
 Combine`BUCKET` with <<esql-where>> to filter rows.
 
+If the desired bucket size is known in advance, simply provide it as the second
+argument, leaving the range out:
+[source.merge.styled,esql]
+----
+include::{esql-specs}/bucket.csv-spec[tag=docsBucketWeeklyHistogramWithSpan]
+----
+[%header.monospaced.styled,format=dsv,separator=|]
+|===
+include::{esql-specs}/bucket.csv-spec[tag=docsBucketWeeklyHistogramWithSpan-result]
+|===
+
+NOTE: When providing the bucket size as the second parameter, it must be a time
+duration or date period.
+
 `BUCKET` can also operate on numeric fields. For example, to create a salary histogram:
 [source.merge.styled,esql]
 ----
@@ -58,6 +76,20 @@ include::{esql-specs}/bucket.csv-spec[tag=docsBucketNumeric-result]
 Unlike the earlier example that intentionally filters on a date range, you rarely want to filter on a numeric range.
 You have to find the `min` and `max` separately. {esql} doesn't yet have an easy way to do that automatically.
 
+The range can be omitted if the desired bucket size is known in advance. Simply
+provide it as the second argument:
+[source.merge.styled,esql]
+----
+include::{esql-specs}/bucket.csv-spec[tag=docsBucketNumericWithSpan]
+----
+[%header.monospaced.styled,format=dsv,separator=|]
+|===
+include::{esql-specs}/bucket.csv-spec[tag=docsBucketNumericWithSpan-result]
+|===
+
+NOTE: When providing the bucket size as the second parameter, it must be
+of a floating point type.
+
 Create hourly buckets for the last 24 hours, and calculate the number of events per hour:
 [source.merge.styled,esql]
 ----
@@ -77,3 +109,15 @@ include::{esql-specs}/bucket.csv-spec[tag=bucket_in_agg]
 include::{esql-specs}/bucket.csv-spec[tag=bucket_in_agg-result]
 |===
 
+`BUCKET` may be used in both the aggregating and grouping part of the
+<<esql-stats-by, STATS ... BY ...>> command provided that in the aggregating
+part the function is referenced by an alias defined in the
+grouping part, or that it is invoked with the exact same expression:
+[source.merge.styled,esql]
+----
+include::{esql-specs}/bucket.csv-spec[tag=reuseGroupingFunctionWithExpression]
+----
+[%header.monospaced.styled,format=dsv,separator=|]
+|===
+include::{esql-specs}/bucket.csv-spec[tag=reuseGroupingFunctionWithExpression-result]
+|===
diff --git a/docs/reference/esql/functions/grouping-functions.asciidoc b/docs/reference/esql/functions/grouping-functions.asciidoc
new file mode 100644
index 0000000000000..ed0caf5ec2a4c
--- /dev/null
+++ b/docs/reference/esql/functions/grouping-functions.asciidoc
@@ -0,0 +1,14 @@
+[[esql-group-functions]]
+==== {esql} grouping functions
+
+++++
+<titleabbrev>Grouping functions</titleabbrev>
+++++
+
+The <<esql-stats-by>> command supports these grouping functions:
+
+// tag::group_list[]
+* <<esql-bucket>>
+// end::group_list[]
+
+include::layout/bucket.asciidoc[]
diff --git a/docs/reference/esql/functions/kibana/definition/bucket.json b/docs/reference/esql/functions/kibana/definition/bucket.json
index bab6cef538b07..986c0e8f91d33 100644
--- a/docs/reference/esql/functions/kibana/definition/bucket.json
+++ b/docs/reference/esql/functions/kibana/definition/bucket.json
@@ -939,7 +939,9 @@
     "FROM employees\n| WHERE hire_date >= \"1985-01-01T00:00:00Z\" AND hire_date < \"1986-01-01T00:00:00Z\"\n| STATS hire_date = MV_SORT(VALUES(hire_date)) BY month = BUCKET(hire_date, 20, \"1985-01-01T00:00:00Z\", \"1986-01-01T00:00:00Z\")\n| SORT hire_date",
     "FROM employees\n| WHERE hire_date >= \"1985-01-01T00:00:00Z\" AND hire_date < \"1986-01-01T00:00:00Z\"\n| STATS hires_per_month = COUNT(*) BY month = BUCKET(hire_date, 20, \"1985-01-01T00:00:00Z\", \"1986-01-01T00:00:00Z\")\n| SORT month",
     "FROM employees\n| WHERE hire_date >= \"1985-01-01T00:00:00Z\" AND hire_date < \"1986-01-01T00:00:00Z\"\n| STATS hires_per_week = COUNT(*) BY week = BUCKET(hire_date, 100, \"1985-01-01T00:00:00Z\", \"1986-01-01T00:00:00Z\")\n| SORT week",
+    "FROM employees\n| WHERE hire_date >= \"1985-01-01T00:00:00Z\" AND hire_date < \"1986-01-01T00:00:00Z\"\n| STATS hires_per_week = COUNT(*) BY week = BUCKET(hire_date, 1 week)\n| SORT week",
     "FROM employees\n| STATS COUNT(*) by bs = BUCKET(salary, 20, 25324, 74999)\n| SORT bs",
+    "FROM employees\n| WHERE hire_date >= \"1985-01-01T00:00:00Z\" AND hire_date < \"1986-01-01T00:00:00Z\"\n| STATS c = COUNT(1) BY b = BUCKET(salary, 5000.)\n| SORT b",
     "FROM sample_data \n| WHERE @timestamp >= NOW() - 1 day and @timestamp < NOW()\n| STATS COUNT(*) BY bucket = BUCKET(@timestamp, 25, NOW() - 1 day, NOW())",
     "FROM employees\n| WHERE hire_date >= \"1985-01-01T00:00:00Z\" AND hire_date < \"1986-01-01T00:00:00Z\"\n| STATS AVG(salary) BY bucket = BUCKET(hire_date, 20, \"1985-01-01T00:00:00Z\", \"1986-01-01T00:00:00Z\")\n| SORT bucket"
   ]
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/bucket.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/bucket.csv-spec
index ad8f16f89cf7e..f41bf3f020eb5 100644
--- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/bucket.csv-spec
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/bucket.csv-spec
@@ -175,13 +175,13 @@ FROM employees
 ;
 
 //tag::docsBucketMonthlyHistogram-result[]
-  hires_per_month:long    |    month:date
-2              |1985-02-01T00:00:00.000Z
-1              |1985-05-01T00:00:00.000Z
-1              |1985-07-01T00:00:00.000Z
-1              |1985-09-01T00:00:00.000Z
-2              |1985-10-01T00:00:00.000Z
-4              |1985-11-01T00:00:00.000Z
+  hires_per_month:long |    month:date
+2                      |1985-02-01T00:00:00.000Z
+1                      |1985-05-01T00:00:00.000Z
+1                      |1985-07-01T00:00:00.000Z
+1                      |1985-09-01T00:00:00.000Z
+2                      |1985-10-01T00:00:00.000Z
+4                      |1985-11-01T00:00:00.000Z
 //end::docsBucketMonthlyHistogram-result[]
 ;
 
@@ -196,15 +196,36 @@ FROM employees
 
 //tag::docsBucketWeeklyHistogram-result[]
   hires_per_week:long    |    week:date
-2              |1985-02-18T00:00:00.000Z
-1              |1985-05-13T00:00:00.000Z
-1              |1985-07-08T00:00:00.000Z
-1              |1985-09-16T00:00:00.000Z
-2              |1985-10-14T00:00:00.000Z
-4              |1985-11-18T00:00:00.000Z
+2                        |1985-02-18T00:00:00.000Z
+1                        |1985-05-13T00:00:00.000Z
+1                        |1985-07-08T00:00:00.000Z
+1                        |1985-09-16T00:00:00.000Z
+2                        |1985-10-14T00:00:00.000Z
+4                        |1985-11-18T00:00:00.000Z
 //end::docsBucketWeeklyHistogram-result[]
 ;
 
+// bucketing in span mode (identical results to above)
+docsBucketWeeklyHistogramWithSpan#[skip:-8.13.99, reason:BUCKET renamed in 8.14]
+//tag::docsBucketWeeklyHistogramWithSpan[]
+FROM employees
+| WHERE hire_date >= "1985-01-01T00:00:00Z" AND hire_date < "1986-01-01T00:00:00Z"
+| STATS hires_per_week = COUNT(*) BY week = BUCKET(hire_date, 1 week)
+| SORT week
+//end::docsBucketWeeklyHistogramWithSpan[]
+;
+
+//tag::docsBucketWeeklyHistogramWithSpan-result[]
+  hires_per_week:long    |    week:date
+2                        |1985-02-18T00:00:00.000Z
+1                        |1985-05-13T00:00:00.000Z
+1                        |1985-07-08T00:00:00.000Z
+1                        |1985-09-16T00:00:00.000Z
+2                        |1985-10-14T00:00:00.000Z
+4                        |1985-11-18T00:00:00.000Z
+//end::docsBucketWeeklyHistogramWithSpan-result[]
+;
+
 docsBucketLast24hr#[skip:-8.13.99, reason:BUCKET renamed in 8.14]
 //tag::docsBucketLast24hr[]
 FROM sample_data 
@@ -218,17 +239,6 @@ FROM sample_data
 //end::docsBucketLast24hr-result[]
 ;
 
-docsGettingStartedBucket#[skip:-8.13.99, reason:BUCKET renamed in 8.14]
-// tag::gs-bucket[]
-FROM sample_data
-| STATS BY bucket = BUCKET(@timestamp, 24, "2023-10-23T00:00:00Z", NOW())
-// end::gs-bucket[]
-| LIMIT 0
-;
-
-bucket:date
-;
-
 docsGettingStartedBucketStatsBy#[skip:-8.13.99, reason:BUCKET renamed in 8.14]
 // tag::gs-bucket-stats-by[]
 FROM sample_data
@@ -352,12 +362,15 @@ FROM employees
 
 // bucketing in span mode (identical results to above)
 bucketNumericWithSpan#[skip:-8.13.99, reason:BUCKET extended in 8.14]
+//tag::docsBucketNumericWithSpan[]
 FROM employees
 | WHERE hire_date >= "1985-01-01T00:00:00Z" AND hire_date < "1986-01-01T00:00:00Z"
 | STATS c = COUNT(1) BY b = BUCKET(salary, 5000.)
 | SORT b
+//end::docsBucketNumericWithSpan[]
 ;
 
+//tag::docsBucketNumericWithSpan-result[]
        c:long  |       b:double
 1              |25000.0
 1              |30000.0
@@ -368,6 +381,7 @@ FROM employees
 1              |60000.0
 1              |65000.0
 1              |70000.0
+//end::docsBucketNumericWithSpan-result[]
 ;
 
 bucketNumericMixedTypes#[skip:-8.13.99, reason:BUCKET extended in 8.14]
@@ -439,14 +453,28 @@ FROM employees
 ;
 
 reuseGroupingFunctionWithExpression#[skip:-8.13.99, reason:BUCKET renamed in 8.14]
+//tag::reuseGroupingFunctionWithExpression[]
 FROM employees
-| STATS sum = BUCKET(salary % 2 + 13, 1.) + 1 BY bucket = BUCKET(salary % 2 + 13, 1.)
-| SORT sum
-;
-
- sum:double    | bucket:double
-14.0           |13.0
-15.0           |14.0
+| STATS s1 = b1 + 1, s2 = BUCKET(salary / 1000 + 999, 50.) + 2 BY b1 = BUCKET(salary / 100 + 99, 50.), b2 = BUCKET(salary / 1000 + 999, 50.)
+| SORT b1, b2
+| KEEP s1, b1, s2, b2
+//end::reuseGroupingFunctionWithExpression[]
+;
+
+//tag::reuseGroupingFunctionWithExpression-result[]
+ s1:double | b1:double | s2:double   | b2:double
+351.0      |350.0      |1002.0       |1000.0
+401.0      |400.0      |1002.0       |1000.0
+451.0      |450.0      |1002.0       |1000.0
+501.0      |500.0      |1002.0       |1000.0
+551.0      |550.0      |1002.0       |1000.0
+601.0      |600.0      |1002.0       |1000.0
+601.0      |600.0      |1052.0       |1050.0
+651.0      |650.0      |1052.0       |1050.0
+701.0      |700.0      |1052.0       |1050.0
+751.0      |750.0      |1052.0       |1050.0
+801.0      |800.0      |1052.0       |1050.0
+//end::reuseGroupingFunctionWithExpression-result[]
 ;
 
 reuseGroupingFunctionWithinAggs#[skip:-8.13.99, reason:BUCKET renamed in 8.14]
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/grouping/Bucket.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/grouping/Bucket.java
index f83c10e4ce1f6..218d469d626f9 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/grouping/Bucket.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/grouping/Bucket.java
@@ -92,6 +92,10 @@ public class Bucket extends GroupingFunction implements Validatable, TwoOptional
         examples = {
             @Example(
                 description = """
+                    `BUCKET` can work in two modes: one in which the size of the bucket is computed
+                    based on a buckets count recommendation (four parameters) and a range and
+                    another in which the bucket size is provided directly (two parameters).
+
                     Using a target number of buckets, a start of a range, and an end of a range,
                     `BUCKET` picks an appropriate bucket size to generate the target number of buckets or fewer.
                     For example, asking for at most 20 buckets over a year results in monthly buckets:""",
@@ -102,12 +106,12 @@ public class Bucket extends GroupingFunction implements Validatable, TwoOptional
                     it's to pick a range that people are comfortable with that provides at most the target number of buckets."""
             ),
             @Example(
-                description = "Combine `BUCKET` with <<esql-stats-by>> to create a histogram:",
+                description = "Combine `BUCKET` with an <<esql-agg-functions,aggregation>> to create a histogram:",
                 file = "bucket",
                 tag = "docsBucketMonthlyHistogram",
                 explanation = """
                     NOTE: `BUCKET` does not create buckets that don't match any documents.
-                    + "That's why this example is missing `1985-03-01` and other dates."""
+                    That's why this example is missing `1985-03-01` and other dates."""
             ),
             @Example(
                 description = """
@@ -120,6 +124,11 @@ public class Bucket extends GroupingFunction implements Validatable, TwoOptional
                     For rows with a value outside of the range, it returns a bucket value that corresponds to a bucket outside the range.
                     Combine`BUCKET` with <<esql-where>> to filter rows."""
             ),
+            @Example(description = """
+                If the desired bucket size is known in advance, simply provide it as the second
+                argument, leaving the range out:""", file = "bucket", tag = "docsBucketWeeklyHistogramWithSpan", explanation = """
+                NOTE: When providing the bucket size as the second parameter, its type must be
+                of a time duration or date period type."""),
             @Example(
                 description = "`BUCKET` can also operate on numeric fields. For example, to create a salary histogram:",
                 file = "bucket",
@@ -128,6 +137,11 @@ public class Bucket extends GroupingFunction implements Validatable, TwoOptional
                     Unlike the earlier example that intentionally filters on a date range, you rarely want to filter on a numeric range.
                     You have to find the `min` and `max` separately. {esql} doesn't yet have an easy way to do that automatically."""
             ),
+            @Example(description = """
+                If the desired bucket size is known in advance, simply provide it as the second
+                argument, leaving the range out:""", file = "bucket", tag = "docsBucketNumericWithSpan", explanation = """
+                NOTE: When providing the bucket size as the second parameter, its type must be
+                of a floating type."""),
             @Example(
                 description = "Create hourly buckets for the last 24 hours, and calculate the number of events per hour:",
                 file = "bucket",

From cfc9e44b63d3a24b7f7b49579547ffe0951f624e Mon Sep 17 00:00:00 2001
From: Simon Cooper <simon.cooper@elastic.co>
Date: Thu, 25 Apr 2024 17:53:13 +0100
Subject: [PATCH 32/58] Reduce parallelism for yaml and java esql rest tests
 (#107890)

This fixes #107879

Reduce parallelism for java rest tests for esql
---
 x-pack/plugin/esql/qa/server/mixed-cluster/build.gradle | 1 +
 1 file changed, 1 insertion(+)

diff --git a/x-pack/plugin/esql/qa/server/mixed-cluster/build.gradle b/x-pack/plugin/esql/qa/server/mixed-cluster/build.gradle
index c25ef858534e0..5515ef0728a72 100644
--- a/x-pack/plugin/esql/qa/server/mixed-cluster/build.gradle
+++ b/x-pack/plugin/esql/qa/server/mixed-cluster/build.gradle
@@ -40,6 +40,7 @@ BuildParams.bwcVersions.withWireCompatible(supportedVersion) { bwcVersion, baseN
     usesBwcDistribution(bwcVersion)
     systemProperty("tests.old_cluster_version", bwcVersion)
     systemProperty("tests.version_parameter_unsupported", versionUnsupported(bwcVersion))
+    maxParallelForks = 1
   }
 
   def yamlRestTest = tasks.register("v${bwcVersion}#yamlRestTest", StandaloneRestIntegTestTask) {

From fdefe090418aed09243fb150548909e3e0c4ff0e Mon Sep 17 00:00:00 2001
From: Panagiotis Bailis <pmpailis@gmail.com>
Date: Thu, 25 Apr 2024 20:11:44 +0300
Subject: [PATCH 33/58] Fix for from parameter when using sub_searches and rank
 (#106253)

---
 docs/changelog/106253.yaml                    |    6 +
 docs/reference/search/retriever.asciidoc      |    4 +-
 docs/reference/search/rrf.asciidoc            |   86 +-
 .../action/search/SearchPhaseController.java  |    4 +
 .../action/search/SearchRequest.java          |    6 +-
 .../search/query/QueryPhase.java              |    2 +-
 .../search/rank/RankBuilder.java              |   22 +-
 .../QueryPhaseRankCoordinatorContext.java     |    8 +-
 .../context/QueryPhaseRankShardContext.java   |   14 +-
 .../action/search/SearchRequestTests.java     |   15 +-
 .../search/rank/TestRankBuilder.java          |    2 +-
 .../RRFQueryPhaseRankCoordinatorContext.java  |   12 +-
 .../rrf/RRFQueryPhaseRankShardContext.java    |   13 +-
 .../xpack/rank/rrf/RRFRankBuilder.java        |   11 +-
 .../xpack/rank/rrf/RRFRetrieverBuilder.java   |   14 +-
 .../xpack/rank/rrf/RRFRankBuilderTests.java   |    6 +-
 .../xpack/rank/rrf/RRFRankContextTests.java   |    4 +-
 .../rrf/RRFRetrieverBuilderParsingTests.java  |    2 +-
 .../rest-api-spec/test/rrf/100_rank_rrf.yml   |   44 +-
 .../test/rrf/150_rank_rrf_pagination.yml      | 1055 +++++++++++++++++
 .../test/rrf/200_rank_rrf_script.yml          |    6 +-
 .../test/rrf/300_rrf_retriever.yml            |    8 +-
 .../test/rrf/400_rrf_retriever_script.yml     |    6 +-
 23 files changed, 1271 insertions(+), 79 deletions(-)
 create mode 100644 docs/changelog/106253.yaml
 create mode 100644 x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/150_rank_rrf_pagination.yml

diff --git a/docs/changelog/106253.yaml b/docs/changelog/106253.yaml
new file mode 100644
index 0000000000000..b80cda37f63c7
--- /dev/null
+++ b/docs/changelog/106253.yaml
@@ -0,0 +1,6 @@
+pr: 106253
+summary: Fix for from parameter when using `sub_searches` and rank
+area: Ranking
+type: bug
+issues:
+ - 99011
diff --git a/docs/reference/search/retriever.asciidoc b/docs/reference/search/retriever.asciidoc
index 42d2129f0fdec..6301f439e9b5b 100644
--- a/docs/reference/search/retriever.asciidoc
+++ b/docs/reference/search/retriever.asciidoc
@@ -189,7 +189,7 @@ GET /index/_search
                 }
             ],
             "rank_constant": ...
-            "window_size": ...
+            "rank_window_size": ...
         }
     }
 }
@@ -202,7 +202,7 @@ The <<search-from-param, `from`>> and <<search-size-param, `size`>>
 parameters are provided globally as part of the general
 <<search-search, search API>>. They are applied to all retrievers in a
 retriever tree unless a specific retriever overrides the `size` parameter
-using a different parameter such as `window_size`. Though, the final
+using a different parameter such as `rank_window_size`. Though, the final
 search hits are always limited to `size`.
 
 ==== Using aggregations with a retriever tree
diff --git a/docs/reference/search/rrf.asciidoc b/docs/reference/search/rrf.asciidoc
index 96477cdee45f1..c541750fff789 100644
--- a/docs/reference/search/rrf.asciidoc
+++ b/docs/reference/search/rrf.asciidoc
@@ -74,7 +74,7 @@ GET example-index/_search
                     }
                 }
             ],
-            "window_size": 50,
+            "rank_window_size": 50,
             "rank_constant": 20
         }
     }
@@ -94,8 +94,8 @@ its global top 50 results.
 the query top documents and rank them based on the RRF formula using parameters from
 the `rrf` retriever to get the combined top documents using the default `size` of `10`.
 
-Note that if `k` from a knn search is larger than `window_size`, the results are
-truncated to `window_size`. If `k` is smaller than `window_size`, the results are
+Note that if `k` from a knn search is larger than `rank_window_size`, the results are
+truncated to `rank_window_size`. If `k` is smaller than `rank_window_size`, the results are
 `k` size.
 
 [[rrf-supported-features]]
@@ -160,7 +160,7 @@ GET example-index/_search
                     }
                 }
             ],
-            "window_size": 50,
+            "rank_window_size": 50,
             "rank_constant": 20
         }
     }
@@ -289,7 +289,7 @@ GET example-index/_search
                     }
                 }
             ],
-            "window_size": 5,
+            "rank_window_size": 5,
             "rank_constant": 1
         }
     },
@@ -510,8 +510,82 @@ _id: 5 =             1.0/(1+4) = 0.2000
 ----
 // NOTCONSOLE
 
-We rank the documents based on the RRF formula with a `window_size` of `5`
+We rank the documents based on the RRF formula with a `rank_window_size` of `5`
 truncating the bottom `2` docs in our RRF result set with a `size` of `3`.
 We end with `_id: 3` as `_rank: 1`, `_id: 2` as `_rank: 2`, and
 `_id: 4` as `_rank: 3`. This ranking matches the result set from the
 original RRF search as expected.
+
+
+==== Pagination in RRF
+
+When using `rrf` you can paginate through the results using the `from` parameter.
+As the final ranking is solely dependent on the original query ranks, to ensure
+consistency when paginating, we have to make sure that while `from` changes, the order
+of what we have already seen remains intact. To that end, we're using a fixed `rank_window_size`
+as the whole available result set upon which we can paginate.
+This essentially means that if:
+
+* `from + size` &le; `rank_window_size` : we could get  `results[from: from+size]` documents back from
+the final `rrf` ranked result set
+
+* `from + size` &gt; `rank_window_size` : we would get 0 results back, as the request would fall outside the
+available `rank_window_size`-sized result set.
+
+An important thing to note here is that since `rank_window_size` is all the results that we'll get to see
+from the individual query components, pagination guarantees consistency,  i.e. no documents are skipped
+or duplicated in multiple pages, iff `rank_window_size` remains the same. If `rank_window_size` changes, then the order
+of the results might change as well, even for the same ranks.
+
+To illustrate all of the above, let's consider the following simplified example where we have
+two queries, `queryA` and `queryB` and their ranked documents:
+[source,python]
+----
+     |  queryA   |  queryB    |
+_id: |  1        |  5         |
+_id: |  2        |  4         |
+_id: |  3        |  3         |
+_id: |  4        |  1         |
+_id: |           |  2         |
+----
+// NOTCONSOLE
+
+For `rank_window_size=5` we would get to see all documents from both `queryA` and `queryB`.
+Assuming a `rank_constant=1`, the `rrf` scores would be:
+[source,python]
+----
+# doc   | queryA     | queryB       | score
+_id: 1 =  1.0/(1+1)  + 1.0/(1+4)      = 0.7
+_id: 2 =  1.0/(1+2)  + 1.0/(1+5)      = 0.5
+_id: 3 =  1.0/(1+3)  + 1.0/(1+3)      = 0.5
+_id: 4 =  1.0/(1+4)  + 1.0/(1+2)      = 0.533
+_id: 5 =    0        + 1.0/(1+1)      = 0.5
+----
+// NOTCONSOLE
+
+So the final ranked result set would be [`1`, `4`, `2`, `3`, `5`] and we would paginate over that, since
+`rank_window_size == len(results)`. In this scenario, we would have:
+
+* `from=0, size=2` would return documents [`1`, `4`] with ranks `[1, 2]`
+* `from=2, size=2` would return documents [`2`, `3`] with ranks `[3, 4]`
+* `from=4, size=2` would return document [`5`] with rank `[5]`
+* `from=6, size=2` would return an empty result set as it there are no more results to iterate over
+
+Now, if we had a `rank_window_size=2`, we would only get to see `[1, 2]` and `[5, 4]` documents
+for queries `queryA` and `queryB` respectively. Working out the math, we would see that the results would now
+be slightly different, because we would have no knowledge of the documents in positions `[3: end]` for either query.
+[source,python]
+----
+# doc   | queryA     | queryB         | score
+_id: 1 =  1.0/(1+1)  + 0              = 0.5
+_id: 2 =  1.0/(1+2)  + 0              = 0.33
+_id: 4 =    0        + 1.0/(1+2)      = 0.33
+_id: 5 =    0        + 1.0/(1+1)      = 0.5
+----
+// NOTCONSOLE
+
+The final ranked result set would be [`1`, `5`, `2`, `4`], and we would be able to paginate
+on the top `rank_window_size` results, i.e. [`1`, `5`]. So for the same params as above, we would now have:
+
+* `from=0, size=2` would return [`1`, `5`] with ranks `[1, 2]`
+* `from=2, size=2` would return an empty result set as it would fall outside the available `rank_window_size` results.
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java
index c9b08b5aed4ac..8cc3c6f003fb5 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java
@@ -643,6 +643,10 @@ static ReducedQueryPhase reducedQueryPhase(
             );
             sortedTopDocs = new SortedTopDocs(rankedDocs, false, null, null, null, 0);
             size = sortedTopDocs.scoreDocs.length;
+            // we need to reset from here as pagination and result trimming has already taken place
+            // within the `QueryPhaseRankCoordinatorContext#rankQueryPhaseResults` and we don't want
+            // to apply it again in the `getHits` method.
+            from = 0;
         }
         final TotalHits totalHits = topDocsStats.getTotalHits();
         return new ReducedQueryPhase(
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java
index a20846ab98a4d..12167c8361513 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java
@@ -397,10 +397,10 @@ public ActionRequestValidationException validate() {
                 if (size == 0) {
                     validationException = addValidationError("[rank] requires [size] greater than [0]", validationException);
                 }
-                if (size > source.rankBuilder().windowSize()) {
+                if (size > source.rankBuilder().rankWindowSize()) {
                     validationException = addValidationError(
-                        "[rank] requires [window_size: "
-                            + source.rankBuilder().windowSize()
+                        "[rank] requires [rank_window_size: "
+                            + source.rankBuilder().rankWindowSize()
                             + "]"
                             + " be greater than or equal to [size: "
                             + size
diff --git a/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java b/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java
index cd8b5494ac31f..828c6d2b4f3e8 100644
--- a/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java
+++ b/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java
@@ -98,7 +98,7 @@ static void executeRank(SearchContext searchContext) throws QueryPhaseExecutionE
                 RankSearchContext rankSearchContext = new RankSearchContext(
                     searchContext,
                     rankQuery,
-                    queryPhaseRankShardContext.windowSize()
+                    queryPhaseRankShardContext.rankWindowSize()
                 )
             ) {
                 QueryPhase.addCollectorsAndSearch(rankSearchContext);
diff --git a/server/src/main/java/org/elasticsearch/search/rank/RankBuilder.java b/server/src/main/java/org/elasticsearch/search/rank/RankBuilder.java
index e0e04c563a9a8..7118c9f49b36d 100644
--- a/server/src/main/java/org/elasticsearch/search/rank/RankBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/rank/RankBuilder.java
@@ -30,22 +30,22 @@
  */
 public abstract class RankBuilder implements VersionedNamedWriteable, ToXContentObject {
 
-    public static final ParseField WINDOW_SIZE_FIELD = new ParseField("window_size");
+    public static final ParseField RANK_WINDOW_SIZE_FIELD = new ParseField("rank_window_size");
 
     public static final int DEFAULT_WINDOW_SIZE = SearchService.DEFAULT_SIZE;
 
-    private final int windowSize;
+    private final int rankWindowSize;
 
-    public RankBuilder(int windowSize) {
-        this.windowSize = windowSize;
+    public RankBuilder(int rankWindowSize) {
+        this.rankWindowSize = rankWindowSize;
     }
 
     public RankBuilder(StreamInput in) throws IOException {
-        windowSize = in.readVInt();
+        rankWindowSize = in.readVInt();
     }
 
     public final void writeTo(StreamOutput out) throws IOException {
-        out.writeVInt(windowSize);
+        out.writeVInt(rankWindowSize);
         doWriteTo(out);
     }
 
@@ -55,7 +55,7 @@ public final void writeTo(StreamOutput out) throws IOException {
     public final XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
         builder.startObject(getWriteableName());
-        builder.field(WINDOW_SIZE_FIELD.getPreferredName(), windowSize);
+        builder.field(RANK_WINDOW_SIZE_FIELD.getPreferredName(), rankWindowSize);
         doXContent(builder, params);
         builder.endObject();
         builder.endObject();
@@ -64,8 +64,8 @@ public final XContentBuilder toXContent(XContentBuilder builder, Params params)
 
     protected abstract void doXContent(XContentBuilder builder, Params params) throws IOException;
 
-    public int windowSize() {
-        return windowSize;
+    public int rankWindowSize() {
+        return rankWindowSize;
     }
 
     /**
@@ -88,14 +88,14 @@ public final boolean equals(Object obj) {
         }
         @SuppressWarnings("unchecked")
         RankBuilder other = (RankBuilder) obj;
-        return Objects.equals(windowSize, other.windowSize()) && doEquals(other);
+        return Objects.equals(rankWindowSize, other.rankWindowSize()) && doEquals(other);
     }
 
     protected abstract boolean doEquals(RankBuilder other);
 
     @Override
     public final int hashCode() {
-        return Objects.hash(getClass(), windowSize, doHashCode());
+        return Objects.hash(getClass(), rankWindowSize, doHashCode());
     }
 
     protected abstract int doHashCode();
diff --git a/server/src/main/java/org/elasticsearch/search/rank/context/QueryPhaseRankCoordinatorContext.java b/server/src/main/java/org/elasticsearch/search/rank/context/QueryPhaseRankCoordinatorContext.java
index 181122380f22d..1be8544758a8f 100644
--- a/server/src/main/java/org/elasticsearch/search/rank/context/QueryPhaseRankCoordinatorContext.java
+++ b/server/src/main/java/org/elasticsearch/search/rank/context/QueryPhaseRankCoordinatorContext.java
@@ -22,15 +22,15 @@
  */
 public abstract class QueryPhaseRankCoordinatorContext {
 
-    protected final int windowSize;
+    protected final int rankWindowSize;
 
-    public QueryPhaseRankCoordinatorContext(int windowSize) {
-        this.windowSize = windowSize;
+    public QueryPhaseRankCoordinatorContext(int rankWindowSize) {
+        this.rankWindowSize = rankWindowSize;
     }
 
     /**
      * This is used to pull information passed back from the shards as part of {@link QuerySearchResult#getRankShardResult()}
-     * and return a {@link ScoreDoc[]} of the `window_size` ranked results. Note that {@link TopDocsStats} is included so that
+     * and return a {@link ScoreDoc[]} of the `rank_window_size` ranked results. Note that {@link TopDocsStats} is included so that
      * appropriate stats may be updated based on rank results.
      * This is called when reducing query results through {@code SearchPhaseController#reducedQueryPhase()}.
      */
diff --git a/server/src/main/java/org/elasticsearch/search/rank/context/QueryPhaseRankShardContext.java b/server/src/main/java/org/elasticsearch/search/rank/context/QueryPhaseRankShardContext.java
index e8bac25009e8c..f562977afb857 100644
--- a/server/src/main/java/org/elasticsearch/search/rank/context/QueryPhaseRankShardContext.java
+++ b/server/src/main/java/org/elasticsearch/search/rank/context/QueryPhaseRankShardContext.java
@@ -22,27 +22,27 @@
 public abstract class QueryPhaseRankShardContext {
 
     protected final List<Query> queries;
-    protected final int windowSize;
+    protected final int rankWindowSize;
 
-    public QueryPhaseRankShardContext(List<Query> queries, int windowSize) {
+    public QueryPhaseRankShardContext(List<Query> queries, int rankWindowSize) {
         this.queries = queries;
-        this.windowSize = windowSize;
+        this.rankWindowSize = rankWindowSize;
     }
 
     public List<Query> queries() {
         return queries;
     }
 
-    public int windowSize() {
-        return windowSize;
+    public int rankWindowSize() {
+        return rankWindowSize;
     }
 
     /**
      * This is used to reduce the number of required results that are serialized
-     * to the coordinating node. Normally we would have to serialize {@code queries * window_size}
+     * to the coordinating node. Normally we would have to serialize {@code queries * rank_window_size}
      * results, but we can infer that there will likely be overlap of document results. Given that we
      * know any searches that match the same document must be on the same shard, we can sort on the shard
-     * instead for a top window_size set of results and reduce the amount of data we serialize.
+     * instead for a top rank_window_size set of results and reduce the amount of data we serialize.
      */
     public abstract RankShardResult combineQueryPhaseResults(List<TopDocs> rankResults);
 }
diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchRequestTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchRequestTests.java
index f7f58ef06ccdc..9fd2cd1206ee8 100644
--- a/server/src/test/java/org/elasticsearch/action/search/SearchRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/SearchRequestTests.java
@@ -400,7 +400,7 @@ public void testValidate() throws IOException {
             assertNotNull(validationErrors);
             assertEquals(1, validationErrors.validationErrors().size());
             assertEquals(
-                "[rank] requires [window_size: 1] be greater than or equal to [size: 2]",
+                "[rank] requires [rank_window_size: 1] be greater than or equal to [size: 2]",
                 validationErrors.validationErrors().get(0)
             );
         }
@@ -437,10 +437,21 @@ public void testValidate() throws IOException {
             assertNotNull(validationErrors);
             assertEquals(1, validationErrors.validationErrors().size());
             assertEquals(
-                "[rank] requires [window_size: 9] be greater than or equal to [size: 10]",
+                "[rank] requires [rank_window_size: 9] be greater than or equal to [size: 10]",
                 validationErrors.validationErrors().get(0)
             );
         }
+        {
+            SearchRequest searchRequest = new SearchRequest().source(
+                new SearchSourceBuilder().rankBuilder(new TestRankBuilder(3))
+                    .query(QueryBuilders.termQuery("field", "term"))
+                    .knnSearch(List.of(new KnnSearchBuilder("vector", new float[] { 0f }, 10, 100, null)))
+                    .size(3)
+                    .from(4)
+            );
+            ActionRequestValidationException validationErrors = searchRequest.validate();
+            assertNull(validationErrors);
+        }
         {
             SearchRequest searchRequest = new SearchRequest().source(
                 new SearchSourceBuilder().rankBuilder(new TestRankBuilder(100))
diff --git a/test/framework/src/main/java/org/elasticsearch/search/rank/TestRankBuilder.java b/test/framework/src/main/java/org/elasticsearch/search/rank/TestRankBuilder.java
index 691d541913716..8e2a2c96a31ab 100644
--- a/test/framework/src/main/java/org/elasticsearch/search/rank/TestRankBuilder.java
+++ b/test/framework/src/main/java/org/elasticsearch/search/rank/TestRankBuilder.java
@@ -35,7 +35,7 @@ public class TestRankBuilder extends RankBuilder {
     );
 
     static {
-        PARSER.declareInt(optionalConstructorArg(), WINDOW_SIZE_FIELD);
+        PARSER.declareInt(optionalConstructorArg(), RANK_WINDOW_SIZE_FIELD);
     }
 
     public static TestRankBuilder fromXContent(XContentParser parser) throws IOException {
diff --git a/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFQueryPhaseRankCoordinatorContext.java b/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFQueryPhaseRankCoordinatorContext.java
index 1b3ebe19ce494..b6a1ad52d5d15 100644
--- a/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFQueryPhaseRankCoordinatorContext.java
+++ b/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFQueryPhaseRankCoordinatorContext.java
@@ -62,7 +62,7 @@ public ScoreDoc[] rankQueryPhaseResults(List<QuerySearchResult> querySearchResul
 
                 for (int qi = 0; qi < queryCount; ++qi) {
                     final int fqi = qi;
-                    queues.add(new PriorityQueue<>(windowSize + from) {
+                    queues.add(new PriorityQueue<>(rankWindowSize) {
                         @Override
                         protected boolean lessThan(RRFRankDoc a, RRFRankDoc b) {
                             float score1 = a.scores[fqi];
@@ -105,7 +105,7 @@ protected boolean lessThan(RRFRankDoc a, RRFRankDoc b) {
         // score if we already saw it as part of a previous query's
         // doc set, otherwise we make a new doc and calculate the
         // initial score
-        Map<RankKey, RRFRankDoc> results = Maps.newMapWithExpectedSize(queryCount * windowSize);
+        Map<RankKey, RRFRankDoc> results = Maps.newMapWithExpectedSize(queryCount * rankWindowSize);
         final int fqc = queryCount;
         for (int qi = 0; qi < queryCount; ++qi) {
             PriorityQueue<RRFRankDoc> queue = queues.get(qi);
@@ -127,6 +127,11 @@ protected boolean lessThan(RRFRankDoc a, RRFRankDoc b) {
             }
         }
 
+        // return if pagination requested is outside the results
+        if (results.values().size() - from <= 0) {
+            return new ScoreDoc[0];
+        }
+
         // sort the results based on rrf score, tiebreaker based on
         // larger individual query score from 1 to n, smaller shard then smaller doc id
         RRFRankDoc[] sortedResults = results.values().toArray(RRFRankDoc[]::new);
@@ -151,9 +156,10 @@ protected boolean lessThan(RRFRankDoc a, RRFRankDoc b) {
             }
             return rrf1.doc < rrf2.doc ? -1 : 1;
         });
+        // trim results to size
         RRFRankDoc[] topResults = new RRFRankDoc[Math.min(size, sortedResults.length - from)];
         for (int rank = 0; rank < topResults.length; ++rank) {
-            topResults[rank] = sortedResults[rank];
+            topResults[rank] = sortedResults[from + rank];
             topResults[rank].rank = rank + 1 + from;
         }
         // update fetch hits for the fetch phase, so we gather any additional
diff --git a/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFQueryPhaseRankShardContext.java b/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFQueryPhaseRankShardContext.java
index 59307e62872fb..9843b14df6903 100644
--- a/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFQueryPhaseRankShardContext.java
+++ b/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFQueryPhaseRankShardContext.java
@@ -25,11 +25,9 @@
 public class RRFQueryPhaseRankShardContext extends QueryPhaseRankShardContext {
 
     private final int rankConstant;
-    private final int from;
 
-    public RRFQueryPhaseRankShardContext(List<Query> queries, int from, int windowSize, int rankConstant) {
-        super(queries, windowSize);
-        this.from = from;
+    public RRFQueryPhaseRankShardContext(List<Query> queries, int rankWindowSize, int rankConstant) {
+        super(queries, rankWindowSize);
         this.rankConstant = rankConstant;
     }
 
@@ -41,7 +39,7 @@ public RRFRankShardResult combineQueryPhaseResults(List<TopDocs> rankResults) {
         // if a doc isn't part of a result set its position will be NO_RANK [0] and
         // its score is [0f]
         int queries = rankResults.size();
-        Map<Integer, RRFRankDoc> docsToRankResults = Maps.newMapWithExpectedSize(windowSize);
+        Map<Integer, RRFRankDoc> docsToRankResults = Maps.newMapWithExpectedSize(rankWindowSize);
         int index = 0;
         for (TopDocs rrfRankResult : rankResults) {
             int rank = 1;
@@ -92,8 +90,9 @@ public RRFRankShardResult combineQueryPhaseResults(List<TopDocs> rankResults) {
             }
             return rrf1.doc < rrf2.doc ? -1 : 1;
         });
-        // trim the results to window size
-        RRFRankDoc[] topResults = new RRFRankDoc[Math.min(windowSize + from, sortedResults.length)];
+        // trim the results if needed, otherwise each shard will always return `rank_window_size` results.
+        // pagination and all else will happen on the coordinator when combining the shard responses
+        RRFRankDoc[] topResults = new RRFRankDoc[Math.min(rankWindowSize, sortedResults.length)];
         for (int rank = 0; rank < topResults.length; ++rank) {
             topResults[rank] = sortedResults[rank];
             topResults[rank].rank = rank + 1;
diff --git a/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRankBuilder.java b/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRankBuilder.java
index fee3c7b5e9cf0..8f3ed15037c08 100644
--- a/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRankBuilder.java
+++ b/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRankBuilder.java
@@ -47,7 +47,7 @@ public class RRFRankBuilder extends RankBuilder {
     });
 
     static {
-        PARSER.declareInt(optionalConstructorArg(), WINDOW_SIZE_FIELD);
+        PARSER.declareInt(optionalConstructorArg(), RANK_WINDOW_SIZE_FIELD);
         PARSER.declareInt(optionalConstructorArg(), RANK_CONSTANT_FIELD);
     }
 
@@ -65,8 +65,8 @@ public void doXContent(XContentBuilder builder, Params params) throws IOExceptio
 
     private final int rankConstant;
 
-    public RRFRankBuilder(int windowSize, int rankConstant) {
-        super(windowSize);
+    public RRFRankBuilder(int rankWindowSize, int rankConstant) {
+        super(rankWindowSize);
         this.rankConstant = rankConstant;
     }
 
@@ -94,14 +94,13 @@ public int rankConstant() {
         return rankConstant;
     }
 
-    @Override
     public QueryPhaseRankShardContext buildQueryPhaseShardContext(List<Query> queries, int from) {
-        return new RRFQueryPhaseRankShardContext(queries, from, windowSize(), rankConstant);
+        return new RRFQueryPhaseRankShardContext(queries, rankWindowSize(), rankConstant);
     }
 
     @Override
     public QueryPhaseRankCoordinatorContext buildQueryPhaseCoordinatorContext(int size, int from) {
-        return new RRFQueryPhaseRankCoordinatorContext(size, from, windowSize(), rankConstant);
+        return new RRFQueryPhaseRankCoordinatorContext(size, from, rankWindowSize(), rankConstant);
     }
 
     @Override
diff --git a/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilder.java b/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilder.java
index ea8255f73af88..077c933fa9add 100644
--- a/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilder.java
+++ b/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilder.java
@@ -38,7 +38,7 @@ public final class RRFRetrieverBuilder extends RetrieverBuilder {
     public static final NodeFeature RRF_RETRIEVER_SUPPORTED = new NodeFeature("rrf_retriever_supported");
 
     public static final ParseField RETRIEVERS_FIELD = new ParseField("retrievers");
-    public static final ParseField WINDOW_SIZE_FIELD = new ParseField("window_size");
+    public static final ParseField RANK_WINDOW_SIZE_FIELD = new ParseField("rank_window_size");
     public static final ParseField RANK_CONSTANT_FIELD = new ParseField("rank_constant");
 
     public static final ObjectParser<RRFRetrieverBuilder, RetrieverParserContext> PARSER = new ObjectParser<>(
@@ -54,7 +54,7 @@ public final class RRFRetrieverBuilder extends RetrieverBuilder {
             p.nextToken();
             return retrieverBuilder;
         }, RETRIEVERS_FIELD);
-        PARSER.declareInt((r, v) -> r.windowSize = v, WINDOW_SIZE_FIELD);
+        PARSER.declareInt((r, v) -> r.rankWindowSize = v, RANK_WINDOW_SIZE_FIELD);
         PARSER.declareInt((r, v) -> r.rankConstant = v, RANK_CONSTANT_FIELD);
 
         RetrieverBuilder.declareBaseParserFields(NAME, PARSER);
@@ -71,7 +71,7 @@ public static RRFRetrieverBuilder fromXContent(XContentParser parser, RetrieverP
     }
 
     List<RetrieverBuilder> retrieverBuilders = Collections.emptyList();
-    int windowSize = RRFRankBuilder.DEFAULT_WINDOW_SIZE;
+    int rankWindowSize = RRFRankBuilder.DEFAULT_WINDOW_SIZE;
     int rankConstant = RRFRankBuilder.DEFAULT_RANK_CONSTANT;
 
     @Override
@@ -88,7 +88,7 @@ public void extractToSearchSourceBuilder(SearchSourceBuilder searchSourceBuilder
             retrieverBuilder.extractToSearchSourceBuilder(searchSourceBuilder, true);
         }
 
-        searchSourceBuilder.rankBuilder(new RRFRankBuilder(windowSize, rankConstant));
+        searchSourceBuilder.rankBuilder(new RRFRankBuilder(rankWindowSize, rankConstant));
     }
 
     // ---- FOR TESTING XCONTENT PARSING ----
@@ -113,21 +113,21 @@ public void doToXContent(XContentBuilder builder, Params params) throws IOExcept
             builder.endArray();
         }
 
-        builder.field(WINDOW_SIZE_FIELD.getPreferredName(), windowSize);
+        builder.field(RANK_WINDOW_SIZE_FIELD.getPreferredName(), rankWindowSize);
         builder.field(RANK_CONSTANT_FIELD.getPreferredName(), rankConstant);
     }
 
     @Override
     public boolean doEquals(Object o) {
         RRFRetrieverBuilder that = (RRFRetrieverBuilder) o;
-        return windowSize == that.windowSize
+        return rankWindowSize == that.rankWindowSize
             && rankConstant == that.rankConstant
             && Objects.equals(retrieverBuilders, that.retrieverBuilders);
     }
 
     @Override
     public int doHashCode() {
-        return Objects.hash(retrieverBuilders, windowSize, rankConstant);
+        return Objects.hash(retrieverBuilders, rankWindowSize, rankConstant);
     }
 
     // ---- END FOR TESTING ----
diff --git a/x-pack/plugin/rank-rrf/src/test/java/org/elasticsearch/xpack/rank/rrf/RRFRankBuilderTests.java b/x-pack/plugin/rank-rrf/src/test/java/org/elasticsearch/xpack/rank/rrf/RRFRankBuilderTests.java
index 001857385c5de..7f251416c0a44 100644
--- a/x-pack/plugin/rank-rrf/src/test/java/org/elasticsearch/xpack/rank/rrf/RRFRankBuilderTests.java
+++ b/x-pack/plugin/rank-rrf/src/test/java/org/elasticsearch/xpack/rank/rrf/RRFRankBuilderTests.java
@@ -29,9 +29,9 @@ protected RRFRankBuilder createTestInstance() {
     @Override
     protected RRFRankBuilder mutateInstance(RRFRankBuilder instance) throws IOException {
         if (randomBoolean()) {
-            return new RRFRankBuilder(instance.windowSize(), instance.rankConstant() == 1 ? 2 : instance.rankConstant() - 1);
+            return new RRFRankBuilder(instance.rankWindowSize(), instance.rankConstant() == 1 ? 2 : instance.rankConstant() - 1);
         } else {
-            return new RRFRankBuilder(instance.windowSize() == 0 ? 1 : instance.windowSize() - 1, instance.rankConstant());
+            return new RRFRankBuilder(instance.rankWindowSize() == 0 ? 1 : instance.rankWindowSize() - 1, instance.rankConstant());
         }
     }
 
@@ -61,7 +61,7 @@ public void testCreateRankContexts() {
         List<Query> queries = List.of(new TermQuery(new Term("field0", "test0")), new TermQuery(new Term("field1", "test1")));
         QueryPhaseRankShardContext queryPhaseRankShardContext = rrfRankBuilder.buildQueryPhaseShardContext(queries, randomInt());
         assertEquals(queries, queryPhaseRankShardContext.queries());
-        assertEquals(rrfRankBuilder.windowSize(), queryPhaseRankShardContext.windowSize());
+        assertEquals(rrfRankBuilder.rankWindowSize(), queryPhaseRankShardContext.rankWindowSize());
 
         assertNotNull(rrfRankBuilder.buildQueryPhaseCoordinatorContext(randomInt(), randomInt()));
     }
diff --git a/x-pack/plugin/rank-rrf/src/test/java/org/elasticsearch/xpack/rank/rrf/RRFRankContextTests.java b/x-pack/plugin/rank-rrf/src/test/java/org/elasticsearch/xpack/rank/rrf/RRFRankContextTests.java
index 50aa1d257d214..61859e280acdf 100644
--- a/x-pack/plugin/rank-rrf/src/test/java/org/elasticsearch/xpack/rank/rrf/RRFRankContextTests.java
+++ b/x-pack/plugin/rank-rrf/src/test/java/org/elasticsearch/xpack/rank/rrf/RRFRankContextTests.java
@@ -35,7 +35,7 @@ private void assertRDEquals(RRFRankDoc rd0, RRFRankDoc rd1) {
     }
 
     public void testShardCombine() {
-        RRFQueryPhaseRankShardContext context = new RRFQueryPhaseRankShardContext(null, 0, 10, 1);
+        RRFQueryPhaseRankShardContext context = new RRFQueryPhaseRankShardContext(null, 10, 1);
         List<TopDocs> topDocs = List.of(
             new TopDocs(
                 null,
@@ -266,7 +266,7 @@ public void testCoordinatorRank() {
     }
 
     public void testShardTieBreaker() {
-        RRFQueryPhaseRankShardContext context = new RRFQueryPhaseRankShardContext(null, 0, 10, 1);
+        RRFQueryPhaseRankShardContext context = new RRFQueryPhaseRankShardContext(null, 10, 1);
 
         List<TopDocs> topDocs = List.of(
             new TopDocs(null, new ScoreDoc[] { new ScoreDoc(1, 10.0f, -1), new ScoreDoc(2, 9.0f, -1) }),
diff --git a/x-pack/plugin/rank-rrf/src/test/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderParsingTests.java b/x-pack/plugin/rank-rrf/src/test/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderParsingTests.java
index d63e8a14b59d5..330c936327b81 100644
--- a/x-pack/plugin/rank-rrf/src/test/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderParsingTests.java
+++ b/x-pack/plugin/rank-rrf/src/test/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderParsingTests.java
@@ -32,7 +32,7 @@ public static RRFRetrieverBuilder createRandomRRFRetrieverBuilder() {
         RRFRetrieverBuilder rrfRetrieverBuilder = new RRFRetrieverBuilder();
 
         if (randomBoolean()) {
-            rrfRetrieverBuilder.windowSize = randomIntBetween(1, 10000);
+            rrfRetrieverBuilder.rankWindowSize = randomIntBetween(1, 10000);
         }
 
         if (randomBoolean()) {
diff --git a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/100_rank_rrf.yml b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/100_rank_rrf.yml
index e55a1897eb701..c9eaa01616175 100644
--- a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/100_rank_rrf.yml
+++ b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/100_rank_rrf.yml
@@ -75,7 +75,7 @@ setup:
               text: term
           rank:
             rrf:
-              window_size: 100
+              rank_window_size: 100
               rank_constant: 1
           size: 10
 
@@ -121,7 +121,7 @@ setup:
           ]
           rank:
             rrf:
-              window_size: 100
+              rank_window_size: 100
               rank_constant: 1
           size: 10
 
@@ -169,7 +169,7 @@ setup:
           ]
           rank:
             rrf:
-              window_size: 100
+              rank_window_size: 100
               rank_constant: 1
           size: 10
 
@@ -189,3 +189,41 @@ setup:
   - match: { hits.hits.2._rank: 3 }
   - match: { hits.hits.2.fields.text.0: "other" }
   - match: { hits.hits.2.fields.keyword.0: "other" }
+
+
+---
+"RRF rank should fail if size > rank_window_size":
+
+  - do:
+      catch: "/\\[rank\\] requires \\[rank_window_size: 2\\] be greater than or equal to \\[size: 10\\]/"
+      search:
+        index: test
+        body:
+          track_total_hits: true
+          fields: [ "text", "keyword" ]
+          knn:
+            field: vector
+            query_vector: [ 0.0 ]
+            k: 3
+            num_candidates: 3
+          sub_searches: [
+            {
+              "query": {
+                "term": {
+                  "text": "term"
+                }
+              }
+            },
+            {
+              "query": {
+                "match": {
+                  "keyword": "keyword"
+                }
+              }
+            }
+          ]
+          rank:
+            rrf:
+              rank_window_size: 2
+              rank_constant: 1
+          size: 10
diff --git a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/150_rank_rrf_pagination.yml b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/150_rank_rrf_pagination.yml
new file mode 100644
index 0000000000000..1c950be5bfbf9
--- /dev/null
+++ b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/150_rank_rrf_pagination.yml
@@ -0,0 +1,1055 @@
+setup:
+  - skip:
+      version: ' - 8.14.99'
+      reason: 'pagination for rrf was added in 8.15'
+
+  - do:
+      indices.create:
+        index: test
+        body:
+          settings:
+            number_of_shards: 1
+            number_of_replicas: 0
+          mappings:
+            properties:
+              number_val:
+                type: keyword
+              char_val:
+                type: keyword
+
+  - do:
+      index:
+        index: test
+        id: 1
+        body:
+          number_val: "1"
+          char_val: "A"
+
+  - do:
+      index:
+        index: test
+        id: 2
+        body:
+          number_val: "2"
+          char_val: "B"
+
+  - do:
+      index:
+        index: test
+        id: 3
+        body:
+          number_val: "3"
+          char_val: "C"
+
+  - do:
+      index:
+        index: test
+        id: 4
+        body:
+          number_val: "4"
+          char_val: "D"
+
+  - do:
+      index:
+        index: test
+        id: 5
+        body:
+          number_val: "5"
+          char_val: "E"
+
+  - do:
+      indices.refresh: {}
+
+---
+"Standard pagination within rank_window_size":
+  # this test retrieves the same results from two queries, and applies a simple pagination skipping the first result
+  - do:
+      search:
+        index: test
+        body:
+          track_total_hits: true
+          sub_searches: [
+            {
+              # this should clause would generate the result set [1, 2, 3, 4]
+              "query": {
+                bool: {
+                  should: [
+                    {
+                      term: {
+                        number_val: {
+                          value: "1",
+                          boost: 10.0
+                        }
+                      }
+                    },
+                    {
+                      term: {
+                        number_val: {
+                          value: "2",
+                          boost: 9.0
+                        }
+                      }
+                    },
+                    {
+                      term: {
+                        number_val: {
+                          value: "3",
+                          boost: 8.0
+                        }
+                      }
+                    },
+                    {
+                      term: {
+                        number_val: {
+                          value: "4",
+                          boost: 7.0
+                        }
+                      }
+                    }
+                  ]
+                }
+              }
+
+            },
+            {
+              # this should clause would generate the result set [1, 2, 3, 4]
+              "query": {
+                bool: {
+                  should: [
+                    {
+                      term: {
+                        char_val: {
+                          value: "A",
+                          boost: 10.0
+                        }
+                      }
+                    },
+                    {
+                      term: {
+                        char_val: {
+                          value: "B",
+                          boost: 9.0
+                        }
+                      }
+                    },
+                    {
+                      term: {
+                        char_val: {
+                          value: "C",
+                          boost: 8.0
+                        }
+                      }
+                    },
+                    {
+                      term: {
+                        char_val: {
+                          value: "D",
+                          boost: 7.0
+                        }
+                      }
+                    }
+                  ]
+                }
+              }
+
+            }
+          ]
+          rank:
+            rrf:
+              rank_window_size: 10
+              rank_constant: 10
+          from : 1
+          size : 10
+
+  - match: { hits.total.value : 4 }
+  - length: { hits.hits : 3 }
+  - match: { hits.hits.0._id: "2" }
+  - match: { hits.hits.0._rank: 2 }
+  - match: { hits.hits.1._id: "3" }
+  - match: { hits.hits.1._rank: 3 }
+  - match: { hits.hits.2._id: "4" }
+  - match: { hits.hits.2._rank: 4 }
+
+---
+"Standard pagination outside rank_window_size":
+  # in this example, from starts *after* rank_window_size so, we expect 0 results to be returned
+  - do:
+      search:
+        index: test
+        body:
+          track_total_hits: true
+          sub_searches: [
+            {
+              # this should clause would generate the result set [1, 2, 3, 4]
+              "query": {
+                bool: {
+                  should: [
+                    {
+                      term: {
+                        number_val: {
+                          value: "1",
+                          boost: 10.0
+                        }
+                      }
+                    },
+                    {
+                      term: {
+                        number_val: {
+                          value: "2",
+                          boost: 9.0
+                        }
+                      }
+                    },
+                    {
+                      term: {
+                        number_val: {
+                          value: "3",
+                          boost: 8.0
+                        }
+                      }
+                    },
+                    {
+                      term: {
+                        number_val: {
+                          value: "4",
+                          boost: 7.0
+                        }
+                      }
+                    }
+                  ]
+                }
+              }
+
+            },
+            {
+              # this should clause would generate the result set [1, 2, 3, 4]
+              "query": {
+                bool: {
+                  should: [
+                    {
+                      term: {
+                        char_val: {
+                          value: "A",
+                          boost: 10.0
+                        }
+                      }
+                    },
+                    {
+                      term: {
+                        char_val: {
+                          value: "B",
+                          boost: 9.0
+                        }
+                      }
+                    },
+                    {
+                      term: {
+                        char_val: {
+                          value: "C",
+                          boost: 8.0
+                        }
+                      }
+                    },
+                    {
+                      term: {
+                        char_val: {
+                          value: "D",
+                          boost: 7.0
+                        }
+                      }
+                    }
+                  ]
+                }
+              }
+
+            }
+          ]
+          rank:
+            rrf:
+              rank_window_size: 2
+              rank_constant: 10
+          from : 10
+          size : 2
+
+  - match: { hits.total.value : 4 }
+  - length: { hits.hits : 0 }
+
+---
+"Standard pagination partially outside rank_window_size":
+  # in this example we have that from starts *within* rank_window_size, but "from + size" goes over
+  - do:
+      search:
+        index: test
+        body:
+          track_total_hits: true
+          sub_searches: [
+            {
+              # this should clause would generate the result set [1, 2, 3, 4]
+              "query": {
+                bool: {
+                  should: [
+                    {
+                      term: {
+                        number_val: {
+                          value: "1",
+                          boost: 10.0
+                        }
+                      }
+                    },
+                    {
+                      term: {
+                        number_val: {
+                          value: "2",
+                          boost: 9.0
+                        }
+                      }
+                    },
+                    {
+                      term: {
+                        number_val: {
+                          value: "3",
+                          boost: 8.0
+                        }
+                      }
+                    },
+                    {
+                      term: {
+                        number_val: {
+                          value: "4",
+                          boost: 7.0
+                        }
+                      }
+                    }
+                  ]
+                }
+              }
+
+            },
+            {
+              # this should clause would generate the result set [1, 2, 3, 4]
+              "query": {
+                bool: {
+                  should: [
+                    {
+                      term: {
+                        char_val: {
+                          value: "A",
+                          boost: 10.0
+                        }
+                      }
+                    },
+                    {
+                      term: {
+                        char_val: {
+                          value: "B",
+                          boost: 9.0
+                        }
+                      }
+                    },
+                    {
+                      term: {
+                        char_val: {
+                          value: "C",
+                          boost: 8.0
+                        }
+                      }
+                    },
+                    {
+                      term: {
+                        char_val: {
+                          value: "D",
+                          boost: 7.0
+                        }
+                      }
+                    }
+                  ]
+                }
+              }
+
+            }
+          ]
+          rank:
+            rrf:
+              rank_window_size: 3
+              rank_constant: 10
+          from : 2
+          size : 2
+
+  - match: { hits.total.value : 4 }
+  - length: { hits.hits : 1 }
+  - match: { hits.hits.0._id: "3" }
+  - match: { hits.hits.0._rank: 3 }
+
+
+---
+"Pagination within interleaved results":
+  # perform two searches with different "from" parameter, ensuring that results are consistent
+  # rank_window_size covers the entire result set for both queries, so pagination should be consistent
+  # queryA has a result set of [1, 2, 3, 4] and
+  # queryB has a result set of [4, 3, 1, 2]
+  # so for rank_constant=10, the expected order is [1, 4, 3, 2]
+  - do:
+      search:
+        index: test
+        body:
+          track_total_hits: true
+          sub_searches: [
+            {
+              # this should clause would generate the result set [1, 2, 3, 4]
+              "query": {
+                bool: {
+                  should: [
+                    {
+                      term: {
+                        number_val: {
+                          value: "1",
+                          boost: 10.0
+                        }
+                      }
+                    },
+                    {
+                      term: {
+                        number_val: {
+                          value: "2",
+                          boost: 9.0
+                        }
+                      }
+                    },
+                    {
+                      term: {
+                        number_val: {
+                          value: "3",
+                          boost: 8.0
+                        }
+                      }
+                    },
+                    {
+                      term: {
+                        number_val: {
+                          value: "4",
+                          boost: 7.0
+                        }
+                      }
+                    }
+                  ]
+                }
+              }
+
+            },
+            {
+              # this should clause would generate the result set [4, 3, 1, 2]
+              "query": {
+                bool: {
+                  should: [
+                    {
+                      term: {
+                        char_val: {
+                          value: "D",
+                          boost: 10.0
+                        }
+                      }
+                    },
+                    {
+                      term: {
+                        char_val: {
+                          value: "C",
+                          boost: 9.0
+                        }
+                      }
+                    },
+                    {
+                      term: {
+                        char_val: {
+                          value: "A",
+                          boost: 8.0
+                        }
+                      }
+                    },
+                    {
+                      term: {
+                        char_val: {
+                          value: "B",
+                          boost: 7.0
+                        }
+                      }
+                    }
+                  ]
+                }
+              }
+
+            }
+          ]
+          rank:
+            rrf:
+              rank_window_size: 10
+              rank_constant: 10
+          from : 0
+          size : 2
+
+  - match: { hits.total.value : 4 }
+  - length: { hits.hits : 2 }
+  - match: { hits.hits.0._id: "1" }
+  - match: { hits.hits.0._rank: 1 }
+  - match: { hits.hits.1._id: "4" }
+  - match: { hits.hits.1._rank: 2 }
+
+  - do:
+      search:
+        index: test
+        body:
+          track_total_hits: true
+          sub_searches: [
+            {
+              # this should clause would generate the result set [1, 2, 3, 4]
+              "query": {
+                bool: {
+                  should: [
+                    {
+                      term: {
+                        number_val: {
+                          value: "1",
+                          boost: 10.0
+                        }
+                      }
+                    },
+                    {
+                      term: {
+                        number_val: {
+                          value: "2",
+                          boost: 9.0
+                        }
+                      }
+                    },
+                    {
+                      term: {
+                        number_val: {
+                          value: "3",
+                          boost: 8.0
+                        }
+                      }
+                    },
+                    {
+                      term: {
+                        number_val: {
+                          value: "4",
+                          boost: 7.0
+                        }
+                      }
+                    }
+                  ]
+                }
+              }
+
+            },
+            {
+              # this should clause would generate the result set [4, 3, 1, 2]
+              "query": {
+                bool: {
+                  should: [
+                    {
+                      term: {
+                        char_val: {
+                          value: "D",
+                          boost: 10.0
+                        }
+                      }
+                    },
+                    {
+                      term: {
+                        char_val: {
+                          value: "C",
+                          boost: 9.0
+                        }
+                      }
+                    },
+                    {
+                      term: {
+                        char_val: {
+                          value: "A",
+                          boost: 8.0
+                        }
+                      }
+                    },
+                    {
+                      term: {
+                        char_val: {
+                          value: "B",
+                          boost: 7.0
+                        }
+                      }
+                    }
+                  ]
+                }
+              }
+
+            }
+          ]
+          rank:
+            rrf:
+              rank_window_size: 10
+              rank_constant: 10
+          from : 2
+          size : 2
+
+  - match: { hits.total.value : 4 }
+  - length: { hits.hits : 2 }
+  - match: { hits.hits.0._id: "3" }
+  - match: { hits.hits.0._rank: 3 }
+  - match: { hits.hits.1._id: "2" }
+  - match: { hits.hits.1._rank: 4 }
+
+---
+"Pagination within interleaved results, different result set sizes, rank_window_size covering all results":
+  # perform multiple searches with different "from" parameter, ensuring that results are consistent
+  # rank_window_size covers the entire result set for both queries, so pagination should be consistent
+  # queryA has a result set of [5, 1] and
+  # queryB has a result set of [4, 3, 1, 2]
+  # so for rank_constant=10, the expected order is [1, 5, 4, 3, 2]
+  - do:
+      search:
+        index: test
+        body:
+          track_total_hits: true
+          sub_searches: [
+            {
+              # this should clause would generate the result set [5, 1]
+              "query": {
+                bool: {
+                  should: [
+                    {
+                      term: {
+                        number_val: {
+                          value: "5",
+                          boost: 10.0
+                        }
+                      }
+                    },
+                    {
+                      term: {
+                        number_val: {
+                          value: "1",
+                          boost: 9.0
+                        }
+                      }
+                    }
+                  ]
+                }
+              }
+
+            },
+            {
+              # this should clause would generate the result set [4, 3, 1, 2]
+              "query": {
+                bool: {
+                  should: [
+                    {
+                      term: {
+                        char_val: {
+                          value: "D",
+                          boost: 10.0
+                        }
+                      }
+                    },
+                    {
+                      term: {
+                        char_val: {
+                          value: "C",
+                          boost: 9.0
+                        }
+                      }
+                    },
+                    {
+                      term: {
+                        char_val: {
+                          value: "A",
+                          boost: 8.0
+                        }
+                      }
+                    },
+                    {
+                      term: {
+                        char_val: {
+                          value: "B",
+                          boost: 7.0
+                        }
+                      }
+                    }
+                  ]
+                }
+              }
+
+            }
+          ]
+          rank:
+            rrf:
+              rank_window_size: 10
+              rank_constant: 10
+          from : 0
+          size : 2
+
+  - match: { hits.total.value : 5 }
+  - length: { hits.hits : 2 }
+  - match: { hits.hits.0._id: "1" }
+  - match: { hits.hits.0._rank: 1 }
+  - match: { hits.hits.1._id: "5" }
+  - match: { hits.hits.1._rank: 2 }
+
+  - do:
+      search:
+        index: test
+        body:
+          track_total_hits: true
+          sub_searches: [
+            {
+              # this should clause would generate the result set [5, 1]
+              "query": {
+                bool: {
+                  should: [
+                    {
+                      term: {
+                        number_val: {
+                          value: "5",
+                          boost: 10.0
+                        }
+                      }
+                    },
+                    {
+                      term: {
+                        number_val: {
+                          value: "1",
+                          boost: 9.0
+                        }
+                      }
+                    }
+                  ]
+                }
+              }
+
+            },
+            {
+              # this should clause would generate the result set [4, 3, 1, 2]
+              "query": {
+                bool: {
+                  should: [
+                    {
+                      term: {
+                        char_val: {
+                          value: "D",
+                          boost: 10.0
+                        }
+                      }
+                    },
+                    {
+                      term: {
+                        char_val: {
+                          value: "C",
+                          boost: 9.0
+                        }
+                      }
+                    },
+                    {
+                      term: {
+                        char_val: {
+                          value: "A",
+                          boost: 8.0
+                        }
+                      }
+                    },
+                    {
+                      term: {
+                        char_val: {
+                          value: "B",
+                          boost: 7.0
+                        }
+                      }
+                    }
+                  ]
+                }
+              }
+
+            }
+          ]
+          rank:
+            rrf:
+              rank_window_size: 10
+              rank_constant: 10
+          from : 2
+          size : 2
+
+  - match: { hits.total.value : 5 }
+  - length: { hits.hits : 2 }
+  - match: { hits.hits.0._id: "4" }
+  - match: { hits.hits.0._rank: 3 }
+  - match: { hits.hits.1._id: "3" }
+  - match: { hits.hits.1._rank: 4 }
+
+  - do:
+      search:
+        index: test
+        body:
+          track_total_hits: true
+          sub_searches: [
+            {
+              # this should clause would generate the result set [5, 1]
+              "query": {
+                bool: {
+                  should: [
+                    {
+                      term: {
+                        number_val: {
+                          value: "5",
+                          boost: 10.0
+                        }
+                      }
+                    },
+                    {
+                      term: {
+                        number_val: {
+                          value: "1",
+                          boost: 9.0
+                        }
+                      }
+                    }
+                  ]
+                }
+              }
+
+            },
+            {
+              # this should clause would generate the result set [4, 3, 1, 2]
+              "query": {
+                bool: {
+                  should: [
+                    {
+                      term: {
+                        char_val: {
+                          value: "D",
+                          boost: 10.0
+                        }
+                      }
+                    },
+                    {
+                      term: {
+                        char_val: {
+                          value: "C",
+                          boost: 9.0
+                        }
+                      }
+                    },
+                    {
+                      term: {
+                        char_val: {
+                          value: "A",
+                          boost: 8.0
+                        }
+                      }
+                    },
+                    {
+                      term: {
+                        char_val: {
+                          value: "B",
+                          boost: 7.0
+                        }
+                      }
+                    }
+                  ]
+                }
+              }
+
+            }
+          ]
+          rank:
+            rrf:
+              rank_window_size: 10
+              rank_constant: 10
+          from: 4
+          size: 2
+
+  - match: { hits.total.value: 5 }
+  - length: { hits.hits: 1 }
+  - match: { hits.hits.0._id: "2" }
+  - match: { hits.hits.0._rank: 5 }
+
+
+---
+"Pagination within interleaved results, different result set sizes, rank_window_size not covering all results":
+  # perform multiple searches with different "from" parameter, ensuring that results are consistent
+  # rank_window_size does not cover the entire result set for both queries, so the results should be different
+  # from the test above. More specifically, we'd get to collect 2 results from each query, so we'd have:
+  # queryA has a result set of [5, 1] and
+  # queryB has a result set of [4, 3]
+  # so for rank_constant=10, the expected order is [5, 4, 1, 3],
+  # and the rank_window_size-sized result set that we'd paginate over is [5, 4]
+  - do:
+      search:
+        index: test
+        body:
+          track_total_hits: true
+          sub_searches: [
+            {
+              # this should clause would generate the result set [5, 1]
+              "query": {
+                bool: {
+                  should: [
+                    {
+                      term: {
+                        number_val: {
+                          value: "5",
+                          boost: 10.0
+                        }
+                      }
+                    },
+                    {
+                      term: {
+                        number_val: {
+                          value: "1",
+                          boost: 9.0
+                        }
+                      }
+                    }
+                  ]
+                }
+              }
+
+            },
+            {
+              # this should clause would generate the result set [4, 3, 1, 2]
+              "query": {
+                bool: {
+                  should: [
+                    {
+                      term: {
+                        char_val: {
+                          value: "D",
+                          boost: 10.0
+                        }
+                      }
+                    },
+                    {
+                      term: {
+                        char_val: {
+                          value: "C",
+                          boost: 9.0
+                        }
+                      }
+                    },
+                    {
+                      term: {
+                        char_val: {
+                          value: "A",
+                          boost: 8.0
+                        }
+                      }
+                    },
+                    {
+                      term: {
+                        char_val: {
+                          value: "B",
+                          boost: 7.0
+                        }
+                      }
+                    }
+                  ]
+                }
+              }
+
+            }
+          ]
+          rank:
+            rrf:
+              rank_window_size: 2
+              rank_constant: 10
+          from : 0
+          size : 2
+
+  - match: { hits.total.value : 5 }
+  - length: { hits.hits : 2 }
+  - match: { hits.hits.0._id: "5" }
+  - match: { hits.hits.0._rank: 1 }
+  - match: { hits.hits.1._id: "4" }
+  - match: { hits.hits.1._rank: 2 }
+
+  - do:
+      search:
+        index: test
+        body:
+          track_total_hits: true
+          sub_searches: [
+            {
+              # this should clause would generate the result set [5, 1]
+              "query": {
+                bool: {
+                  should: [
+                    {
+                      term: {
+                        number_val: {
+                          value: "5",
+                          boost: 10.0
+                        }
+                      }
+                    },
+                    {
+                      term: {
+                        number_val: {
+                          value: "1",
+                          boost: 9.0
+                        }
+                      }
+                    }
+                  ]
+                }
+              }
+
+            },
+            {
+              # this should clause would generate the result set [4, 3, 1, 2]
+              "query": {
+                bool: {
+                  should: [
+                    {
+                      term: {
+                        char_val: {
+                          value: "D",
+                          boost: 10.0
+                        }
+                      }
+                    },
+                    {
+                      term: {
+                        char_val: {
+                          value: "C",
+                          boost: 9.0
+                        }
+                      }
+                    },
+                    {
+                      term: {
+                        char_val: {
+                          value: "A",
+                          boost: 8.0
+                        }
+                      }
+                    },
+                    {
+                      term: {
+                        char_val: {
+                          value: "B",
+                          boost: 7.0
+                        }
+                      }
+                    }
+                  ]
+                }
+              }
+
+            }
+          ]
+          rank:
+            rrf:
+              rank_window_size: 2
+              rank_constant: 10
+          from : 2
+          size : 2
+
+  - match: { hits.total.value : 5 }
+  - length: { hits.hits : 0 }
diff --git a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/200_rank_rrf_script.yml b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/200_rank_rrf_script.yml
index de5b29b21da72..0583e6d7ae51a 100644
--- a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/200_rank_rrf_script.yml
+++ b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/200_rank_rrf_script.yml
@@ -124,7 +124,7 @@ setup:
                   ]
           rank:
             rrf:
-              window_size: 100
+              rank_window_size: 100
               rank_constant: 1
           size: 5
 
@@ -189,7 +189,7 @@ setup:
                   ]
           rank:
             rrf:
-              window_size: 3
+              rank_window_size: 3
               rank_constant: 1
           size: 1
 
@@ -267,7 +267,7 @@ setup:
                   ]
           rank:
             rrf:
-              window_size: 100
+              rank_window_size: 100
               rank_constant: 1
           size: 5
 
diff --git a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/300_rrf_retriever.yml b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/300_rrf_retriever.yml
index 1387c37349cd4..d3d45ef2b18e8 100644
--- a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/300_rrf_retriever.yml
+++ b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/300_rrf_retriever.yml
@@ -86,7 +86,7 @@ setup:
                   }
                 }
               ]
-              window_size: 100
+              rank_window_size: 100
               rank_constant: 1
           size: 10
 
@@ -136,7 +136,7 @@ setup:
                   }
                 }
               ]
-              window_size: 100
+              rank_window_size: 100
               rank_constant: 1
           size: 10
 
@@ -191,7 +191,7 @@ setup:
                   }
                 }
               ]
-              window_size: 100
+              rank_window_size: 100
               rank_constant: 1
           size: 10
 
@@ -260,7 +260,7 @@ setup:
                   }
                 }
               ]
-              window_size: 2
+              rank_window_size: 2
               rank_constant: 1
 
   - match: { hits.total.value : 3 }
diff --git a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/400_rrf_retriever_script.yml b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/400_rrf_retriever_script.yml
index 2c2b59f306ee3..520389d51b737 100644
--- a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/400_rrf_retriever_script.yml
+++ b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/400_rrf_retriever_script.yml
@@ -138,7 +138,7 @@ setup:
                   }
                 }
               ]
-              window_size: 100
+              rank_window_size: 100
               rank_constant: 1
           aggs:
             sums:
@@ -207,7 +207,7 @@ setup:
                   }
                 }
               ]
-              window_size: 3
+              rank_window_size: 3
               rank_constant: 1
           aggs:
             sums:
@@ -308,7 +308,7 @@ setup:
                   }
                 }
               ]
-              window_size: 100
+              rank_window_size: 100
               rank_constant: 1
           aggs:
             sums:

From 4ef8b3825ec40051ead58d2c6180d6b80be060be Mon Sep 17 00:00:00 2001
From: Oleksandr Kolomiiets <oleksandr.kolomiiets@elastic.co>
Date: Thu, 25 Apr 2024 10:27:32 -0700
Subject: [PATCH 34/58] Revert "Format default values of IP ranges to match
 other range bound" (#107910)

This reverts commit acb5139 (a part of #107081). This commit impacts search behaviour for IP range fields and so needs to be reverted.
---
 .../test/range/20_synthetic_source.yml        |   2 +-
 .../index/mapper/RangeFieldMapper.java        | 112 ++++++++----------
 .../elasticsearch/index/mapper/RangeType.java |  29 -----
 .../index/mapper/IpRangeFieldMapperTests.java |  31 -----
 .../index/mapper/RangeFieldMapperTests.java   |   4 +-
 5 files changed, 54 insertions(+), 124 deletions(-)

diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/range/20_synthetic_source.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/range/20_synthetic_source.yml
index 60c61ddbb698e..3551d022c2f4a 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/range/20_synthetic_source.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/range/20_synthetic_source.yml
@@ -515,7 +515,7 @@ setup:
         id: "7"
   - match:
       _source:
-        ip_range: { "gte": "0.0.0.0", "lte": "10.10.10.10" }
+        ip_range: { "gte": "::", "lte": "10.10.10.10" }
 
   - do:
       get:
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java
index 885e0c9f2642d..f84a1b540a2be 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java
@@ -381,76 +381,66 @@ protected boolean supportsParsingObject() {
 
     @Override
     protected void parseCreateField(DocumentParserContext context) throws IOException {
+        Range range;
         XContentParser parser = context.parser();
-        if (parser.currentToken() == XContentParser.Token.VALUE_NULL) {
-            return;
-        }
-
-        Range range = parseRange(parser);
-        context.doc().addAll(fieldType().rangeType.createFields(context, name(), range, index, hasDocValues, store));
-
-        if (hasDocValues == false && (index || store)) {
-            context.addToFieldNames(fieldType().name());
-        }
-    }
-
-    private Range parseRange(XContentParser parser) throws IOException {
         final XContentParser.Token start = parser.currentToken();
-        if (fieldType().rangeType == RangeType.IP && start == XContentParser.Token.VALUE_STRING) {
-            return parseIpRangeFromCidr(parser);
-        }
-
-        if (start != XContentParser.Token.START_OBJECT) {
+        if (start == XContentParser.Token.VALUE_NULL) {
+            return;
+        } else if (start == XContentParser.Token.START_OBJECT) {
+            RangeFieldType fieldType = fieldType();
+            RangeType rangeType = fieldType.rangeType;
+            String fieldName = null;
+            Object from = rangeType.minValue();
+            Object to = rangeType.maxValue();
+            boolean includeFrom = DEFAULT_INCLUDE_LOWER;
+            boolean includeTo = DEFAULT_INCLUDE_UPPER;
+            XContentParser.Token token;
+            while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+                if (token == XContentParser.Token.FIELD_NAME) {
+                    fieldName = parser.currentName();
+                } else {
+                    if (fieldName.equals(GT_FIELD.getPreferredName())) {
+                        includeFrom = false;
+                        if (parser.currentToken() != XContentParser.Token.VALUE_NULL) {
+                            from = rangeType.parseFrom(fieldType, parser, coerce.value(), includeFrom);
+                        }
+                    } else if (fieldName.equals(GTE_FIELD.getPreferredName())) {
+                        includeFrom = true;
+                        if (parser.currentToken() != XContentParser.Token.VALUE_NULL) {
+                            from = rangeType.parseFrom(fieldType, parser, coerce.value(), includeFrom);
+                        }
+                    } else if (fieldName.equals(LT_FIELD.getPreferredName())) {
+                        includeTo = false;
+                        if (parser.currentToken() != XContentParser.Token.VALUE_NULL) {
+                            to = rangeType.parseTo(fieldType, parser, coerce.value(), includeTo);
+                        }
+                    } else if (fieldName.equals(LTE_FIELD.getPreferredName())) {
+                        includeTo = true;
+                        if (parser.currentToken() != XContentParser.Token.VALUE_NULL) {
+                            to = rangeType.parseTo(fieldType, parser, coerce.value(), includeTo);
+                        }
+                    } else {
+                        throw new DocumentParsingException(
+                            parser.getTokenLocation(),
+                            "error parsing field [" + name() + "], with unknown parameter [" + fieldName + "]"
+                        );
+                    }
+                }
+            }
+            range = new Range(rangeType, from, to, includeFrom, includeTo);
+        } else if (fieldType().rangeType == RangeType.IP && start == XContentParser.Token.VALUE_STRING) {
+            range = parseIpRangeFromCidr(parser);
+        } else {
             throw new DocumentParsingException(
                 parser.getTokenLocation(),
                 "error parsing field [" + name() + "], expected an object but got " + parser.currentName()
             );
         }
+        context.doc().addAll(fieldType().rangeType.createFields(context, name(), range, index, hasDocValues, store));
 
-        RangeFieldType fieldType = fieldType();
-        RangeType rangeType = fieldType.rangeType;
-        String fieldName = null;
-        Object parsedFrom = null;
-        Object parsedTo = null;
-        boolean includeFrom = DEFAULT_INCLUDE_LOWER;
-        boolean includeTo = DEFAULT_INCLUDE_UPPER;
-        XContentParser.Token token;
-        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-            if (token == XContentParser.Token.FIELD_NAME) {
-                fieldName = parser.currentName();
-            } else {
-                if (fieldName.equals(GT_FIELD.getPreferredName())) {
-                    includeFrom = false;
-                    if (parser.currentToken() != XContentParser.Token.VALUE_NULL) {
-                        parsedFrom = rangeType.parseFrom(fieldType, parser, coerce.value(), includeFrom);
-                    }
-                } else if (fieldName.equals(GTE_FIELD.getPreferredName())) {
-                    includeFrom = true;
-                    if (parser.currentToken() != XContentParser.Token.VALUE_NULL) {
-                        parsedFrom = rangeType.parseFrom(fieldType, parser, coerce.value(), includeFrom);
-                    }
-                } else if (fieldName.equals(LT_FIELD.getPreferredName())) {
-                    includeTo = false;
-                    if (parser.currentToken() != XContentParser.Token.VALUE_NULL) {
-                        parsedTo = rangeType.parseTo(fieldType, parser, coerce.value(), includeTo);
-                    }
-                } else if (fieldName.equals(LTE_FIELD.getPreferredName())) {
-                    includeTo = true;
-                    if (parser.currentToken() != XContentParser.Token.VALUE_NULL) {
-                        parsedTo = rangeType.parseTo(fieldType, parser, coerce.value(), includeTo);
-                    }
-                } else {
-                    throw new DocumentParsingException(
-                        parser.getTokenLocation(),
-                        "error parsing field [" + name() + "], with unknown parameter [" + fieldName + "]"
-                    );
-                }
-            }
+        if (hasDocValues == false && (index || store)) {
+            context.addToFieldNames(fieldType().name());
         }
-        Object from = parsedFrom != null ? parsedFrom : rangeType.defaultFrom(parsedTo);
-        Object to = parsedTo != null ? parsedTo : rangeType.defaultTo(parsedFrom);
-
-        return new Range(rangeType, from, to, includeFrom, includeTo);
     }
 
     private static Range parseIpRangeFromCidr(final XContentParser parser) throws IOException {
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RangeType.java b/server/src/main/java/org/elasticsearch/index/mapper/RangeType.java
index 24a1eb869cf25..bb4fb1acc0b14 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/RangeType.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/RangeType.java
@@ -31,7 +31,6 @@
 import org.elasticsearch.xcontent.XContentParser;
 
 import java.io.IOException;
-import java.net.Inet4Address;
 import java.net.InetAddress;
 import java.time.Instant;
 import java.time.ZoneId;
@@ -65,26 +64,6 @@ public InetAddress parseTo(RangeFieldMapper.RangeFieldType fieldType, XContentPa
             return included ? address : nextDown(address);
         }
 
-        public Object defaultFrom(Object parsedTo) {
-            if (parsedTo == null) {
-                return minValue();
-            }
-
-            // Make sure that we keep the range inside the same address family.
-            // `minValue()` is always IPv6 so we need to adjust it.
-            return parsedTo instanceof Inet4Address ? InetAddressPoint.decode(new byte[4]) : minValue();
-        }
-
-        public Object defaultTo(Object parsedFrom) {
-            if (parsedFrom == null) {
-                return maxValue();
-            }
-
-            // Make sure that we keep the range inside the same address family.
-            // `maxValue()` is always IPv6 so we need to adjust it.
-            return parsedFrom instanceof Inet4Address ? InetAddressPoint.decode(new byte[] { -1, -1, -1, -1 }) : maxValue();
-        }
-
         @Override
         public InetAddress parseValue(Object value, boolean coerce, @Nullable DateMathParser dateMathParser) {
             if (value instanceof InetAddress) {
@@ -865,14 +844,6 @@ public Object parseTo(RangeFieldMapper.RangeFieldType fieldType, XContentParser
         return included ? value : (Number) nextDown(value);
     }
 
-    public Object defaultFrom(Object parsedTo) {
-        return minValue();
-    }
-
-    public Object defaultTo(Object parsedFrom) {
-        return maxValue();
-    }
-
     public abstract Object minValue();
 
     public abstract Object maxValue();
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IpRangeFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IpRangeFieldMapperTests.java
index 279c9263c98a9..ddec4b8ca65e5 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/IpRangeFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/IpRangeFieldMapperTests.java
@@ -87,37 +87,6 @@ public void testStoreCidr() throws Exception {
         }
     }
 
-    @Override
-    public void testNullBounds() throws IOException {
-        DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> {
-            minimalMapping(b);
-            b.field("store", true);
-        }));
-
-        ParsedDocument bothNull = mapper.parse(source(b -> b.startObject("field").nullField("gte").nullField("lte").endObject()));
-        assertThat(storedValue(bothNull), equalTo("[:: : ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff]"));
-
-        ParsedDocument onlyFromIPv4 = mapper.parse(
-            source(b -> b.startObject("field").field("gte", rangeValue()).nullField("lte").endObject())
-        );
-        assertThat(storedValue(onlyFromIPv4), equalTo("[192.168.1.7 : 255.255.255.255]"));
-
-        ParsedDocument onlyToIPv4 = mapper.parse(
-            source(b -> b.startObject("field").nullField("gte").field("lte", rangeValue()).endObject())
-        );
-        assertThat(storedValue(onlyToIPv4), equalTo("[0.0.0.0 : 192.168.1.7]"));
-
-        ParsedDocument onlyFromIPv6 = mapper.parse(
-            source(b -> b.startObject("field").field("gte", "2001:db8::").nullField("lte").endObject())
-        );
-        assertThat(storedValue(onlyFromIPv6), equalTo("[2001:db8:: : ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff]"));
-
-        ParsedDocument onlyToIPv6 = mapper.parse(
-            source(b -> b.startObject("field").nullField("gte").field("lte", "2001:db8::").endObject())
-        );
-        assertThat(storedValue(onlyToIPv6), equalTo("[:: : 2001:db8::]"));
-    }
-
     @SuppressWarnings("unchecked")
     public void testValidSyntheticSource() throws IOException {
         CheckedConsumer<XContentBuilder, IOException> mapping = b -> {
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java
index d87d8dbc2bb4e..54c2d93ab73fa 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java
@@ -236,14 +236,14 @@ private void assertNullBounds(CheckedConsumer<XContentBuilder, IOException> toCh
         }
     }
 
-    protected static String storedValue(ParsedDocument doc) {
+    private static String storedValue(ParsedDocument doc) {
         assertEquals(3, doc.rootDoc().getFields("field").size());
         List<IndexableField> fields = doc.rootDoc().getFields("field");
         IndexableField storedField = fields.get(2);
         return storedField.stringValue();
     }
 
-    public void testNullBounds() throws IOException {
+    public final void testNullBounds() throws IOException {
 
         // null, null => min, max
         assertNullBounds(b -> b.startObject("field").nullField("gte").nullField("lte").endObject(), true, true);

From e1d902d33ba3773123d651e2f60b51b0e9e95670 Mon Sep 17 00:00:00 2001
From: Oleksandr Kolomiiets <oleksandr.kolomiiets@elastic.co>
Date: Thu, 25 Apr 2024 10:31:27 -0700
Subject: [PATCH 35/58] Implement synthetic source support for annotated text
 field (#107735)

This PR adds synthetic source support for annotated_text fields. Existing implementation for text is reused including test infrastructure so the majority of the change is moving and making things accessible.

Contributes to #106460, #78744.
---
 docs/changelog/107735.yaml                    |   5 +
 docs/plugins/mapper-annotated-text.asciidoc   | 161 +++++++++++---
 .../mapping/fields/synthetic-source.asciidoc  |   1 +
 .../src/main/java/module-info.java            |  19 ++
 .../AnnotatedTextFieldMapper.java             |  52 ++++-
 .../index/mapper/annotatedtext/Features.java  |  26 +++
 ...lasticsearch.features.FeatureSpecification |   9 +
 .../AnnotatedTextFieldMapperTests.java        |  25 ++-
 .../20_synthetic_source.yml                   | 197 +++++++++++++++++
 .../index/mapper/KeywordFieldMapper.java      |   2 +-
 .../index/mapper/TextFieldMapper.java         |  46 ++--
 .../index/mapper/KeywordFieldMapperTests.java | 110 +---------
 .../index/mapper/TextFieldMapperTests.java    | 136 +-----------
 .../KeywordFieldSyntheticSourceSupport.java   | 126 +++++++++++
 .../index/mapper/MapperTestCase.java          |   2 +-
 ...xtFieldFamilySyntheticSourceTestSetup.java | 207 ++++++++++++++++++
 16 files changed, 824 insertions(+), 300 deletions(-)
 create mode 100644 docs/changelog/107735.yaml
 create mode 100644 plugins/mapper-annotated-text/src/main/java/module-info.java
 create mode 100644 plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/Features.java
 create mode 100644 plugins/mapper-annotated-text/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification
 create mode 100644 plugins/mapper-annotated-text/src/yamlRestTest/resources/rest-api-spec/test/mapper_annotatedtext/20_synthetic_source.yml
 create mode 100644 test/framework/src/main/java/org/elasticsearch/index/mapper/KeywordFieldSyntheticSourceSupport.java
 create mode 100644 test/framework/src/main/java/org/elasticsearch/index/mapper/TextFieldFamilySyntheticSourceTestSetup.java

diff --git a/docs/changelog/107735.yaml b/docs/changelog/107735.yaml
new file mode 100644
index 0000000000000..372cb59ba8b1f
--- /dev/null
+++ b/docs/changelog/107735.yaml
@@ -0,0 +1,5 @@
+pr: 107735
+summary: Implement synthetic source support for annotated text field
+area: Mapping
+type: feature
+issues: []
diff --git a/docs/plugins/mapper-annotated-text.asciidoc b/docs/plugins/mapper-annotated-text.asciidoc
index 14669d857817c..900eaa5e97a04 100644
--- a/docs/plugins/mapper-annotated-text.asciidoc
+++ b/docs/plugins/mapper-annotated-text.asciidoc
@@ -6,7 +6,7 @@ experimental[]
 The mapper-annotated-text plugin provides the ability to index text that is a
 combination of free-text and special markup that is typically used to identify
 items of interest such as people or organisations (see NER or Named Entity Recognition
-tools). 
+tools).
 
 
 The elasticsearch markup allows one or more additional tokens to be injected, unchanged, into the token
@@ -18,7 +18,7 @@ include::install_remove.asciidoc[]
 [[mapper-annotated-text-usage]]
 ==== Using the `annotated-text` field
 
-The `annotated-text` tokenizes text content as per the more common {ref}/text.html[`text`] field (see 
+The `annotated-text` tokenizes text content as per the more common {ref}/text.html[`text`] field (see
 "limitations" below) but also injects any marked-up annotation tokens directly into
 the search index:
 
@@ -49,7 +49,7 @@ in the search index:
 --------------------------
 GET my-index-000001/_analyze
 {
-  "field": "my_field", 
+  "field": "my_field",
   "text":"Investors in [Apple](Apple+Inc.) rejoiced."
 }
 --------------------------
@@ -76,7 +76,7 @@ Response:
       "position": 1
     },
     {
-      "token": "Apple Inc.", <1> 
+      "token": "Apple Inc.", <1>
       "start_offset": 13,
       "end_offset": 18,
       "type": "annotation",
@@ -106,7 +106,7 @@ the token stream and at the same position (position 2) as the text token (`apple
 
 
 We can now perform searches for annotations using regular `term` queries that don't tokenize
-the provided search values. Annotations are a more precise way of matching as can be seen 
+the provided search values. Annotations are a more precise way of matching as can be seen
 in this example where a search for `Beck` will not match `Jeff Beck` :
 
 [source,console]
@@ -133,18 +133,119 @@ GET my-index-000001/_search
 }
 --------------------------
 
-<1> As well as tokenising the plain text into single words e.g. `beck`, here we 
+<1> As well as tokenising the plain text into single words e.g. `beck`, here we
 inject the single token value `Beck` at the same position as `beck` in the token stream.
 <2> Note annotations can inject multiple tokens at the same position - here we inject both
 the very specific value `Jeff Beck` and the broader term `Guitarist`. This enables
 broader positional queries e.g. finding mentions of a `Guitarist` near to `strat`.
-<3> A benefit of searching with these carefully defined annotation tokens is that a query for 
+<3> A benefit of searching with these carefully defined annotation tokens is that a query for
 `Beck` will not match document 2 that contains the tokens `jeff`, `beck` and `Jeff Beck`
 
-WARNING: Any use of `=` signs in annotation values eg `[Prince](person=Prince)` will 
+WARNING: Any use of `=` signs in annotation values eg `[Prince](person=Prince)` will
 cause the document to be rejected with a parse failure. In future we hope to have a use for
 the equals signs so wil actively reject documents that contain this today.
 
+[[annotated-text-synthetic-source]]
+===== Synthetic `_source`
+
+IMPORTANT: Synthetic `_source` is Generally Available only for TSDB indices
+(indices that have `index.mode` set to `time_series`). For other indices
+synthetic `_source` is in technical preview. Features in technical preview may
+be changed or removed in a future release. Elastic will work to fix
+any issues, but features in technical preview are not subject to the support SLA
+of official GA features.
+
+`annotated_text` fields support {ref}/mapping-source-field.html#synthetic-source[synthetic `_source`] if they have
+a {ref}/keyword.html#keyword-synthetic-source[`keyword`] sub-field that supports synthetic
+`_source` or if the `text` field sets `store` to `true`. Either way, it may
+not have {ref}/copy-to.html[`copy_to`].
+
+If using a sub-`keyword` field then the values are sorted in the same way as
+a `keyword` field's values are sorted. By default, that means sorted with
+duplicates removed. So:
+[source,console,id=synthetic-source-text-example-default]
+----
+PUT idx
+{
+  "mappings": {
+    "_source": { "mode": "synthetic" },
+    "properties": {
+      "text": {
+        "type": "annotated_text",
+        "fields": {
+          "raw": {
+            "type": "keyword"
+          }
+        }
+      }
+    }
+  }
+}
+PUT idx/_doc/1
+{
+  "text": [
+    "the quick brown fox",
+    "the quick brown fox",
+    "jumped over the lazy dog"
+  ]
+}
+----
+// TEST[s/$/\nGET idx\/_doc\/1?filter_path=_source\n/]
+
+Will become:
+[source,console-result]
+----
+{
+  "text": [
+    "jumped over the lazy dog",
+    "the quick brown fox"
+  ]
+}
+----
+// TEST[s/^/{"_source":/ s/\n$/}/]
+
+NOTE: Reordering text fields can have an effect on {ref}/query-dsl-match-query-phrase.html[phrase]
+and {ref}/span-queries.html[span] queries. See the discussion about {ref}/position-increment-gap.html[`position_increment_gap`] for more detail. You
+can avoid this by making sure the `slop` parameter on the phrase queries
+is lower than the `position_increment_gap`. This is the default.
+
+If the `annotated_text` field sets `store` to true then order and duplicates
+are preserved.
+[source,console,id=synthetic-source-text-example-stored]
+----
+PUT idx
+{
+  "mappings": {
+    "_source": { "mode": "synthetic" },
+    "properties": {
+      "text": { "type": "annotated_text", "store": true }
+    }
+  }
+}
+PUT idx/_doc/1
+{
+  "text": [
+    "the quick brown fox",
+    "the quick brown fox",
+    "jumped over the lazy dog"
+  ]
+}
+----
+// TEST[s/$/\nGET idx\/_doc\/1?filter_path=_source\n/]
+
+Will become:
+[source,console-result]
+----
+{
+  "text": [
+    "the quick brown fox",
+    "the quick brown fox",
+    "jumped over the lazy dog"
+  ]
+}
+----
+// TEST[s/^/{"_source":/ s/\n$/}/]
+
 
 [[mapper-annotated-text-tips]]
 ==== Data modelling tips
@@ -153,13 +254,13 @@ the equals signs so wil actively reject documents that contain this today.
 Annotations are normally a way of weaving structured information into unstructured text for
 higher-precision search.
 
-`Entity resolution` is a form of document enrichment undertaken by specialist software or people 
+`Entity resolution` is a form of document enrichment undertaken by specialist software or people
 where references to entities in a document are disambiguated by attaching a canonical ID.
 The ID is used to resolve any number of aliases or distinguish between people with the
-same name. The hyperlinks connecting Wikipedia's articles are a good example of resolved 
-entity IDs woven into text. 
+same name. The hyperlinks connecting Wikipedia's articles are a good example of resolved
+entity IDs woven into text.
 
-These IDs can be embedded as annotations in an annotated_text field but it often makes 
+These IDs can be embedded as annotations in an annotated_text field but it often makes
 sense to include them in dedicated structured fields to support discovery via aggregations:
 
 [source,console]
@@ -214,20 +315,20 @@ GET my-index-000001/_search
 --------------------------
 
 <1> Note the `my_twitter_handles` contains a list of the annotation values
-also used in the unstructured text. (Note the annotated_text syntax requires escaping). 
-By repeating the annotation values in a structured field this application has ensured that 
-the tokens discovered in the structured field can be used for search and highlighting 
-in the unstructured field.  
+also used in the unstructured text. (Note the annotated_text syntax requires escaping).
+By repeating the annotation values in a structured field this application has ensured that
+the tokens discovered in the structured field can be used for search and highlighting
+in the unstructured field.
 <2> In this example we search for documents that talk about components of the elastic stack
 <3> We use the `my_twitter_handles` field here to discover people who are significantly
 associated with the elastic stack.
 
 ===== Avoiding over-matching annotations
-By design, the regular text tokens and the annotation tokens co-exist in the same indexed 
+By design, the regular text tokens and the annotation tokens co-exist in the same indexed
 field but in rare cases this can lead to some over-matching.
 
 The value of an annotation often denotes a _named entity_ (a person, place or company).
-The tokens for these named entities are inserted untokenized, and differ from typical text 
+The tokens for these named entities are inserted untokenized, and differ from typical text
 tokens because they are normally:
 
 * Mixed case e.g. `Madonna`
@@ -235,19 +336,19 @@ tokens because they are normally:
 * Can have punctuation or numbers e.g. `Apple Inc.` or `@kimchy`
 
 This means, for the most part, a search for a named entity in the annotated text field will
-not have any false positives e.g. when selecting `Apple Inc.` from an aggregation result 
-you can drill down to highlight uses in the text without "over matching" on any text tokens 
+not have any false positives e.g. when selecting `Apple Inc.` from an aggregation result
+you can drill down to highlight uses in the text without "over matching" on any text tokens
 like the word `apple` in this context:
 
     the apple was very juicy
-    
-However, a problem arises if your named entity happens to be a single term and lower-case e.g. the 
+
+However, a problem arises if your named entity happens to be a single term and lower-case e.g. the
 company `elastic`. In this case, a search on the annotated text field for the token `elastic`
 may match a text document such as this:
 
     they fired an elastic band
 
-To avoid such false matches users should consider prefixing annotation values to ensure 
+To avoid such false matches users should consider prefixing annotation values to ensure
 they don't name clash with text tokens e.g.
 
     [elastic](Company_elastic) released version 7.0 of the elastic stack today
@@ -273,7 +374,7 @@ GET my-index-000001/_search
 {
   "query": {
     "query_string": {
-        "query": "cats" 
+        "query": "cats"
     }
   },
   "highlight": {
@@ -291,21 +392,21 @@ GET my-index-000001/_search
 
 The annotated highlighter is based on the `unified` highlighter and supports the same
 settings but does not use the `pre_tags` or `post_tags` parameters. Rather than using
-html-like markup such as `<em>cat</em>` the annotated highlighter uses the same 
+html-like markup such as `<em>cat</em>` the annotated highlighter uses the same
 markdown-like syntax used for annotations and injects a key=value annotation where `_hit_term`
-is the key and the matched search term is the value e.g. 
+is the key and the matched search term is the value e.g.
 
     The [cat](_hit_term=cat) sat on the [mat](sku3578)
 
-The annotated highlighter tries to be respectful of any existing markup in the original 
+The annotated highlighter tries to be respectful of any existing markup in the original
 text:
 
-* If the search term matches exactly the location of an existing annotation then the 
+* If the search term matches exactly the location of an existing annotation then the
 `_hit_term` key is merged into the url-like syntax used in the `(...)` part of the
-existing annotation. 
+existing annotation.
 * However, if the search term overlaps the span of an existing annotation it would break
 the markup formatting so the original annotation is removed in favour of a new annotation
-with just the search hit information in the results. 
+with just the search hit information in the results.
 * Any non-overlapping annotations in the original text are preserved in highlighter
 selections
 
diff --git a/docs/reference/mapping/fields/synthetic-source.asciidoc b/docs/reference/mapping/fields/synthetic-source.asciidoc
index ec6f51f78eda5..21e98cd55bf3a 100644
--- a/docs/reference/mapping/fields/synthetic-source.asciidoc
+++ b/docs/reference/mapping/fields/synthetic-source.asciidoc
@@ -41,6 +41,7 @@ There are a couple of restrictions to be aware of:
 types:
 
 ** <<aggregate-metric-double-synthetic-source, `aggregate_metric_double`>>
+** {plugins}/mapper-annotated-text-usage.html#annotated-text-synthetic-source[`annotated-text`]
 ** <<binary-synthetic-source,`binary`>>
 ** <<boolean-synthetic-source,`boolean`>>
 ** <<numeric-synthetic-source,`byte`>>
diff --git a/plugins/mapper-annotated-text/src/main/java/module-info.java b/plugins/mapper-annotated-text/src/main/java/module-info.java
new file mode 100644
index 0000000000000..3aa8e46e2980c
--- /dev/null
+++ b/plugins/mapper-annotated-text/src/main/java/module-info.java
@@ -0,0 +1,19 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+module org.elasticsearch.index.mapper.annotatedtext {
+    requires org.elasticsearch.base;
+    requires org.elasticsearch.server;
+    requires org.elasticsearch.xcontent;
+    requires org.apache.lucene.core;
+    requires org.apache.lucene.highlighter;
+
+    // exports nothing
+
+    provides org.elasticsearch.features.FeatureSpecification with org.elasticsearch.index.mapper.annotatedtext.Features;
+}
diff --git a/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java b/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java
index fae2ab19aee39..6d2b83185d5b7 100644
--- a/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java
+++ b/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java
@@ -21,17 +21,22 @@
 import org.apache.lucene.document.FieldType;
 import org.apache.lucene.index.IndexOptions;
 import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.analysis.AnalyzerScope;
 import org.elasticsearch.index.analysis.IndexAnalyzers;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.mapper.DocumentParserContext;
 import org.elasticsearch.index.mapper.FieldMapper;
+import org.elasticsearch.index.mapper.KeywordFieldMapper;
 import org.elasticsearch.index.mapper.MapperBuilderContext;
+import org.elasticsearch.index.mapper.SourceLoader;
+import org.elasticsearch.index.mapper.StringStoredFieldFieldLoader;
 import org.elasticsearch.index.mapper.TextFieldMapper;
 import org.elasticsearch.index.mapper.TextParams;
 import org.elasticsearch.index.mapper.TextSearchInfo;
 import org.elasticsearch.index.similarity.SimilarityProvider;
+import org.elasticsearch.xcontent.XContentBuilder;
 
 import java.io.IOException;
 import java.io.Reader;
@@ -41,6 +46,7 @@
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
@@ -58,6 +64,8 @@
  **/
 public class AnnotatedTextFieldMapper extends FieldMapper {
 
+    public static final NodeFeature SYNTHETIC_SOURCE_SUPPORT = new NodeFeature("mapper.annotated_text.synthetic_source");
+
     public static final String CONTENT_TYPE = "annotated_text";
 
     private static Builder builder(FieldMapper in) {
@@ -114,7 +122,7 @@ protected Parameter<?>[] getParameters() {
                 meta };
         }
 
-        private AnnotatedTextFieldType buildFieldType(FieldType fieldType, MapperBuilderContext context) {
+        private AnnotatedTextFieldType buildFieldType(FieldType fieldType, MapperBuilderContext context, MultiFields multiFields) {
             TextSearchInfo tsi = new TextSearchInfo(
                 fieldType,
                 similarity.get(),
@@ -126,12 +134,14 @@ private AnnotatedTextFieldType buildFieldType(FieldType fieldType, MapperBuilder
                 store.getValue(),
                 tsi,
                 context.isSourceSynthetic(),
+                TextFieldMapper.SyntheticSourceHelper.syntheticSourceDelegate(fieldType, multiFields),
                 meta.getValue()
             );
         }
 
         @Override
         public AnnotatedTextFieldMapper build(MapperBuilderContext context) {
+            MultiFields multiFields = multiFieldsBuilder.build(this, context);
             FieldType fieldType = TextParams.buildFieldType(() -> true, store, indexOptions, norms, termVectors);
             if (fieldType.indexOptions() == IndexOptions.NONE) {
                 throw new IllegalArgumentException("[" + CONTENT_TYPE + "] fields must be indexed");
@@ -146,8 +156,8 @@ public AnnotatedTextFieldMapper build(MapperBuilderContext context) {
             return new AnnotatedTextFieldMapper(
                 name(),
                 fieldType,
-                buildFieldType(fieldType, context),
-                multiFieldsBuilder.build(this, context),
+                buildFieldType(fieldType, context, multiFields),
+                multiFields,
                 copyTo,
                 this
             );
@@ -472,15 +482,15 @@ private void emitAnnotation(int firstSpannedTextPosInc, int annotationPosLen) th
     }
 
     public static final class AnnotatedTextFieldType extends TextFieldMapper.TextFieldType {
-
         private AnnotatedTextFieldType(
             String name,
             boolean store,
             TextSearchInfo tsi,
             boolean isSyntheticSource,
+            KeywordFieldMapper.KeywordFieldType syntheticSourceDelegate,
             Map<String, String> meta
         ) {
-            super(name, true, store, tsi, isSyntheticSource, null, meta, false, false);
+            super(name, true, store, tsi, isSyntheticSource, syntheticSourceDelegate, meta, false, false);
         }
 
         public AnnotatedTextFieldType(String name, Map<String, String> meta) {
@@ -544,4 +554,36 @@ protected String contentType() {
     public FieldMapper.Builder getMergeBuilder() {
         return new Builder(simpleName(), builder.indexCreatedVersion, builder.analyzers.indexAnalyzers).init(this);
     }
+
+    @Override
+    public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
+        if (copyTo.copyToFields().isEmpty() != true) {
+            throw new IllegalArgumentException(
+                "field [" + name() + "] of type [" + typeName() + "] doesn't support synthetic source because it declares copy_to"
+            );
+        }
+        if (fieldType.stored()) {
+            return new StringStoredFieldFieldLoader(name(), simpleName(), null) {
+                @Override
+                protected void write(XContentBuilder b, Object value) throws IOException {
+                    b.value((String) value);
+                }
+            };
+        }
+
+        var kwd = TextFieldMapper.SyntheticSourceHelper.getKeywordFieldMapperForSyntheticSource(this);
+        if (kwd != null) {
+            return kwd.syntheticFieldLoader(simpleName());
+        }
+
+        throw new IllegalArgumentException(
+            String.format(
+                Locale.ROOT,
+                "field [%s] of type [%s] doesn't support synthetic source unless it is stored or has a sub-field of"
+                    + " type [keyword] with doc values or stored and without a normalizer",
+                name(),
+                typeName()
+            )
+        );
+    }
 }
diff --git a/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/Features.java b/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/Features.java
new file mode 100644
index 0000000000000..1c4bd22e88145
--- /dev/null
+++ b/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/Features.java
@@ -0,0 +1,26 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.index.mapper.annotatedtext;
+
+import org.elasticsearch.features.FeatureSpecification;
+import org.elasticsearch.features.NodeFeature;
+
+import java.util.Set;
+
+/**
+ * Provides features for annotated text mapper.
+ */
+public class Features implements FeatureSpecification {
+    @Override
+    public Set<NodeFeature> getFeatures() {
+        return Set.of(
+            AnnotatedTextFieldMapper.SYNTHETIC_SOURCE_SUPPORT // Added in 8.15
+        );
+    }
+}
diff --git a/plugins/mapper-annotated-text/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification b/plugins/mapper-annotated-text/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification
new file mode 100644
index 0000000000000..a19d9deb9c522
--- /dev/null
+++ b/plugins/mapper-annotated-text/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification
@@ -0,0 +1,9 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License
+# 2.0 and the Server Side Public License, v 1; you may not use this file except
+# in compliance with, at your election, the Elastic License 2.0 or the Server
+# Side Public License, v 1.
+#
+
+org.elasticsearch.index.mapper.annotatedtext.Features
diff --git a/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java b/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java
index 9f1d063433d88..3b27cdb132851 100644
--- a/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java
+++ b/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java
@@ -14,6 +14,7 @@
 import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
 import org.apache.lucene.analysis.en.EnglishAnalyzer;
 import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.DocValuesType;
 import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.IndexableField;
@@ -29,6 +30,7 @@
 import org.elasticsearch.index.analysis.CharFilterFactory;
 import org.elasticsearch.index.analysis.CustomAnalyzer;
 import org.elasticsearch.index.analysis.IndexAnalyzers;
+import org.elasticsearch.index.analysis.LowercaseNormalizer;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.analysis.StandardTokenizerFactory;
 import org.elasticsearch.index.analysis.TokenFilterFactory;
@@ -38,6 +40,7 @@
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.MapperTestCase;
 import org.elasticsearch.index.mapper.ParsedDocument;
+import org.elasticsearch.index.mapper.TextFieldFamilySyntheticSourceTestSetup;
 import org.elasticsearch.index.mapper.TextFieldMapper;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.xcontent.ToXContent;
@@ -54,6 +57,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.function.Function;
 
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
@@ -144,7 +148,8 @@ public TokenStream create(TokenStream tokenStream) {
             )
         );
         return IndexAnalyzers.of(
-            Map.of("default", dflt, "standard", standard, "keyword", keyword, "whitespace", whitespace, "my_stop_analyzer", stop)
+            Map.of("default", dflt, "standard", standard, "keyword", keyword, "whitespace", whitespace, "my_stop_analyzer", stop),
+            Map.of("lowercase", new NamedAnalyzer("lowercase", AnalyzerScope.INDEX, new LowercaseNormalizer()))
         );
     }
 
@@ -595,7 +600,23 @@ protected boolean supportsIgnoreMalformed() {
 
     @Override
     protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed) {
-        throw new AssumptionViolatedException("not supported");
+        assumeFalse("ignore_malformed not supported", ignoreMalformed);
+        return TextFieldFamilySyntheticSourceTestSetup.syntheticSourceSupport("annotated_text", false);
+    }
+
+    @Override
+    protected BlockReaderSupport getSupportedReaders(MapperService mapper, String loaderFieldName) {
+        return TextFieldFamilySyntheticSourceTestSetup.getSupportedReaders(mapper, loaderFieldName);
+    }
+
+    @Override
+    protected Function<Object, Object> loadBlockExpected(BlockReaderSupport blockReaderSupport, boolean columnReader) {
+        return TextFieldFamilySyntheticSourceTestSetup.loadBlockExpected(blockReaderSupport, columnReader);
+    }
+
+    @Override
+    protected void validateRoundTripReader(String syntheticSource, DirectoryReader reader, DirectoryReader roundTripReader) {
+        TextFieldFamilySyntheticSourceTestSetup.validateRoundTripReader(syntheticSource, reader, roundTripReader);
     }
 
     @Override
diff --git a/plugins/mapper-annotated-text/src/yamlRestTest/resources/rest-api-spec/test/mapper_annotatedtext/20_synthetic_source.yml b/plugins/mapper-annotated-text/src/yamlRestTest/resources/rest-api-spec/test/mapper_annotatedtext/20_synthetic_source.yml
new file mode 100644
index 0000000000000..54a51e60f56df
--- /dev/null
+++ b/plugins/mapper-annotated-text/src/yamlRestTest/resources/rest-api-spec/test/mapper_annotatedtext/20_synthetic_source.yml
@@ -0,0 +1,197 @@
+---
+setup:
+  - requires:
+      cluster_features: ["mapper.annotated_text.synthetic_source"]
+      reason: introduced in 8.15.0
+
+---
+stored annotated_text field:
+  - do:
+      indices.create:
+        index: test
+        body:
+          mappings:
+            _source:
+              mode: synthetic
+            properties:
+              annotated_text:
+                type: annotated_text
+                store: true
+
+  - do:
+      index:
+        index: test
+        id: 1
+        refresh: true
+        body:
+          annotated_text: the quick brown fox
+
+  - do:
+      search:
+        index: test
+
+  - match:
+      hits.hits.0._source:
+        annotated_text: the quick brown fox
+
+---
+annotated_text field with keyword multi-field:
+  - do:
+      indices.create:
+        index: test
+        body:
+          mappings:
+            _source:
+              mode: synthetic
+            properties:
+              annotated_text:
+                type: annotated_text
+                fields:
+                  keyword:
+                    type: keyword
+
+  - do:
+      index:
+        index: test
+        id: 1
+        refresh: true
+        body:
+          annotated_text: the quick brown fox
+
+  - do:
+      search:
+        index: test
+
+  - match:
+      hits.hits.0._source:
+        annotated_text: the quick brown fox
+
+---
+multiple values in stored annotated_text field:
+  - do:
+      indices.create:
+        index: test
+        body:
+          mappings:
+            _source:
+              mode: synthetic
+            properties:
+              annotated_text:
+                type: annotated_text
+                store: true
+
+  - do:
+      index:
+        index: test
+        id: 1
+        refresh: true
+        body:
+          annotated_text: ["world", "hello", "world"]
+
+  - do:
+      search:
+        index: test
+
+  - match:
+      hits.hits.0._source:
+        annotated_text: ["world", "hello", "world"]
+
+---
+multiple values in annotated_text field with keyword multi-field:
+  - do:
+      indices.create:
+        index: test
+        body:
+          mappings:
+            _source:
+              mode: synthetic
+            properties:
+              annotated_text:
+                type: annotated_text
+                fields:
+                  keyword:
+                    type: keyword
+
+  - do:
+      index:
+        index: test
+        id: 1
+        refresh: true
+        body:
+          annotated_text: ["world", "hello", "world"]
+
+  - do:
+      search:
+        index: test
+
+  - match:
+      hits.hits.0._source:
+        annotated_text: ["hello", "world"]
+
+
+---
+multiple values in annotated_text field with stored keyword multi-field:
+  - do:
+      indices.create:
+        index: test
+        body:
+          mappings:
+            _source:
+              mode: synthetic
+            properties:
+              annotated_text:
+                type: annotated_text
+                fields:
+                  keyword:
+                    type: keyword
+                    store: true
+                    doc_values: false
+
+  - do:
+      index:
+        index: test
+        id: 1
+        refresh: true
+        body:
+          annotated_text: ["world", "hello", "world"]
+
+  - do:
+      search:
+        index: test
+
+  - match:
+      hits.hits.0._source:
+        annotated_text: ["world", "hello", "world"]
+
+---
+multiple values in stored annotated_text field with keyword multi-field:
+  - do:
+      indices.create:
+        index: test
+        body:
+          mappings:
+            _source:
+              mode: synthetic
+            properties:
+              annotated_text:
+                type: annotated_text
+                store: true
+                fields:
+                  keyword:
+                    type: keyword
+
+  - do:
+      index:
+        index: test
+        id: 1
+        refresh: true
+        body:
+          annotated_text: ["world", "hello", "world"]
+
+  - do:
+      search:
+        index: test
+
+  - match:
+      hits.hits.0._source:
+        annotated_text: ["world", "hello", "world"]
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java
index bdf25307d3343..eeb452204091d 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java
@@ -1026,7 +1026,7 @@ public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
         return syntheticFieldLoader(simpleName());
     }
 
-    SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String simpleName) {
+    public SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String simpleName) {
         if (hasScript()) {
             return SourceLoader.SyntheticFieldLoader.NOTHING;
         }
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java
index ef512e2bbd46b..57dd2fa0b920d 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java
@@ -390,7 +390,7 @@ private TextFieldType buildFieldType(
                     store.getValue(),
                     tsi,
                     context.isSourceSynthetic(),
-                    syntheticSourceDelegate(fieldType, multiFields),
+                    SyntheticSourceHelper.syntheticSourceDelegate(fieldType, multiFields),
                     meta.getValue(),
                     eagerGlobalOrdinals.getValue(),
                     indexPhrases.getValue()
@@ -402,17 +402,6 @@ private TextFieldType buildFieldType(
             return ft;
         }
 
-        private static KeywordFieldMapper.KeywordFieldType syntheticSourceDelegate(FieldType fieldType, MultiFields multiFields) {
-            if (fieldType.stored()) {
-                return null;
-            }
-            var kwd = getKeywordFieldMapperForSyntheticSource(multiFields);
-            if (kwd != null) {
-                return kwd.fieldType();
-            }
-            return null;
-        }
-
         private SubFieldInfo buildPrefixInfo(MapperBuilderContext context, FieldType fieldType, TextFieldType tft) {
             if (indexPrefixes.get() == null) {
                 return null;
@@ -1094,7 +1083,7 @@ public boolean isSyntheticSource() {
             return isSyntheticSource;
         }
 
-        KeywordFieldMapper.KeywordFieldType syntheticSourceDelegate() {
+        public KeywordFieldMapper.KeywordFieldType syntheticSourceDelegate() {
             return syntheticSourceDelegate;
         }
     }
@@ -1473,7 +1462,7 @@ protected void write(XContentBuilder b, Object value) throws IOException {
             };
         }
 
-        var kwd = getKeywordFieldMapperForSyntheticSource(this);
+        var kwd = SyntheticSourceHelper.getKeywordFieldMapperForSyntheticSource(this);
         if (kwd != null) {
             return kwd.syntheticFieldLoader(simpleName());
         }
@@ -1489,16 +1478,29 @@ protected void write(XContentBuilder b, Object value) throws IOException {
         );
     }
 
-    private static KeywordFieldMapper getKeywordFieldMapperForSyntheticSource(Iterable<? extends Mapper> multiFields) {
-        for (Mapper sub : multiFields) {
-            if (sub.typeName().equals(KeywordFieldMapper.CONTENT_TYPE)) {
-                KeywordFieldMapper kwd = (KeywordFieldMapper) sub;
-                if (kwd.hasNormalizer() == false && (kwd.fieldType().hasDocValues() || kwd.fieldType().isStored())) {
-                    return kwd;
-                }
+    public static class SyntheticSourceHelper {
+        public static KeywordFieldMapper.KeywordFieldType syntheticSourceDelegate(FieldType fieldType, MultiFields multiFields) {
+            if (fieldType.stored()) {
+                return null;
+            }
+            var kwd = getKeywordFieldMapperForSyntheticSource(multiFields);
+            if (kwd != null) {
+                return kwd.fieldType();
             }
+            return null;
         }
 
-        return null;
+        public static KeywordFieldMapper getKeywordFieldMapperForSyntheticSource(Iterable<? extends Mapper> multiFields) {
+            for (Mapper sub : multiFields) {
+                if (sub.typeName().equals(KeywordFieldMapper.CONTENT_TYPE)) {
+                    KeywordFieldMapper kwd = (KeywordFieldMapper) sub;
+                    if (kwd.hasNormalizer() == false && (kwd.fieldType().hasDocValues() || kwd.fieldType().isStored())) {
+                        return kwd;
+                    }
+                }
+            }
+
+            return null;
+        }
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java
index 70e375a89d5e7..4824bd337f5b0 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java
@@ -25,7 +25,6 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.core.Tuple;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.analysis.AnalyzerScope;
@@ -45,14 +44,11 @@
 import org.elasticsearch.xcontent.XContentBuilder;
 
 import java.io.IOException;
-import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.function.Function;
-import java.util.stream.Collectors;
 
 import static java.util.Collections.singletonList;
 import static java.util.Collections.singletonMap;
@@ -658,7 +654,7 @@ protected Function<Object, Object> loadBlockExpected() {
     @Override
     protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed) {
         assertFalse("keyword doesn't support ignore_malformed", ignoreMalformed);
-        return new KeywordSyntheticSourceSupport(
+        return new KeywordFieldSyntheticSourceSupport(
             randomBoolean() ? null : between(10, 100),
             randomBoolean(),
             usually() ? null : randomAlphaOfLength(2),
@@ -666,110 +662,6 @@ protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed)
         );
     }
 
-    static class KeywordSyntheticSourceSupport implements SyntheticSourceSupport {
-        private final Integer ignoreAbove;
-        private final boolean allIgnored;
-        private final boolean store;
-        private final boolean docValues;
-        private final String nullValue;
-        private final boolean exampleSortsUsingIgnoreAbove;
-
-        KeywordSyntheticSourceSupport(Integer ignoreAbove, boolean store, String nullValue, boolean exampleSortsUsingIgnoreAbove) {
-            this.ignoreAbove = ignoreAbove;
-            this.allIgnored = ignoreAbove != null && rarely();
-            this.store = store;
-            this.nullValue = nullValue;
-            this.exampleSortsUsingIgnoreAbove = exampleSortsUsingIgnoreAbove;
-            this.docValues = store ? randomBoolean() : true;
-        }
-
-        @Override
-        public SyntheticSourceExample example(int maxValues) {
-            return example(maxValues, false);
-        }
-
-        public SyntheticSourceExample example(int maxValues, boolean loadBlockFromSource) {
-            if (randomBoolean()) {
-                Tuple<String, String> v = generateValue();
-                Object loadBlock = v.v2();
-                if (loadBlockFromSource == false && ignoreAbove != null && v.v2().length() > ignoreAbove) {
-                    loadBlock = null;
-                }
-                return new SyntheticSourceExample(v.v1(), v.v2(), loadBlock, this::mapping);
-            }
-            List<Tuple<String, String>> values = randomList(1, maxValues, this::generateValue);
-            List<String> in = values.stream().map(Tuple::v1).toList();
-            List<String> outPrimary = new ArrayList<>();
-            List<String> outExtraValues = new ArrayList<>();
-            values.stream().map(Tuple::v2).forEach(v -> {
-                if (exampleSortsUsingIgnoreAbove && ignoreAbove != null && v.length() > ignoreAbove) {
-                    outExtraValues.add(v);
-                } else {
-                    outPrimary.add(v);
-                }
-            });
-            List<String> outList = store ? outPrimary : new HashSet<>(outPrimary).stream().sorted().collect(Collectors.toList());
-            List<String> loadBlock;
-            if (loadBlockFromSource) {
-                // The block loader infrastructure will never return nulls. Just zap them all.
-                loadBlock = in.stream().filter(m -> m != null).toList();
-            } else if (docValues) {
-                loadBlock = new HashSet<>(outPrimary).stream().sorted().collect(Collectors.toList());
-            } else {
-                loadBlock = List.copyOf(outList);
-            }
-            Object loadBlockResult = loadBlock.size() == 1 ? loadBlock.get(0) : loadBlock;
-            outList.addAll(outExtraValues);
-            Object out = outList.size() == 1 ? outList.get(0) : outList;
-            return new SyntheticSourceExample(in, out, loadBlockResult, this::mapping);
-        }
-
-        private Tuple<String, String> generateValue() {
-            if (nullValue != null && randomBoolean()) {
-                return Tuple.tuple(null, nullValue);
-            }
-            int length = 5;
-            if (ignoreAbove != null && (allIgnored || randomBoolean())) {
-                length = ignoreAbove + 5;
-            }
-            String v = randomAlphaOfLength(length);
-            return Tuple.tuple(v, v);
-        }
-
-        private void mapping(XContentBuilder b) throws IOException {
-            b.field("type", "keyword");
-            if (nullValue != null) {
-                b.field("null_value", nullValue);
-            }
-            if (ignoreAbove != null) {
-                b.field("ignore_above", ignoreAbove);
-            }
-            if (store) {
-                b.field("store", true);
-            }
-            if (docValues == false) {
-                b.field("doc_values", false);
-            }
-        }
-
-        @Override
-        public List<SyntheticSourceInvalidExample> invalidExample() throws IOException {
-            return List.of(
-                new SyntheticSourceInvalidExample(
-                    equalTo(
-                        "field [field] of type [keyword] doesn't support synthetic source because "
-                            + "it doesn't have doc values and isn't stored"
-                    ),
-                    b -> b.field("type", "keyword").field("doc_values", false)
-                ),
-                new SyntheticSourceInvalidExample(
-                    equalTo("field [field] of type [keyword] doesn't support synthetic source because it declares a normalizer"),
-                    b -> b.field("type", "keyword").field("normalizer", "lowercase")
-                )
-            );
-        }
-    }
-
     @Override
     protected IngestScriptSupport ingestScriptSupport() {
         return new IngestScriptSupport() {
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java
index 5d0c1c01ecdcf..50d15be2256ed 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java
@@ -75,7 +75,6 @@
 import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentFactory;
-import org.hamcrest.Matcher;
 import org.junit.AssumptionViolatedException;
 
 import java.io.IOException;
@@ -1178,120 +1177,12 @@ protected boolean supportsIgnoreMalformed() {
     @Override
     protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed) {
         assumeFalse("ignore_malformed not supported", ignoreMalformed);
-        boolean storeTextField = randomBoolean();
-        boolean storedKeywordField = storeTextField || randomBoolean();
-        boolean indexText = randomBoolean();
-        Integer ignoreAbove = randomBoolean() ? null : between(10, 100);
-        KeywordFieldMapperTests.KeywordSyntheticSourceSupport keywordSupport = new KeywordFieldMapperTests.KeywordSyntheticSourceSupport(
-            ignoreAbove,
-            storedKeywordField,
-            null,
-            false == storeTextField
-        );
-        return new SyntheticSourceSupport() {
-            @Override
-            public SyntheticSourceExample example(int maxValues) {
-                if (storeTextField) {
-                    SyntheticSourceExample delegate = keywordSupport.example(maxValues, true);
-                    return new SyntheticSourceExample(
-                        delegate.inputValue(),
-                        delegate.expectedForSyntheticSource(),
-                        delegate.expectedForBlockLoader(),
-                        b -> {
-                            b.field("type", "text");
-                            b.field("store", true);
-                            if (indexText == false) {
-                                b.field("index", false);
-                            }
-                        }
-                    );
-                }
-                // We'll load from _source if ignore_above is defined, otherwise we load from the keyword field.
-                boolean loadingFromSource = ignoreAbove != null;
-                SyntheticSourceExample delegate = keywordSupport.example(maxValues, loadingFromSource);
-                return new SyntheticSourceExample(
-                    delegate.inputValue(),
-                    delegate.expectedForSyntheticSource(),
-                    delegate.expectedForBlockLoader(),
-                    b -> {
-                        b.field("type", "text");
-                        if (indexText == false) {
-                            b.field("index", false);
-                        }
-                        b.startObject("fields");
-                        {
-                            b.startObject(randomAlphaOfLength(4));
-                            delegate.mapping().accept(b);
-                            b.endObject();
-                        }
-                        b.endObject();
-                    }
-                );
-            }
-
-            @Override
-            public List<SyntheticSourceInvalidExample> invalidExample() throws IOException {
-                Matcher<String> err = equalTo(
-                    "field [field] of type [text] doesn't support synthetic source unless it is stored or"
-                        + " has a sub-field of type [keyword] with doc values or stored and without a normalizer"
-                );
-                return List.of(
-                    new SyntheticSourceInvalidExample(err, TextFieldMapperTests.this::minimalMapping),
-                    new SyntheticSourceInvalidExample(err, b -> {
-                        b.field("type", "text");
-                        b.startObject("fields");
-                        {
-                            b.startObject("l");
-                            b.field("type", "long");
-                            b.endObject();
-                        }
-                        b.endObject();
-                    }),
-                    new SyntheticSourceInvalidExample(err, b -> {
-                        b.field("type", "text");
-                        b.startObject("fields");
-                        {
-                            b.startObject("kwd");
-                            b.field("type", "keyword");
-                            b.field("normalizer", "lowercase");
-                            b.endObject();
-                        }
-                        b.endObject();
-                    }),
-                    new SyntheticSourceInvalidExample(err, b -> {
-                        b.field("type", "text");
-                        b.startObject("fields");
-                        {
-                            b.startObject("kwd");
-                            b.field("type", "keyword");
-                            b.field("doc_values", "false");
-                            b.endObject();
-                        }
-                        b.endObject();
-                    })
-                );
-            }
-        };
+        return TextFieldFamilySyntheticSourceTestSetup.syntheticSourceSupport("text", true);
     }
 
     @Override
     protected Function<Object, Object> loadBlockExpected(BlockReaderSupport blockReaderSupport, boolean columnReader) {
-        if (nullLoaderExpected(blockReaderSupport.mapper(), blockReaderSupport.loaderFieldName())) {
-            return null;
-        }
-        return v -> ((BytesRef) v).utf8ToString();
-    }
-
-    private boolean nullLoaderExpected(MapperService mapper, String fieldName) {
-        MappedFieldType type = mapper.fieldType(fieldName);
-        if (type instanceof TextFieldType t) {
-            if (t.isSyntheticSource() == false || t.canUseSyntheticSourceDelegateForQuerying() || t.isStored()) {
-                return false;
-            }
-            String parentField = mapper.mappingLookup().parentField(fieldName);
-            return parentField == null || nullLoaderExpected(mapper, parentField);
-        }
-        return false;
+        return TextFieldFamilySyntheticSourceTestSetup.loadBlockExpected(blockReaderSupport, columnReader);
     }
 
     @Override
@@ -1300,9 +1191,8 @@ protected IngestScriptSupport ingestScriptSupport() {
     }
 
     @Override
-    protected void validateRoundTripReader(String syntheticSource, DirectoryReader reader, DirectoryReader roundTripReader)
-        throws IOException {
-        // Disabled because it currently fails
+    protected void validateRoundTripReader(String syntheticSource, DirectoryReader reader, DirectoryReader roundTripReader) {
+        TextFieldFamilySyntheticSourceTestSetup.validateRoundTripReader(syntheticSource, reader, roundTripReader);
     }
 
     public void testUnknownAnalyzerOnLegacyIndex() throws IOException {
@@ -1433,21 +1323,7 @@ public void testEmpty() throws Exception {
 
     @Override
     protected BlockReaderSupport getSupportedReaders(MapperService mapper, String loaderFieldName) {
-        MappedFieldType ft = mapper.fieldType(loaderFieldName);
-        String parentName = mapper.mappingLookup().parentField(ft.name());
-        if (parentName == null) {
-            TextFieldMapper.TextFieldType text = (TextFieldType) ft;
-            boolean supportsColumnAtATimeReader = text.syntheticSourceDelegate() != null
-                && text.syntheticSourceDelegate().hasDocValues()
-                && text.canUseSyntheticSourceDelegateForQuerying();
-            return new BlockReaderSupport(supportsColumnAtATimeReader, mapper, loaderFieldName);
-        }
-        MappedFieldType parent = mapper.fieldType(parentName);
-        if (false == parent.typeName().equals(KeywordFieldMapper.CONTENT_TYPE)) {
-            throw new UnsupportedOperationException();
-        }
-        KeywordFieldMapper.KeywordFieldType kwd = (KeywordFieldMapper.KeywordFieldType) parent;
-        return new BlockReaderSupport(kwd.hasDocValues(), mapper, loaderFieldName);
+        return TextFieldFamilySyntheticSourceTestSetup.getSupportedReaders(mapper, loaderFieldName);
     }
 
     public void testBlockLoaderFromParentColumnReader() throws IOException {
@@ -1460,7 +1336,7 @@ public void testBlockLoaderParentFromRowStrideReader() throws IOException {
 
     private void testBlockLoaderFromParent(boolean columnReader, boolean syntheticSource) throws IOException {
         boolean storeParent = randomBoolean();
-        KeywordFieldMapperTests.KeywordSyntheticSourceSupport kwdSupport = new KeywordFieldMapperTests.KeywordSyntheticSourceSupport(
+        KeywordFieldSyntheticSourceSupport kwdSupport = new KeywordFieldSyntheticSourceSupport(
             null,
             storeParent,
             null,
diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/KeywordFieldSyntheticSourceSupport.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/KeywordFieldSyntheticSourceSupport.java
new file mode 100644
index 0000000000000..53ecb75c18d9a
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/KeywordFieldSyntheticSourceSupport.java
@@ -0,0 +1,126 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.index.mapper;
+
+import org.apache.lucene.tests.util.LuceneTestCase;
+import org.elasticsearch.core.Tuple;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class KeywordFieldSyntheticSourceSupport implements MapperTestCase.SyntheticSourceSupport {
+    private final Integer ignoreAbove;
+    private final boolean allIgnored;
+    private final boolean store;
+    private final boolean docValues;
+    private final String nullValue;
+    private final boolean exampleSortsUsingIgnoreAbove;
+
+    KeywordFieldSyntheticSourceSupport(Integer ignoreAbove, boolean store, String nullValue, boolean exampleSortsUsingIgnoreAbove) {
+        this.ignoreAbove = ignoreAbove;
+        this.allIgnored = ignoreAbove != null && LuceneTestCase.rarely();
+        this.store = store;
+        this.nullValue = nullValue;
+        this.exampleSortsUsingIgnoreAbove = exampleSortsUsingIgnoreAbove;
+        this.docValues = store ? ESTestCase.randomBoolean() : true;
+    }
+
+    @Override
+    public MapperTestCase.SyntheticSourceExample example(int maxValues) {
+        return example(maxValues, false);
+    }
+
+    public MapperTestCase.SyntheticSourceExample example(int maxValues, boolean loadBlockFromSource) {
+        if (ESTestCase.randomBoolean()) {
+            Tuple<String, String> v = generateValue();
+            Object loadBlock = v.v2();
+            if (loadBlockFromSource == false && ignoreAbove != null && v.v2().length() > ignoreAbove) {
+                loadBlock = null;
+            }
+            return new MapperTestCase.SyntheticSourceExample(v.v1(), v.v2(), loadBlock, this::mapping);
+        }
+        List<Tuple<String, String>> values = ESTestCase.randomList(1, maxValues, this::generateValue);
+        List<String> in = values.stream().map(Tuple::v1).toList();
+        List<String> outPrimary = new ArrayList<>();
+        List<String> outExtraValues = new ArrayList<>();
+        values.stream().map(Tuple::v2).forEach(v -> {
+            if (exampleSortsUsingIgnoreAbove && ignoreAbove != null && v.length() > ignoreAbove) {
+                outExtraValues.add(v);
+            } else {
+                outPrimary.add(v);
+            }
+        });
+        List<String> outList = store ? outPrimary : new HashSet<>(outPrimary).stream().sorted().collect(Collectors.toList());
+        List<String> loadBlock;
+        if (loadBlockFromSource) {
+            // The block loader infrastructure will never return nulls. Just zap them all.
+            loadBlock = in.stream().filter(m -> m != null).toList();
+        } else if (docValues) {
+            loadBlock = new HashSet<>(outPrimary).stream().sorted().collect(Collectors.toList());
+        } else {
+            loadBlock = List.copyOf(outList);
+        }
+        Object loadBlockResult = loadBlock.size() == 1 ? loadBlock.get(0) : loadBlock;
+        outList.addAll(outExtraValues);
+        Object out = outList.size() == 1 ? outList.get(0) : outList;
+        return new MapperTestCase.SyntheticSourceExample(in, out, loadBlockResult, this::mapping);
+    }
+
+    private Tuple<String, String> generateValue() {
+        if (nullValue != null && ESTestCase.randomBoolean()) {
+            return Tuple.tuple(null, nullValue);
+        }
+        int length = 5;
+        if (ignoreAbove != null && (allIgnored || ESTestCase.randomBoolean())) {
+            length = ignoreAbove + 5;
+        }
+        String v = ESTestCase.randomAlphaOfLength(length);
+        return Tuple.tuple(v, v);
+    }
+
+    private void mapping(XContentBuilder b) throws IOException {
+        b.field("type", "keyword");
+        if (nullValue != null) {
+            b.field("null_value", nullValue);
+        }
+        if (ignoreAbove != null) {
+            b.field("ignore_above", ignoreAbove);
+        }
+        if (store) {
+            b.field("store", true);
+        }
+        if (docValues == false) {
+            b.field("doc_values", false);
+        }
+    }
+
+    @Override
+    public List<MapperTestCase.SyntheticSourceInvalidExample> invalidExample() throws IOException {
+        return List.of(
+            new MapperTestCase.SyntheticSourceInvalidExample(
+                equalTo(
+                    "field [field] of type [keyword] doesn't support synthetic source because "
+                        + "it doesn't have doc values and isn't stored"
+                ),
+                b -> b.field("type", "keyword").field("doc_values", false)
+            ),
+            new MapperTestCase.SyntheticSourceInvalidExample(
+                equalTo("field [field] of type [keyword] doesn't support synthetic source because it declares a normalizer"),
+                b -> b.field("type", "keyword").field("normalizer", "lowercase")
+            )
+        );
+    }
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java
index fa0f0e1b95f54..097c23b96bb76 100644
--- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java
@@ -1286,7 +1286,7 @@ protected BlockReaderSupport getSupportedReaders(MapperService mapper, String lo
      * @param loaderFieldName the field name to use for loading the field
      */
     public record BlockReaderSupport(boolean columnAtATimeReader, boolean syntheticSource, MapperService mapper, String loaderFieldName) {
-        BlockReaderSupport(boolean columnAtATimeReader, MapperService mapper, String loaderFieldName) {
+        public BlockReaderSupport(boolean columnAtATimeReader, MapperService mapper, String loaderFieldName) {
             this(columnAtATimeReader, true, mapper, loaderFieldName);
         }
 
diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/TextFieldFamilySyntheticSourceTestSetup.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/TextFieldFamilySyntheticSourceTestSetup.java
new file mode 100644
index 0000000000000..df4377adc3e35
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/TextFieldFamilySyntheticSourceTestSetup.java
@@ -0,0 +1,207 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.index.mapper;
+
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.util.BytesRef;
+import org.hamcrest.Matcher;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Locale;
+import java.util.function.Function;
+
+import static org.elasticsearch.test.ESTestCase.between;
+import static org.elasticsearch.test.ESTestCase.randomAlphaOfLength;
+import static org.elasticsearch.test.ESTestCase.randomBoolean;
+import static org.hamcrest.Matchers.equalTo;
+
+/**
+ * Provides functionality needed to test synthetic source support in text and text-like fields (e.g. "text", "annotated_text").
+ */
+public final class TextFieldFamilySyntheticSourceTestSetup {
+    public static MapperTestCase.SyntheticSourceSupport syntheticSourceSupport(String fieldType, boolean supportsCustomIndexConfiguration) {
+        return new TextFieldFamilySyntheticSourceSupport(fieldType, supportsCustomIndexConfiguration);
+    }
+
+    public static MapperTestCase.BlockReaderSupport getSupportedReaders(MapperService mapper, String loaderFieldName) {
+        MappedFieldType ft = mapper.fieldType(loaderFieldName);
+        String parentName = mapper.mappingLookup().parentField(ft.name());
+        if (parentName == null) {
+            TextFieldMapper.TextFieldType text = (TextFieldMapper.TextFieldType) ft;
+            boolean supportsColumnAtATimeReader = text.syntheticSourceDelegate() != null
+                && text.syntheticSourceDelegate().hasDocValues()
+                && text.canUseSyntheticSourceDelegateForQuerying();
+            return new MapperTestCase.BlockReaderSupport(supportsColumnAtATimeReader, mapper, loaderFieldName);
+        }
+        MappedFieldType parent = mapper.fieldType(parentName);
+        if (false == parent.typeName().equals(KeywordFieldMapper.CONTENT_TYPE)) {
+            throw new UnsupportedOperationException();
+        }
+        KeywordFieldMapper.KeywordFieldType kwd = (KeywordFieldMapper.KeywordFieldType) parent;
+        return new MapperTestCase.BlockReaderSupport(kwd.hasDocValues(), mapper, loaderFieldName);
+    }
+
+    public static Function<Object, Object> loadBlockExpected(MapperTestCase.BlockReaderSupport blockReaderSupport, boolean columnReader) {
+        if (nullLoaderExpected(blockReaderSupport.mapper(), blockReaderSupport.loaderFieldName())) {
+            return null;
+        }
+        return v -> ((BytesRef) v).utf8ToString();
+    }
+
+    private static boolean nullLoaderExpected(MapperService mapper, String fieldName) {
+        MappedFieldType type = mapper.fieldType(fieldName);
+        if (type instanceof TextFieldMapper.TextFieldType t) {
+            if (t.isSyntheticSource() == false || t.canUseSyntheticSourceDelegateForQuerying() || t.isStored()) {
+                return false;
+            }
+            String parentField = mapper.mappingLookup().parentField(fieldName);
+            return parentField == null || nullLoaderExpected(mapper, parentField);
+        }
+        return false;
+    }
+
+    public static void validateRoundTripReader(String syntheticSource, DirectoryReader reader, DirectoryReader roundTripReader) {
+        // `reader` here is reader of original document and `roundTripReader` reads document
+        // created from synthetic source.
+        // This check fails when synthetic source is constructed using keyword subfield
+        // since in that case values are sorted (due to being read from doc values) but original document isn't.
+        //
+        // So it is disabled.
+    }
+
+    private static class TextFieldFamilySyntheticSourceSupport implements MapperTestCase.SyntheticSourceSupport {
+        private final String fieldType;
+        private final boolean storeTextField;
+        private final boolean storedKeywordField;
+        private final boolean indexText;
+        private final Integer ignoreAbove;
+        private final KeywordFieldSyntheticSourceSupport keywordSupport;
+
+        TextFieldFamilySyntheticSourceSupport(String fieldType, boolean supportsCustomIndexConfiguration) {
+            this.fieldType = fieldType;
+            this.storeTextField = randomBoolean();
+            this.storedKeywordField = storeTextField || randomBoolean();
+            this.indexText = supportsCustomIndexConfiguration ? randomBoolean() : true;
+            this.ignoreAbove = randomBoolean() ? null : between(10, 100);
+            this.keywordSupport = new KeywordFieldSyntheticSourceSupport(ignoreAbove, storedKeywordField, null, false == storeTextField);
+        }
+
+        @Override
+        public MapperTestCase.SyntheticSourceExample example(int maxValues) {
+            if (storeTextField) {
+                MapperTestCase.SyntheticSourceExample delegate = keywordSupport.example(maxValues, true);
+                return new MapperTestCase.SyntheticSourceExample(
+                    delegate.inputValue(),
+                    delegate.expectedForSyntheticSource(),
+                    delegate.expectedForBlockLoader(),
+                    b -> {
+                        b.field("type", fieldType);
+                        b.field("store", true);
+                        if (indexText == false) {
+                            b.field("index", false);
+                        }
+                    }
+                );
+            }
+            // We'll load from _source if ignore_above is defined, otherwise we load from the keyword field.
+            boolean loadingFromSource = ignoreAbove != null;
+            MapperTestCase.SyntheticSourceExample delegate = keywordSupport.example(maxValues, loadingFromSource);
+            return new MapperTestCase.SyntheticSourceExample(
+                delegate.inputValue(),
+                delegate.expectedForSyntheticSource(),
+                delegate.expectedForBlockLoader(),
+                b -> {
+                    b.field("type", fieldType);
+                    if (indexText == false) {
+                        b.field("index", false);
+                    }
+                    b.startObject("fields");
+                    {
+                        b.startObject(randomAlphaOfLength(4));
+                        delegate.mapping().accept(b);
+                        b.endObject();
+                    }
+                    b.endObject();
+                }
+            );
+        }
+
+        @Override
+        public List<MapperTestCase.SyntheticSourceInvalidExample> invalidExample() throws IOException {
+            Matcher<String> err = equalTo(
+                String.format(
+                    Locale.ROOT,
+                    "field [field] of type [%s] doesn't support synthetic source unless it is stored or"
+                        + " has a sub-field of type [keyword] with doc values or stored and without a normalizer",
+                    fieldType
+                )
+            );
+            return List.of(
+                new MapperTestCase.SyntheticSourceInvalidExample(err, b -> b.field("type", fieldType)),
+                new MapperTestCase.SyntheticSourceInvalidExample(err, b -> {
+                    b.field("type", fieldType);
+                    b.startObject("fields");
+                    {
+                        b.startObject("l");
+                        b.field("type", "long");
+                        b.endObject();
+                    }
+                    b.endObject();
+                }),
+                new MapperTestCase.SyntheticSourceInvalidExample(err, b -> {
+                    b.field("type", fieldType);
+                    b.startObject("fields");
+                    {
+                        b.startObject("kwd");
+                        b.field("type", "keyword");
+                        b.field("normalizer", "lowercase");
+                        b.endObject();
+                    }
+                    b.endObject();
+                }),
+                new MapperTestCase.SyntheticSourceInvalidExample(err, b -> {
+                    b.field("type", fieldType);
+                    b.startObject("fields");
+                    {
+                        b.startObject("kwd");
+                        b.field("type", "keyword");
+                        b.field("doc_values", "false");
+                        b.endObject();
+                    }
+                    b.endObject();
+                }),
+                new MapperTestCase.SyntheticSourceInvalidExample(err, b -> {
+                    b.field("type", fieldType);
+                    b.field("store", "false");
+                    b.startObject("fields");
+                    {
+                        b.startObject("kwd");
+                        b.field("type", "keyword");
+                        b.field("doc_values", "false");
+                        b.endObject();
+                    }
+                    b.endObject();
+                }),
+                new MapperTestCase.SyntheticSourceInvalidExample(err, b -> {
+                    b.field("type", fieldType);
+                    b.startObject("fields");
+                    {
+                        b.startObject("kwd");
+                        b.field("type", "keyword");
+                        b.field("doc_values", "false");
+                        b.field("store", "false");
+                        b.endObject();
+                    }
+                    b.endObject();
+                })
+            );
+        }
+    }
+}

From e206f6e44e76f6957f445b8de8189219c0c47a91 Mon Sep 17 00:00:00 2001
From: Mark Vieira <portugee@gmail.com>
Date: Thu, 25 Apr 2024 12:45:30 -0700
Subject: [PATCH 36/58] Move muted tests file and allow for additional files to
 be configured (#107916)

Some refactoring to the muted tests plugin to better support usage in a
composite build configuration.
---
 .../gradle/internal/test/MutedTestPlugin.java | 19 +++++++++---
 .../internal/test/MutedTestsBuildService.java | 29 ++++++++++++++-----
 .../fixtures/AbstractGradleFuncTest.groovy    |  2 +-
 .../muted-tests.yml => muted-tests.yml        |  0
 4 files changed, 37 insertions(+), 13 deletions(-)
 rename build-tools-internal/muted-tests.yml => muted-tests.yml (100%)

diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/MutedTestPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/MutedTestPlugin.java
index 4df99e7454f32..baa7704463a6d 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/MutedTestPlugin.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/MutedTestPlugin.java
@@ -8,23 +8,34 @@
 
 package org.elasticsearch.gradle.internal.test;
 
-import org.elasticsearch.gradle.internal.conventions.util.Util;
 import org.elasticsearch.gradle.internal.info.BuildParams;
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
+import org.gradle.api.file.RegularFile;
 import org.gradle.api.provider.Provider;
 import org.gradle.api.tasks.testing.Test;
 
-import java.io.File;
+import java.util.Arrays;
+import java.util.List;
 
 public class MutedTestPlugin implements Plugin<Project> {
+    private static final String ADDITIONAL_FILES_PROPERTY = "org.elasticsearch.additional.muted.tests";
+
     @Override
     public void apply(Project project) {
-        File infoPath = new File(Util.locateElasticsearchWorkspace(project.getGradle()), "build-tools-internal");
+        String additionalFilePaths = project.hasProperty(ADDITIONAL_FILES_PROPERTY)
+            ? project.property(ADDITIONAL_FILES_PROPERTY).toString()
+            : "";
+        List<RegularFile> additionalFiles = Arrays.stream(additionalFilePaths.split(","))
+            .filter(p -> p.isEmpty() == false)
+            .map(p -> project.getRootProject().getLayout().getProjectDirectory().file(p))
+            .toList();
+
         Provider<MutedTestsBuildService> mutedTestsProvider = project.getGradle()
             .getSharedServices()
             .registerIfAbsent("mutedTests", MutedTestsBuildService.class, spec -> {
-                spec.getParameters().getInfoPath().set(infoPath);
+                spec.getParameters().getInfoPath().set(project.getRootProject().getProjectDir());
+                spec.getParameters().getAdditionalFiles().set(additionalFiles);
             });
 
         project.getTasks().withType(Test.class).configureEach(test -> {
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/MutedTestsBuildService.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/MutedTestsBuildService.java
index 9e4a92f26d4dd..0fdb134c81649 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/MutedTestsBuildService.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/MutedTestsBuildService.java
@@ -13,7 +13,9 @@
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
 
+import org.gradle.api.file.RegularFile;
 import org.gradle.api.file.RegularFileProperty;
+import org.gradle.api.provider.ListProperty;
 import org.gradle.api.services.BuildService;
 import org.gradle.api.services.BuildServiceParameters;
 
@@ -28,17 +30,15 @@
 import java.util.List;
 
 public abstract class MutedTestsBuildService implements BuildService<MutedTestsBuildService.Params> {
-    private final List<String> excludePatterns;
+    private final List<String> excludePatterns = new ArrayList<>();
+    private final ObjectMapper objectMapper = new ObjectMapper(new YAMLFactory());
 
     public MutedTestsBuildService() {
         File infoPath = getParameters().getInfoPath().get().getAsFile();
         File mutedTestsFile = new File(infoPath, "muted-tests.yml");
-        try (InputStream is = new BufferedInputStream(new FileInputStream(mutedTestsFile))) {
-            ObjectMapper objectMapper = new ObjectMapper(new YAMLFactory());
-            List<MutedTest> mutedTests = objectMapper.readValue(is, MutedTests.class).getTests();
-            excludePatterns = buildExcludePatterns(mutedTests == null ? Collections.emptyList() : mutedTests);
-        } catch (IOException e) {
-            throw new UncheckedIOException(e);
+        excludePatterns.addAll(buildExcludePatterns(mutedTestsFile));
+        for (RegularFile regularFile : getParameters().getAdditionalFiles().get()) {
+            excludePatterns.addAll(buildExcludePatterns(regularFile.getAsFile()));
         }
     }
 
@@ -46,7 +46,18 @@ public List<String> getExcludePatterns() {
         return excludePatterns;
     }
 
-    private static List<String> buildExcludePatterns(List<MutedTest> mutedTests) {
+    private List<String> buildExcludePatterns(File file) {
+        List<MutedTest> mutedTests;
+
+        try (InputStream is = new BufferedInputStream(new FileInputStream(file))) {
+            mutedTests = objectMapper.readValue(is, MutedTests.class).getTests();
+            if (mutedTests == null) {
+                return Collections.emptyList();
+            }
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+
         List<String> excludes = new ArrayList<>();
         if (mutedTests.isEmpty() == false) {
             for (MutedTestsBuildService.MutedTest mutedTest : mutedTests) {
@@ -84,6 +95,8 @@ private static List<String> buildExcludePatterns(List<MutedTest> mutedTests) {
 
     public interface Params extends BuildServiceParameters {
         RegularFileProperty getInfoPath();
+
+        ListProperty<RegularFile> getAdditionalFiles();
     }
 
     public static class MutedTest {
diff --git a/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy b/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy
index 2f1d1f5d36e87..41f6f445f58ec 100644
--- a/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy
+++ b/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy
@@ -59,7 +59,7 @@ abstract class AbstractGradleFuncTest extends Specification {
                 id 'base'
             }
         """
-        def mutedTestsFile = Files.createFile(Path.of(testProjectDir.newFolder("build-tools-internal").path, "muted-tests.yml"))
+        def mutedTestsFile = testProjectDir.newFile("muted-tests.yml")
         mutedTestsFile << """
             tests: []
         """
diff --git a/build-tools-internal/muted-tests.yml b/muted-tests.yml
similarity index 100%
rename from build-tools-internal/muted-tests.yml
rename to muted-tests.yml

From ed81cb59f80c9d767c12c2eec5809faadc1ecba2 Mon Sep 17 00:00:00 2001
From: Brian Seeders <brian.seeders@elastic.co>
Date: Thu, 25 Apr 2024 16:05:27 -0400
Subject: [PATCH 37/58] [ci] Shrink platform-support Windows instances
 (#107912)

---
 .buildkite/pipelines/periodic-platform-support.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/pipelines/periodic-platform-support.yml b/.buildkite/pipelines/periodic-platform-support.yml
index a3922d8226924..f454d20fc542e 100644
--- a/.buildkite/pipelines/periodic-platform-support.yml
+++ b/.buildkite/pipelines/periodic-platform-support.yml
@@ -52,7 +52,7 @@ steps:
         agents:
           provider: gcp
           image: family/elasticsearch-{{matrix.image}}
-          machineType: n1-standard-32
+          machineType: n1-standard-16
           diskType: pd-ssd
           diskSizeGb: 350
         env:

From e3c84af98962012d2ea1d8fe6bd93c9a7ab45544 Mon Sep 17 00:00:00 2001
From: Mikhail Berezovskiy <mikhail@elastic.co>
Date: Thu, 25 Apr 2024 14:37:03 -0700
Subject: [PATCH 38/58] Add null and empty string validation to S3 bucket
 (#107883)

Add basic validation to S3 bucket name - nullity and empty string. It is
aligned with public
[docs](https://www.elastic.co/guide/en/elasticsearch/reference/8.13/repository-s3.html#repository-s3-repository)
for "bucket" as required field. We might want to add more validations
based on S3 naming rules. This PR should not be a breaking change
because missing bucket  will eventually throw exception later in the
code with obscure error.

I've added yaml test to modules
[repository_s3/10_basic.yml](https://github.com/elastic/elasticsearch/compare/main...mhl-b:elasticsearch:s3-bucket-validation?expand=1#diff-08cf26742fe939f5575961254c4d3b4bff6915141cdd6abe4cd28a743d1b70ba),
not sure if it's a right place.

Addresses #107840
---
 .../repositories/s3/S3Repository.java         |  4 +-
 .../s3/RepositoryCredentialsTests.java        |  9 +++-
 .../repositories/s3/S3RepositoryTests.java    | 24 ++++++++-
 .../test/repository_s3/10_basic.yml           | 50 +++++++++++++++----
 4 files changed, 71 insertions(+), 16 deletions(-)

diff --git a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java
index 26b1b1158dea0..1ba5801a09d02 100644
--- a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java
+++ b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java
@@ -223,8 +223,8 @@ class S3Repository extends MeteredBlobStoreRepository {
 
         // Parse and validate the user's S3 Storage Class setting
         this.bucket = BUCKET_SETTING.get(metadata.settings());
-        if (bucket == null) {
-            throw new RepositoryException(metadata.name(), "No bucket defined for s3 repository");
+        if (Strings.hasLength(bucket) == false) {
+            throw new IllegalArgumentException("Invalid S3 bucket name, cannot be null or empty");
         }
 
         this.bufferSize = BUFFER_SIZE_SETTING.get(metadata.settings());
diff --git a/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/RepositoryCredentialsTests.java b/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/RepositoryCredentialsTests.java
index 10f19f04da002..9a1d12fab0af5 100644
--- a/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/RepositoryCredentialsTests.java
+++ b/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/RepositoryCredentialsTests.java
@@ -82,6 +82,7 @@ protected Settings nodeSettings() {
     public void testRepositoryCredentialsOverrideSecureCredentials() {
         final String repositoryName = "repo-creds-override";
         final Settings.Builder repositorySettings = Settings.builder()
+            .put(S3Repository.BUCKET_SETTING.getKey(), "bucket")
             // repository settings for credentials override node secure settings
             .put(S3Repository.ACCESS_KEY_SETTING.getKey(), "insecure_aws_key")
             .put(S3Repository.SECRET_KEY_SETTING.getKey(), "insecure_aws_secret");
@@ -115,7 +116,7 @@ public void testRepositoryCredentialsOverrideSecureCredentials() {
     public void testReinitSecureCredentials() {
         final String clientName = randomFrom("default", "other");
 
-        final Settings.Builder repositorySettings = Settings.builder();
+        final Settings.Builder repositorySettings = Settings.builder().put(S3Repository.BUCKET_SETTING.getKey(), "bucket");
         final boolean hasInsecureSettings = randomBoolean();
         if (hasInsecureSettings) {
             // repository settings for credentials override node secure settings
@@ -153,7 +154,10 @@ public void testReinitSecureCredentials() {
             final MockSecureSettings newSecureSettings = new MockSecureSettings();
             newSecureSettings.setString("s3.client." + clientName + ".access_key", "new_secret_aws_key");
             newSecureSettings.setString("s3.client." + clientName + ".secret_key", "new_secret_aws_secret");
-            final Settings newSettings = Settings.builder().setSecureSettings(newSecureSettings).build();
+            final Settings newSettings = Settings.builder()
+                .put(S3Repository.BUCKET_SETTING.getKey(), "bucket")
+                .setSecureSettings(newSecureSettings)
+                .build();
             // reload S3 plugin settings
             final PluginsService plugins = getInstanceFromNode(PluginsService.class);
             final ProxyS3RepositoryPlugin plugin = plugins.filterPlugins(ProxyS3RepositoryPlugin.class).findFirst().get();
@@ -202,6 +206,7 @@ public void testInsecureRepositoryCredentials() throws Exception {
         createRepository(
             repositoryName,
             Settings.builder()
+                .put(S3Repository.BUCKET_SETTING.getKey(), "bucket")
                 .put(S3Repository.ACCESS_KEY_SETTING.getKey(), "insecure_aws_key")
                 .put(S3Repository.SECRET_KEY_SETTING.getKey(), "insecure_aws_secret")
                 .build()
diff --git a/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryTests.java b/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryTests.java
index c1f862a7628c5..fcb0e82505dac 100644
--- a/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryTests.java
+++ b/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryTests.java
@@ -89,6 +89,7 @@ public void testInvalidChunkBufferSizeSettings() {
 
     private Settings bufferAndChunkSettings(long buffer, long chunk) {
         return Settings.builder()
+            .put(S3Repository.BUCKET_SETTING.getKey(), "bucket")
             .put(S3Repository.BUFFER_SIZE_SETTING.getKey(), new ByteSizeValue(buffer, ByteSizeUnit.MB).getStringRep())
             .put(S3Repository.CHUNK_SIZE_SETTING.getKey(), new ByteSizeValue(chunk, ByteSizeUnit.MB).getStringRep())
             .build();
@@ -102,7 +103,10 @@ public void testBasePathSetting() {
         final RepositoryMetadata metadata = new RepositoryMetadata(
             "dummy-repo",
             "mock",
-            Settings.builder().put(S3Repository.BASE_PATH_SETTING.getKey(), "foo/bar").build()
+            Settings.builder()
+                .put(S3Repository.BUCKET_SETTING.getKey(), "bucket")
+                .put(S3Repository.BASE_PATH_SETTING.getKey(), "foo/bar")
+                .build()
         );
         try (S3Repository s3repo = createS3Repo(metadata)) {
             assertEquals("foo/bar/", s3repo.basePath().buildAsString());
@@ -110,7 +114,11 @@ public void testBasePathSetting() {
     }
 
     public void testDefaultBufferSize() {
-        final RepositoryMetadata metadata = new RepositoryMetadata("dummy-repo", "mock", Settings.EMPTY);
+        final RepositoryMetadata metadata = new RepositoryMetadata(
+            "dummy-repo",
+            "mock",
+            Settings.builder().put(S3Repository.BUCKET_SETTING.getKey(), "bucket").build()
+        );
         try (S3Repository s3repo = createS3Repo(metadata)) {
             assertThat(s3repo.getBlobStore(), is(nullValue()));
             s3repo.start();
@@ -121,6 +129,17 @@ public void testDefaultBufferSize() {
         }
     }
 
+    public void testMissingBucketName() {
+        final var metadata = new RepositoryMetadata("repo", "mock", Settings.EMPTY);
+        assertThrows(IllegalArgumentException.class, () -> createS3Repo(metadata));
+    }
+
+    public void testEmptyBucketName() {
+        final var settings = Settings.builder().put(S3Repository.BUCKET_SETTING.getKey(), "").build();
+        final var metadata = new RepositoryMetadata("repo", "mock", settings);
+        assertThrows(IllegalArgumentException.class, () -> createS3Repo(metadata));
+    }
+
     private S3Repository createS3Repo(RepositoryMetadata metadata) {
         return new S3Repository(
             metadata,
@@ -132,4 +151,5 @@ private S3Repository createS3Repo(RepositoryMetadata metadata) {
             S3RepositoriesMetrics.NOOP
         );
     }
+
 }
diff --git a/modules/repository-s3/src/yamlRestTest/resources/rest-api-spec/test/repository_s3/10_basic.yml b/modules/repository-s3/src/yamlRestTest/resources/rest-api-spec/test/repository_s3/10_basic.yml
index 7ff857ffa0bf2..e1ca66f47114b 100644
--- a/modules/repository-s3/src/yamlRestTest/resources/rest-api-spec/test/repository_s3/10_basic.yml
+++ b/modules/repository-s3/src/yamlRestTest/resources/rest-api-spec/test/repository_s3/10_basic.yml
@@ -1,16 +1,46 @@
 # Integration tests for repository-s3
 #
 "Module repository-s3 is loaded":
-    - skip:
-        reason: "contains is a newly added assertion"
-        features: contains
-    - do:
-        cluster.state: {}
+  - skip:
+      reason: "contains is a newly added assertion"
+      features: contains
+  - do:
+      cluster.state: { }
 
-    # Get master node id
-    - set: { master_node: master }
+  # Get master node id
+  - set: { master_node: master }
 
-    - do:
-        nodes.info: {}
+  - do:
+      nodes.info: { }
 
-    - contains:  { nodes.$master.modules: { name: repository-s3 } }
+  - contains: { nodes.$master.modules: { name: repository-s3 } }
+
+---
+"Create S3 snapshot repository":
+  - do:
+      snapshot.create_repository:
+        repository: test-snapshot-repo
+        verify: false
+        body:
+          type: s3
+          settings:
+            bucket: test-bucket
+
+---
+"Create S3 snapshot repository without bucket":
+  - do:
+      catch: /illegal_argument_exception/
+      snapshot.create_repository:
+        repository: test-snapshot-repo-without-bucket
+        verify: false
+        body:
+          type: s3
+  - do:
+      catch: /illegal_argument_exception/
+      snapshot.create_repository:
+        repository: test-snapshot-repo-without-bucket
+        verify: false
+        body:
+          type: s3
+          settings:
+            bucket: ""

From fe767702bde11bcbc91f33593e24a28675e6d383 Mon Sep 17 00:00:00 2001
From: Ignacio Vera <ignacio.vera@elastic.co>
Date: Fri, 26 Apr 2024 07:59:07 +0200
Subject: [PATCH 39/58] Optimise composite aggregations for single value fields
 (#107897)

 this commit optimize composite aggregations for single value fields.
---
 docs/changelog/107897.yaml                    |   5 +
 .../bucket/composite/BinaryValuesSource.java  |  22 ++++
 .../bucket/composite/DoubleValuesSource.java  |  23 ++++
 .../composite/GlobalOrdinalValuesSource.java  | 104 +++++++++++++-----
 .../composite/HistogramValuesSource.java      |  26 ++++-
 .../bucket/composite/LongValuesSource.java    |  23 ++++
 .../composite/RoundingValuesSource.java       |  43 +++++++-
 7 files changed, 215 insertions(+), 31 deletions(-)
 create mode 100644 docs/changelog/107897.yaml

diff --git a/docs/changelog/107897.yaml b/docs/changelog/107897.yaml
new file mode 100644
index 0000000000000..e4a2a5270475d
--- /dev/null
+++ b/docs/changelog/107897.yaml
@@ -0,0 +1,5 @@
+pr: 107897
+summary: Optimise composite aggregations for single value fields
+area: Aggregations
+type: enhancement
+issues: []
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/BinaryValuesSource.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/BinaryValuesSource.java
index 62f587f5249d1..3eb55ec9dd82e 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/BinaryValuesSource.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/BinaryValuesSource.java
@@ -8,6 +8,7 @@
 
 package org.elasticsearch.search.aggregations.bucket.composite;
 
+import org.apache.lucene.index.BinaryDocValues;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.search.MatchAllDocsQuery;
@@ -18,6 +19,7 @@
 import org.elasticsearch.common.util.ObjectArray;
 import org.elasticsearch.core.CheckedFunction;
 import org.elasticsearch.core.Releasables;
+import org.elasticsearch.index.fielddata.FieldData;
 import org.elasticsearch.index.fielddata.SortedBinaryDocValues;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.StringFieldType;
@@ -163,6 +165,11 @@ BytesRef toComparable(int slot) {
     @Override
     LeafBucketCollector getLeafCollector(LeafReaderContext context, LeafBucketCollector next) throws IOException {
         final SortedBinaryDocValues dvs = docValuesFunc.apply(context);
+        final BinaryDocValues singleton = FieldData.unwrapSingleton(dvs);
+        return singleton != null ? getLeafCollector(singleton, next) : getLeafCollector(dvs, next);
+    }
+
+    private LeafBucketCollector getLeafCollector(SortedBinaryDocValues dvs, LeafBucketCollector next) {
         return new LeafBucketCollector() {
             @Override
             public void collect(int doc, long bucket) throws IOException {
@@ -180,6 +187,21 @@ public void collect(int doc, long bucket) throws IOException {
         };
     }
 
+    private LeafBucketCollector getLeafCollector(BinaryDocValues dvs, LeafBucketCollector next) {
+        return new LeafBucketCollector() {
+            @Override
+            public void collect(int doc, long bucket) throws IOException {
+                if (dvs.advanceExact(doc)) {
+                    currentValue = dvs.binaryValue();
+                    next.collect(doc, bucket);
+                } else if (missingBucket) {
+                    currentValue = null;
+                    next.collect(doc, bucket);
+                }
+            }
+        };
+    }
+
     @Override
     LeafBucketCollector getLeafCollector(Comparable<BytesRef> value, LeafReaderContext context, LeafBucketCollector next) {
         if (value.getClass() != BytesRef.class) {
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/DoubleValuesSource.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/DoubleValuesSource.java
index 752c4ecf97401..6af1ec28c91e3 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/DoubleValuesSource.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/DoubleValuesSource.java
@@ -16,6 +16,8 @@
 import org.elasticsearch.common.util.DoubleArray;
 import org.elasticsearch.core.CheckedFunction;
 import org.elasticsearch.core.Releasables;
+import org.elasticsearch.index.fielddata.FieldData;
+import org.elasticsearch.index.fielddata.NumericDoubleValues;
 import org.elasticsearch.index.fielddata.SortedNumericDoubleValues;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.search.DocValueFormat;
@@ -154,6 +156,11 @@ Double toComparable(int slot) {
     @Override
     LeafBucketCollector getLeafCollector(LeafReaderContext context, LeafBucketCollector next) throws IOException {
         final SortedNumericDoubleValues dvs = docValuesFunc.apply(context);
+        final NumericDoubleValues singleton = FieldData.unwrapSingleton(dvs);
+        return singleton != null ? getLeafCollector(singleton, next) : getLeafCollector(dvs, next);
+    }
+
+    private LeafBucketCollector getLeafCollector(SortedNumericDoubleValues dvs, LeafBucketCollector next) {
         return new LeafBucketCollector() {
             @Override
             public void collect(int doc, long bucket) throws IOException {
@@ -176,6 +183,22 @@ public void collect(int doc, long bucket) throws IOException {
         };
     }
 
+    private LeafBucketCollector getLeafCollector(NumericDoubleValues dvs, LeafBucketCollector next) {
+        return new LeafBucketCollector() {
+            @Override
+            public void collect(int doc, long bucket) throws IOException {
+                if (dvs.advanceExact(doc)) {
+                    currentValue = dvs.doubleValue();
+                    missingCurrentValue = false;
+                    next.collect(doc, bucket);
+                } else if (missingBucket) {
+                    missingCurrentValue = true;
+                    next.collect(doc, bucket);
+                }
+            }
+        };
+    }
+
     @Override
     LeafBucketCollector getLeafCollector(Comparable<Double> value, LeafReaderContext context, LeafBucketCollector next) {
         if (value.getClass() != Double.class) {
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/GlobalOrdinalValuesSource.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/GlobalOrdinalValuesSource.java
index f833bb39b3b56..e6f90bfbdf975 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/GlobalOrdinalValuesSource.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/GlobalOrdinalValuesSource.java
@@ -8,9 +8,11 @@
 
 package org.elasticsearch.search.aggregations.bucket.composite;
 
+import org.apache.lucene.index.DocValues;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.PostingsEnum;
+import org.apache.lucene.index.SortedDocValues;
 import org.apache.lucene.index.SortedSetDocValues;
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.index.TermsEnum;
@@ -218,27 +220,49 @@ LeafBucketCollector getLeafCollector(LeafReaderContext context, LeafBucketCollec
         final CompetitiveIterator competitiveIterator = fieldType == null ? null : new CompetitiveIterator(context, fieldType.name());
         currentCompetitiveIterator = competitiveIterator;
 
-        return new LeafBucketCollector() {
+        final SortedDocValues singleton = DocValues.unwrapSingleton(dvs);
+        if (singleton != null) {
+            return new LeafBucketCollector() {
 
-            @Override
-            public void collect(int doc, long bucket) throws IOException {
-                if (dvs.advanceExact(doc)) {
-                    long ord;
-                    while ((ord = dvs.nextOrd()) != NO_MORE_ORDS) {
-                        currentValue = ord;
+                @Override
+                public void collect(int doc, long bucket) throws IOException {
+                    if (singleton.advanceExact(doc)) {
+                        currentValue = singleton.ordValue();
+                        next.collect(doc, bucket);
+                    } else if (missingBucket) {
+                        currentValue = MISSING_VALUE_FLAG;
                         next.collect(doc, bucket);
                     }
-                } else if (missingBucket) {
-                    currentValue = MISSING_VALUE_FLAG;
-                    next.collect(doc, bucket);
                 }
-            }
 
-            @Override
-            public DocIdSetIterator competitiveIterator() {
-                return competitiveIterator;
-            }
-        };
+                @Override
+                public DocIdSetIterator competitiveIterator() {
+                    return competitiveIterator;
+                }
+            };
+        } else {
+            return new LeafBucketCollector() {
+
+                @Override
+                public void collect(int doc, long bucket) throws IOException {
+                    if (dvs.advanceExact(doc)) {
+                        long ord;
+                        while ((ord = dvs.nextOrd()) != NO_MORE_ORDS) {
+                            currentValue = ord;
+                            next.collect(doc, bucket);
+                        }
+                    } else if (missingBucket) {
+                        currentValue = MISSING_VALUE_FLAG;
+                        next.collect(doc, bucket);
+                    }
+                }
+
+                @Override
+                public DocIdSetIterator competitiveIterator() {
+                    return competitiveIterator;
+                }
+            };
+        }
     }
 
     @Override
@@ -253,27 +277,49 @@ LeafBucketCollector getLeafCollector(Comparable<BytesRef> value, LeafReaderConte
         if (lookup == null) {
             initLookup(dvs);
         }
-        return new LeafBucketCollector() {
-            boolean currentValueIsSet = false;
+        final SortedDocValues singleton = DocValues.unwrapSingleton(dvs);
+        if (singleton != null) {
+            return new LeafBucketCollector() {
+                boolean currentValueIsSet = false;
 
-            @Override
-            public void collect(int doc, long bucket) throws IOException {
-                if (currentValueIsSet == false) {
-                    if (dvs.advanceExact(doc)) {
-                        long ord;
-                        while ((ord = dvs.nextOrd()) != NO_MORE_ORDS) {
+                @Override
+                public void collect(int doc, long bucket) throws IOException {
+                    if (currentValueIsSet == false) {
+                        if (singleton.advanceExact(doc)) {
+                            long ord = singleton.ordValue();
                             if (term.equals(lookup.lookupOrd(ord))) {
                                 currentValueIsSet = true;
                                 currentValue = ord;
-                                break;
                             }
                         }
                     }
+                    assert currentValueIsSet;
+                    next.collect(doc, bucket);
                 }
-                assert currentValueIsSet;
-                next.collect(doc, bucket);
-            }
-        };
+            };
+        } else {
+            return new LeafBucketCollector() {
+                boolean currentValueIsSet = false;
+
+                @Override
+                public void collect(int doc, long bucket) throws IOException {
+                    if (currentValueIsSet == false) {
+                        if (dvs.advanceExact(doc)) {
+                            long ord;
+                            while ((ord = dvs.nextOrd()) != NO_MORE_ORDS) {
+                                if (term.equals(lookup.lookupOrd(ord))) {
+                                    currentValueIsSet = true;
+                                    currentValue = ord;
+                                    break;
+                                }
+                            }
+                        }
+                    }
+                    assert currentValueIsSet;
+                    next.collect(doc, bucket);
+                }
+            };
+        }
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/HistogramValuesSource.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/HistogramValuesSource.java
index ef06c0a724c0c..13cc0a4e661da 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/HistogramValuesSource.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/HistogramValuesSource.java
@@ -10,6 +10,7 @@
 
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.SortedNumericDocValues;
+import org.elasticsearch.index.fielddata.NumericDoubleValues;
 import org.elasticsearch.index.fielddata.SortedBinaryDocValues;
 import org.elasticsearch.index.fielddata.SortedNumericDoubleValues;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
@@ -36,7 +37,16 @@ public boolean isFloatingPoint() {
 
     @Override
     public SortedNumericDoubleValues doubleValues(LeafReaderContext context) throws IOException {
-        SortedNumericDoubleValues values = vs.doubleValues(context);
+        final SortedNumericDoubleValues values = vs.doubleValues(context);
+        final NumericDoubleValues singleton = org.elasticsearch.index.fielddata.FieldData.unwrapSingleton(values);
+        if (singleton != null) {
+            return org.elasticsearch.index.fielddata.FieldData.singleton(doubleSingleValues(singleton));
+        } else {
+            return doubleMultiValues(values);
+        }
+    }
+
+    private SortedNumericDoubleValues doubleMultiValues(SortedNumericDoubleValues values) {
         return new SortedNumericDoubleValues() {
             @Override
             public double nextValue() throws IOException {
@@ -55,6 +65,20 @@ public boolean advanceExact(int target) throws IOException {
         };
     }
 
+    private NumericDoubleValues doubleSingleValues(NumericDoubleValues values) {
+        return new NumericDoubleValues() {
+            @Override
+            public double doubleValue() throws IOException {
+                return Math.floor(values.doubleValue() / interval) * interval;
+            }
+
+            @Override
+            public boolean advanceExact(int target) throws IOException {
+                return values.advanceExact(target);
+            }
+        };
+    }
+
     @Override
     public SortedBinaryDocValues bytesValues(LeafReaderContext context) throws IOException {
         throw new UnsupportedOperationException("not applicable");
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/LongValuesSource.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/LongValuesSource.java
index ca9968834e611..c6b0df4a4cfa8 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/LongValuesSource.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/LongValuesSource.java
@@ -10,8 +10,10 @@
 
 import org.apache.lucene.document.IntPoint;
 import org.apache.lucene.document.LongPoint;
+import org.apache.lucene.index.DocValues;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.NumericDocValues;
 import org.apache.lucene.index.SortedNumericDocValues;
 import org.apache.lucene.search.BoostQuery;
 import org.apache.lucene.search.ConstantScoreQuery;
@@ -170,6 +172,11 @@ Long toComparable(int slot) {
     @Override
     LeafBucketCollector getLeafCollector(LeafReaderContext context, LeafBucketCollector next) throws IOException {
         final SortedNumericDocValues dvs = docValuesFunc.apply(context);
+        final NumericDocValues singleton = DocValues.unwrapSingleton(dvs);
+        return singleton != null ? getLeafCollector(singleton, next) : getLeafCollector(dvs, next);
+    }
+
+    private LeafBucketCollector getLeafCollector(SortedNumericDocValues dvs, LeafBucketCollector next) {
         return new LeafBucketCollector() {
             @Override
             public void collect(int doc, long bucket) throws IOException {
@@ -192,6 +199,22 @@ public void collect(int doc, long bucket) throws IOException {
         };
     }
 
+    private LeafBucketCollector getLeafCollector(NumericDocValues dvs, LeafBucketCollector next) {
+        return new LeafBucketCollector() {
+            @Override
+            public void collect(int doc, long bucket) throws IOException {
+                if (dvs.advanceExact(doc)) {
+                    currentValue = dvs.longValue();
+                    missingCurrentValue = false;
+                    next.collect(doc, bucket);
+                } else if (missingBucket) {
+                    missingCurrentValue = true;
+                    next.collect(doc, bucket);
+                }
+            }
+        };
+    }
+
     @Override
     LeafBucketCollector getLeafCollector(Comparable<Long> value, LeafReaderContext context, LeafBucketCollector next) {
         if (value.getClass() != Long.class) {
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/RoundingValuesSource.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/RoundingValuesSource.java
index df1fafe941931..79e3cc4a5a5b2 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/RoundingValuesSource.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/RoundingValuesSource.java
@@ -8,7 +8,9 @@
 
 package org.elasticsearch.search.aggregations.bucket.composite;
 
+import org.apache.lucene.index.DocValues;
 import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.NumericDocValues;
 import org.apache.lucene.index.SortedNumericDocValues;
 import org.elasticsearch.common.Rounding;
 import org.elasticsearch.index.fielddata.SortedBinaryDocValues;
@@ -54,7 +56,12 @@ public double roundingSize(Rounding.DateTimeUnit unit) {
 
     @Override
     public SortedNumericDocValues longValues(LeafReaderContext context) throws IOException {
-        SortedNumericDocValues values = vs.longValues(context);
+        final SortedNumericDocValues values = vs.longValues(context);
+        final NumericDocValues singleton = DocValues.unwrapSingleton(values);
+        return singleton != null ? DocValues.singleton(longSingleValues(singleton)) : longMultiValues(values);
+    }
+
+    private SortedNumericDocValues longMultiValues(SortedNumericDocValues values) {
         return new SortedNumericDocValues() {
             @Override
             public long nextValue() throws IOException {
@@ -93,6 +100,40 @@ public long cost() {
         };
     }
 
+    private NumericDocValues longSingleValues(NumericDocValues values) {
+        return new NumericDocValues() {
+            @Override
+            public long longValue() throws IOException {
+                return round(values.longValue());
+            }
+
+            @Override
+            public boolean advanceExact(int target) throws IOException {
+                return values.advanceExact(target);
+            }
+
+            @Override
+            public int docID() {
+                return values.docID();
+            }
+
+            @Override
+            public int nextDoc() throws IOException {
+                return values.nextDoc();
+            }
+
+            @Override
+            public int advance(int target) throws IOException {
+                return values.advance(target);
+            }
+
+            @Override
+            public long cost() {
+                return values.cost();
+            }
+        };
+    }
+
     @Override
     public SortedBinaryDocValues bytesValues(LeafReaderContext context) throws IOException {
         throw new UnsupportedOperationException("not applicable");

From 0b747ac43ef3985cfd999029e9ac8cdba2debac7 Mon Sep 17 00:00:00 2001
From: Yang Wang <yang.wang@elastic.co>
Date: Fri, 26 Apr 2024 16:05:28 +1000
Subject: [PATCH 40/58] [Test] More randomization for snapshot names (#107884)

Increase the randomization for snapshot names to avoid duplicates which
fail to create map in test. See similar fix #101603

Resolves: #107816
---
 .../xpack/core/slm/SnapshotRetentionConfigurationTests.java   | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/slm/SnapshotRetentionConfigurationTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/slm/SnapshotRetentionConfigurationTests.java
index 2ac6b633c0f09..cf4672d183ada 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/slm/SnapshotRetentionConfigurationTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/slm/SnapshotRetentionConfigurationTests.java
@@ -349,7 +349,7 @@ private SnapshotInfo makeFailureInfo(long startTime) {
         }
         assert failureCount == failures.size();
         SnapshotInfo snapInfo = new SnapshotInfo(
-            new Snapshot(REPO, new SnapshotId("snap-fail-" + randomAlphaOfLength(3), "uuid-fail")),
+            new Snapshot(REPO, new SnapshotId("snap-fail-" + randomUUID(), "uuid-fail")),
             Collections.singletonList("foo-fail"),
             Collections.singletonList("bar-fail"),
             Collections.emptyList(),
@@ -377,7 +377,7 @@ private SnapshotInfo makePartialInfo(long startTime) {
         }
         assert failureCount == failures.size();
         SnapshotInfo snapInfo = new SnapshotInfo(
-            new Snapshot(REPO, new SnapshotId("snap-fail-" + randomAlphaOfLength(3), "uuid-fail")),
+            new Snapshot(REPO, new SnapshotId("snap-fail-" + randomUUID(), "uuid-fail")),
             Collections.singletonList("foo-fail"),
             Collections.singletonList("bar-fail"),
             Collections.emptyList(),

From a6a29d0946e8344da4847fcd31c60ae0e9f630f3 Mon Sep 17 00:00:00 2001
From: Mary Gouseti <mary.gouseti@elastic.co>
Date: Fri, 26 Apr 2024 10:37:58 +0300
Subject: [PATCH 41/58] [Data stream lifecycle]Add warning when the user's
 retention is not the effective retention (#107781)

---
 .../MetadataIndexTemplateServiceTests.java    |   5 +-
 .../cluster/metadata/DataStreamLifecycle.java |  36 +++
 .../metadata/MetadataDataStreamsService.java  |  17 +-
 .../MetadataIndexTemplateService.java         |  44 ++-
 .../elasticsearch/node/NodeConstruction.java  |  45 ++-
 ...vedComposableIndexTemplateActionTests.java |  10 +-
 .../metadata/DataStreamLifecycleTests.java    |   7 +-
 ...amLifecycleWithRetentionWarningsTests.java | 281 ++++++++++++++++++
 .../MetadataDataStreamsServiceTests.java      |  12 +-
 .../MetadataIndexTemplateServiceTests.java    |   9 +-
 10 files changed, 424 insertions(+), 42 deletions(-)
 create mode 100644 server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleWithRetentionWarningsTests.java

diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataIndexTemplateServiceTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataIndexTemplateServiceTests.java
index d1e07aacaddce..022e33621c4f4 100644
--- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataIndexTemplateServiceTests.java
+++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataIndexTemplateServiceTests.java
@@ -12,6 +12,8 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.ComponentTemplate;
 import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
+import org.elasticsearch.cluster.metadata.DataStreamFactoryRetention;
+import org.elasticsearch.cluster.metadata.DataStreamGlobalRetentionResolver;
 import org.elasticsearch.cluster.metadata.DataStreamLifecycle;
 import org.elasticsearch.cluster.metadata.MetadataCreateIndexService;
 import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService;
@@ -213,7 +215,8 @@ private MetadataIndexTemplateService getMetadataIndexTemplateService() {
             new IndexScopedSettings(Settings.EMPTY, IndexScopedSettings.BUILT_IN_INDEX_SETTINGS),
             xContentRegistry(),
             EmptySystemIndices.INSTANCE,
-            indexSettingProviders
+            indexSettingProviders,
+            new DataStreamGlobalRetentionResolver(DataStreamFactoryRetention.emptyFactoryRetention())
         );
     }
 
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamLifecycle.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamLifecycle.java
index 6db7b2cf670bc..9e23ffed6e8c5 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamLifecycle.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamLifecycle.java
@@ -18,6 +18,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.logging.HeaderWarning;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.Nullable;
@@ -186,6 +187,41 @@ public TimeValue getDataStreamRetention() {
         return dataRetention == null ? null : dataRetention.value;
     }
 
+    /**
+     * This method checks if the effective retention is matching what the user has configured; if the effective retention
+     * does not match then it adds a warning informing the user about the effective retention and the source.
+     */
+    public void addWarningHeaderIfDataRetentionNotEffective(@Nullable DataStreamGlobalRetention globalRetention) {
+        if (globalRetention == null) {
+            return;
+        }
+        Tuple<TimeValue, DataStreamLifecycle.RetentionSource> effectiveDataRetentionWithSource = getEffectiveDataRetentionWithSource(
+            globalRetention
+        );
+        String effectiveRetentionStringRep = effectiveDataRetentionWithSource.v1().getStringRep();
+        switch (effectiveDataRetentionWithSource.v2()) {
+            case DEFAULT_GLOBAL_RETENTION -> HeaderWarning.addWarning(
+                "Not providing a retention is not allowed for this project. The default retention of ["
+                    + effectiveRetentionStringRep
+                    + "] will be applied."
+            );
+            case MAX_GLOBAL_RETENTION -> {
+                String retentionProvidedPart = getDataStreamRetention() == null
+                    ? "Not providing a retention is not allowed for this project."
+                    : "The retention provided ["
+                        + (getDataStreamRetention() == null ? "infinite" : getDataStreamRetention().getStringRep())
+                        + "] is exceeding the max allowed data retention of this project ["
+                        + effectiveRetentionStringRep
+                        + "].";
+                HeaderWarning.addWarning(
+                    retentionProvidedPart + " The max retention of [" + effectiveRetentionStringRep + "] will be applied"
+                );
+            }
+            case DATA_STREAM_CONFIGURATION -> {
+            }
+        }
+    }
+
     /**
      * The configuration as provided by the user about the least amount of time data should be kept by elasticsearch.
      * This method differentiates between a missing retention and a nullified retention and this is useful for template
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsService.java
index 29001a078956a..20afd7f9eb3ed 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsService.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsService.java
@@ -41,12 +41,18 @@ public class MetadataDataStreamsService {
 
     private final ClusterService clusterService;
     private final IndicesService indicesService;
+    private final DataStreamGlobalRetentionResolver globalRetentionResolver;
     private final MasterServiceTaskQueue<UpdateLifecycleTask> updateLifecycleTaskQueue;
     private final MasterServiceTaskQueue<SetRolloverOnWriteTask> setRolloverOnWriteTaskQueue;
 
-    public MetadataDataStreamsService(ClusterService clusterService, IndicesService indicesService) {
+    public MetadataDataStreamsService(
+        ClusterService clusterService,
+        IndicesService indicesService,
+        DataStreamGlobalRetentionResolver globalRetentionResolver
+    ) {
         this.clusterService = clusterService;
         this.indicesService = indicesService;
+        this.globalRetentionResolver = globalRetentionResolver;
         ClusterStateTaskExecutor<UpdateLifecycleTask> updateLifecycleExecutor = new SimpleBatchedAckListenerTaskExecutor<>() {
 
             @Override
@@ -199,17 +205,16 @@ static ClusterState modifyDataStream(
      * Creates an updated cluster state in which the requested data streams have the data stream lifecycle provided.
      * Visible for testing.
      */
-    static ClusterState updateDataLifecycle(
-        ClusterState currentState,
-        List<String> dataStreamNames,
-        @Nullable DataStreamLifecycle lifecycle
-    ) {
+    ClusterState updateDataLifecycle(ClusterState currentState, List<String> dataStreamNames, @Nullable DataStreamLifecycle lifecycle) {
         Metadata metadata = currentState.metadata();
         Metadata.Builder builder = Metadata.builder(metadata);
         for (var dataStreamName : dataStreamNames) {
             var dataStream = validateDataStream(metadata, dataStreamName);
             builder.put(dataStream.copy().setLifecycle(lifecycle).build());
         }
+        if (lifecycle != null) {
+            lifecycle.addWarningHeaderIfDataRetentionNotEffective(globalRetentionResolver.resolve(currentState));
+        }
         return ClusterState.builder(currentState).metadata(builder.build()).build();
     }
 
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java
index 0daa12b7ed71f..11e660d0ea42f 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java
@@ -136,6 +136,7 @@ public class MetadataIndexTemplateService {
     private final NamedXContentRegistry xContentRegistry;
     private final SystemIndices systemIndices;
     private final Set<IndexSettingProvider> indexSettingProviders;
+    private final DataStreamGlobalRetentionResolver globalRetentionResolver;
 
     /**
      * This is the cluster state task executor for all template-based actions.
@@ -180,7 +181,8 @@ public MetadataIndexTemplateService(
         IndexScopedSettings indexScopedSettings,
         NamedXContentRegistry xContentRegistry,
         SystemIndices systemIndices,
-        IndexSettingProviders indexSettingProviders
+        IndexSettingProviders indexSettingProviders,
+        DataStreamGlobalRetentionResolver globalRetentionResolver
     ) {
         this.clusterService = clusterService;
         this.taskQueue = clusterService.createTaskQueue("index-templates", Priority.URGENT, TEMPLATE_TASK_EXECUTOR);
@@ -190,6 +192,7 @@ public MetadataIndexTemplateService(
         this.xContentRegistry = xContentRegistry;
         this.systemIndices = systemIndices;
         this.indexSettingProviders = indexSettingProviders.getIndexSettingProviders();
+        this.globalRetentionResolver = globalRetentionResolver;
     }
 
     public void removeTemplates(final RemoveRequest request, final ActionListener<AcknowledgedResponse> listener) {
@@ -333,10 +336,11 @@ public ClusterState addComponentTemplate(
                 final String composableTemplateName = entry.getKey();
                 final ComposableIndexTemplate composableTemplate = entry.getValue();
                 try {
-                    validateLifecycleIsOnlyAppliedOnDataStreams(
+                    validateLifecycle(
                         tempStateWithComponentTemplateAdded.metadata(),
                         composableTemplateName,
-                        composableTemplate
+                        composableTemplate,
+                        globalRetentionResolver.resolve(currentState)
                     );
                     validateIndexTemplateV2(composableTemplateName, composableTemplate, tempStateWithComponentTemplateAdded);
                 } catch (Exception e) {
@@ -359,6 +363,12 @@ public ClusterState addComponentTemplate(
             }
         }
 
+        if (finalComponentTemplate.template().lifecycle() != null) {
+            finalComponentTemplate.template()
+                .lifecycle()
+                .addWarningHeaderIfDataRetentionNotEffective(globalRetentionResolver.resolve(currentState));
+        }
+
         logger.info("{} component template [{}]", existing == null ? "adding" : "updating", name);
         return ClusterState.builder(currentState)
             .metadata(Metadata.builder(currentState.metadata()).put(name, finalComponentTemplate))
@@ -715,7 +725,7 @@ private void validateIndexTemplateV2(String name, ComposableIndexTemplate indexT
 
         validate(name, templateToValidate);
         validateDataStreamsStillReferenced(currentState, name, templateToValidate);
-        validateLifecycleIsOnlyAppliedOnDataStreams(currentState.metadata(), name, templateToValidate);
+        validateLifecycle(currentState.metadata(), name, templateToValidate, globalRetentionResolver.resolve(currentState));
 
         if (templateToValidate.isDeprecated() == false) {
             validateUseOfDeprecatedComponentTemplates(name, templateToValidate, currentState.metadata().componentTemplates());
@@ -784,19 +794,25 @@ private void emitWarningIfPipelineIsDeprecated(String name, Map<String, Pipeline
             );
     }
 
-    private static void validateLifecycleIsOnlyAppliedOnDataStreams(
+    // Visible for testing
+    static void validateLifecycle(
         Metadata metadata,
         String indexTemplateName,
-        ComposableIndexTemplate template
+        ComposableIndexTemplate template,
+        @Nullable DataStreamGlobalRetention globalRetention
     ) {
-        boolean hasLifecycle = (template.template() != null && template.template().lifecycle() != null)
-            || resolveLifecycle(template, metadata.componentTemplates()) != null;
-        if (hasLifecycle && template.getDataStreamTemplate() == null) {
-            throw new IllegalArgumentException(
-                "index template ["
-                    + indexTemplateName
-                    + "] specifies lifecycle configuration that can only be used in combination with a data stream"
-            );
+        DataStreamLifecycle lifecycle = template.template() != null && template.template().lifecycle() != null
+            ? template.template().lifecycle()
+            : resolveLifecycle(template, metadata.componentTemplates());
+        if (lifecycle != null) {
+            if (template.getDataStreamTemplate() == null) {
+                throw new IllegalArgumentException(
+                    "index template ["
+                        + indexTemplateName
+                        + "] specifies lifecycle configuration that can only be used in combination with a data stream"
+                );
+            }
+            lifecycle.addWarningHeaderIfDataRetentionNotEffective(globalRetention);
         }
     }
 
diff --git a/server/src/main/java/org/elasticsearch/node/NodeConstruction.java b/server/src/main/java/org/elasticsearch/node/NodeConstruction.java
index 5f02275356756..14e8ce80fcf26 100644
--- a/server/src/main/java/org/elasticsearch/node/NodeConstruction.java
+++ b/server/src/main/java/org/elasticsearch/node/NodeConstruction.java
@@ -586,6 +586,27 @@ private ScriptService createScriptService(SettingsModule settingsModule, ThreadP
         return scriptService;
     }
 
+    private DataStreamGlobalRetentionResolver createDataStreamServicesAndGlobalRetentionResolver(
+        ThreadPool threadPool,
+        ClusterService clusterService,
+        IndicesService indicesService,
+        MetadataCreateIndexService metadataCreateIndexService
+    ) {
+        DataStreamGlobalRetentionResolver dataStreamGlobalRetentionResolver = new DataStreamGlobalRetentionResolver(
+            DataStreamFactoryRetention.load(pluginsService, clusterService.getClusterSettings())
+        );
+        modules.bindToInstance(DataStreamGlobalRetentionResolver.class, dataStreamGlobalRetentionResolver);
+        modules.bindToInstance(
+            MetadataCreateDataStreamService.class,
+            new MetadataCreateDataStreamService(threadPool, clusterService, metadataCreateIndexService)
+        );
+        modules.bindToInstance(
+            MetadataDataStreamsService.class,
+            new MetadataDataStreamsService(clusterService, indicesService, dataStreamGlobalRetentionResolver)
+        );
+        return dataStreamGlobalRetentionResolver;
+    }
+
     private UpdateHelper createUpdateHelper(DocumentParsingProvider documentParsingProvider, ScriptService scriptService) {
         UpdateHelper updateHelper = new UpdateHelper(scriptService, documentParsingProvider);
 
@@ -746,6 +767,7 @@ private void construct(
         );
 
         final ShardLimitValidator shardLimitValidator = new ShardLimitValidator(settings, clusterService);
+
         final MetadataCreateIndexService metadataCreateIndexService = new MetadataCreateIndexService(
             settings,
             clusterService,
@@ -761,12 +783,6 @@ private void construct(
             indexSettingProviders
         );
 
-        modules.bindToInstance(
-            MetadataCreateDataStreamService.class,
-            new MetadataCreateDataStreamService(threadPool, clusterService, metadataCreateIndexService)
-        );
-        modules.bindToInstance(MetadataDataStreamsService.class, new MetadataDataStreamsService(clusterService, indicesService));
-
         final MetadataUpdateSettingsService metadataUpdateSettingsService = new MetadataUpdateSettingsService(
             clusterService,
             clusterModule.getAllocationService(),
@@ -776,10 +792,12 @@ private void construct(
             threadPool
         );
 
-        DataStreamGlobalRetentionResolver dataStreamGlobalRetentionResolver = new DataStreamGlobalRetentionResolver(
-            DataStreamFactoryRetention.load(pluginsService, clusterService.getClusterSettings())
+        final DataStreamGlobalRetentionResolver dataStreamGlobalRetentionResolver = createDataStreamServicesAndGlobalRetentionResolver(
+            threadPool,
+            clusterService,
+            indicesService,
+            metadataCreateIndexService
         );
-        modules.bindToInstance(DataStreamGlobalRetentionResolver.class, dataStreamGlobalRetentionResolver);
 
         record PluginServiceInstances(
             Client client,
@@ -851,7 +869,8 @@ record PluginServiceInstances(
                 indicesService,
                 systemIndices,
                 indexSettingProviders,
-                metadataCreateIndexService
+                metadataCreateIndexService,
+                dataStreamGlobalRetentionResolver
             ),
             pluginsService.loadSingletonServiceProvider(RestExtension.class, RestExtension::allowAll)
         );
@@ -1426,7 +1445,8 @@ private List<ReservedClusterStateHandler<?>> buildReservedStateHandlers(
         IndicesService indicesService,
         SystemIndices systemIndices,
         IndexSettingProviders indexSettingProviders,
-        MetadataCreateIndexService metadataCreateIndexService
+        MetadataCreateIndexService metadataCreateIndexService,
+        DataStreamGlobalRetentionResolver globalRetentionResolver
     ) {
         List<ReservedClusterStateHandler<?>> reservedStateHandlers = new ArrayList<>();
 
@@ -1440,7 +1460,8 @@ private List<ReservedClusterStateHandler<?>> buildReservedStateHandlers(
             settingsModule.getIndexScopedSettings(),
             xContentRegistry,
             systemIndices,
-            indexSettingProviders
+            indexSettingProviders,
+            globalRetentionResolver
         );
         reservedStateHandlers.add(new ReservedComposableIndexTemplateAction(templateService, settingsModule.getIndexScopedSettings()));
 
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/template/reservedstate/ReservedComposableIndexTemplateActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/template/reservedstate/ReservedComposableIndexTemplateActionTests.java
index 84921effed9f5..8cd0bb02b0e7c 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/template/reservedstate/ReservedComposableIndexTemplateActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/template/reservedstate/ReservedComposableIndexTemplateActionTests.java
@@ -17,6 +17,8 @@
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
+import org.elasticsearch.cluster.metadata.DataStreamFactoryRetention;
+import org.elasticsearch.cluster.metadata.DataStreamGlobalRetentionResolver;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.metadata.MetadataCreateIndexService;
@@ -73,6 +75,7 @@ public class ReservedComposableIndexTemplateActionTests extends ESTestCase {
     ClusterService clusterService;
     IndexScopedSettings indexScopedSettings;
     IndicesService indicesService;
+    private DataStreamGlobalRetentionResolver globalRetentionResolver;
 
     @Before
     public void setup() throws IOException {
@@ -89,6 +92,7 @@ public void setup() throws IOException {
         doReturn(mapperService).when(indexService).mapperService();
         doReturn(indexService).when(indicesService).createIndex(any(), any(), anyBoolean());
 
+        globalRetentionResolver = new DataStreamGlobalRetentionResolver(DataStreamFactoryRetention.emptyFactoryRetention());
         templateService = new MetadataIndexTemplateService(
             clusterService,
             mock(MetadataCreateIndexService.class),
@@ -96,7 +100,8 @@ public void setup() throws IOException {
             indexScopedSettings,
             mock(NamedXContentRegistry.class),
             mock(SystemIndices.class),
-            new IndexSettingProviders(Set.of())
+            new IndexSettingProviders(Set.of()),
+            globalRetentionResolver
         );
     }
 
@@ -890,7 +895,8 @@ public void testTemplatesWithReservedPrefix() throws Exception {
             indexScopedSettings,
             mock(NamedXContentRegistry.class),
             mock(SystemIndices.class),
-            new IndexSettingProviders(Set.of())
+            new IndexSettingProviders(Set.of()),
+            globalRetentionResolver
         );
 
         ClusterState state = ClusterState.builder(new ClusterName("elasticsearch")).metadata(metadata).build();
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleTests.java
index 38b09f3690870..f6f915b0e1a3d 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleTests.java
@@ -32,6 +32,7 @@
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Set;
+import java.util.concurrent.TimeUnit;
 import java.util.stream.Stream;
 
 import static org.elasticsearch.cluster.metadata.DataStreamLifecycle.RetentionSource.DATA_STREAM_CONFIGURATION;
@@ -356,12 +357,12 @@ private static DataStreamLifecycle.Retention randomRetention() {
         return switch (randomInt(2)) {
             case 0 -> null;
             case 1 -> DataStreamLifecycle.Retention.NULL;
-            default -> new DataStreamLifecycle.Retention(TimeValue.timeValueMillis(randomMillisUpToYear9999()));
+            default -> new DataStreamLifecycle.Retention(randomTimeValue(1, 365, TimeUnit.DAYS));
         };
     }
 
     @Nullable
-    private static DataStreamLifecycle.Downsampling randomDownsampling() {
+    static DataStreamLifecycle.Downsampling randomDownsampling() {
         return switch (randomInt(2)) {
             case 0 -> null;
             case 1 -> DataStreamLifecycle.Downsampling.NULL;
@@ -369,7 +370,7 @@ private static DataStreamLifecycle.Downsampling randomDownsampling() {
                 var count = randomIntBetween(0, 9);
                 List<DataStreamLifecycle.Downsampling.Round> rounds = new ArrayList<>();
                 var previous = new DataStreamLifecycle.Downsampling.Round(
-                    TimeValue.timeValueDays(randomIntBetween(1, 365)),
+                    randomTimeValue(1, 365, TimeUnit.DAYS),
                     new DownsampleConfig(new DateHistogramInterval(randomIntBetween(1, 24) + "h"))
                 );
                 rounds.add(previous);
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleWithRetentionWarningsTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleWithRetentionWarningsTests.java
new file mode 100644
index 0000000000000..7e338c52a0a17
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleWithRetentionWarningsTests.java
@@ -0,0 +1,281 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.cluster.metadata;
+
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.logging.HeaderWarning;
+import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.settings.IndexScopedSettings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.core.Tuple;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.IndexService;
+import org.elasticsearch.index.IndexSettingProviders;
+import org.elasticsearch.indices.EmptySystemIndices;
+import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+
+import static org.elasticsearch.cluster.metadata.DataStreamLifecycleTests.randomDownsampling;
+import static org.elasticsearch.common.settings.Settings.builder;
+import static org.elasticsearch.indices.ShardLimitValidatorTests.createTestShardLimitService;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * We test the warnings added when user configured retention exceeds the global retention in this test,
+ * so we can disable the warning check without impacting all the other test cases
+ */
+public class DataStreamLifecycleWithRetentionWarningsTests extends ESTestCase {
+    @Override
+    protected boolean enableWarningsCheck() {
+        // this test expects warnings
+        return false;
+    }
+
+    public void testNoHeaderWarning() {
+        ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
+        HeaderWarning.setThreadContext(threadContext);
+
+        DataStreamLifecycle noRetentionLifecycle = DataStreamLifecycle.newBuilder().downsampling(randomDownsampling()).build();
+        noRetentionLifecycle.addWarningHeaderIfDataRetentionNotEffective(null);
+        Map<String, List<String>> responseHeaders = threadContext.getResponseHeaders();
+        assertThat(responseHeaders.isEmpty(), is(true));
+
+        TimeValue dataStreamRetention = TimeValue.timeValueDays(randomIntBetween(5, 100));
+        DataStreamLifecycle lifecycleWithRetention = DataStreamLifecycle.newBuilder()
+            .dataRetention(dataStreamRetention)
+            .downsampling(randomDownsampling())
+            .build();
+        DataStreamGlobalRetention globalRetention = new DataStreamGlobalRetention(
+            TimeValue.timeValueDays(2),
+            TimeValue.timeValueDays(dataStreamRetention.days() + randomIntBetween(1, 5))
+        );
+        lifecycleWithRetention.addWarningHeaderIfDataRetentionNotEffective(globalRetention);
+        responseHeaders = threadContext.getResponseHeaders();
+        assertThat(responseHeaders.isEmpty(), is(true));
+    }
+
+    public void testDefaultRetentionHeaderWarning() {
+        ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
+        HeaderWarning.setThreadContext(threadContext);
+
+        DataStreamLifecycle noRetentionLifecycle = DataStreamLifecycle.newBuilder().downsampling(randomDownsampling()).build();
+        DataStreamGlobalRetention globalRetention = new DataStreamGlobalRetention(
+            randomTimeValue(2, 10, TimeUnit.DAYS),
+            randomBoolean() ? null : TimeValue.timeValueDays(20)
+        );
+        noRetentionLifecycle.addWarningHeaderIfDataRetentionNotEffective(globalRetention);
+        Map<String, List<String>> responseHeaders = threadContext.getResponseHeaders();
+        assertThat(responseHeaders.size(), is(1));
+        assertThat(
+            responseHeaders.get("Warning").get(0),
+            containsString(
+                "Not providing a retention is not allowed for this project. The default retention of ["
+                    + globalRetention.getDefaultRetention().getStringRep()
+                    + "] will be applied."
+            )
+        );
+    }
+
+    public void testMaxRetentionHeaderWarning() {
+        ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
+        HeaderWarning.setThreadContext(threadContext);
+        TimeValue maxRetention = randomTimeValue(2, 100, TimeUnit.DAYS);
+        DataStreamLifecycle lifecycle = DataStreamLifecycle.newBuilder()
+            .dataRetention(randomBoolean() ? null : TimeValue.timeValueDays(maxRetention.days() + 1))
+            .downsampling(randomDownsampling())
+            .build();
+        DataStreamGlobalRetention globalRetention = new DataStreamGlobalRetention(null, maxRetention);
+        lifecycle.addWarningHeaderIfDataRetentionNotEffective(globalRetention);
+        Map<String, List<String>> responseHeaders = threadContext.getResponseHeaders();
+        assertThat(responseHeaders.size(), is(1));
+        String userRetentionPart = lifecycle.getDataStreamRetention() == null
+            ? "Not providing a retention is not allowed for this project."
+            : "The retention provided ["
+                + lifecycle.getDataStreamRetention().getStringRep()
+                + "] is exceeding the max allowed data retention of this project ["
+                + maxRetention.getStringRep()
+                + "].";
+        assertThat(
+            responseHeaders.get("Warning").get(0),
+            containsString(userRetentionPart + " The max retention of [" + maxRetention.getStringRep() + "] will be applied")
+        );
+    }
+
+    public void testUpdatingLifecycleOnADataStream() {
+        ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
+        HeaderWarning.setThreadContext(threadContext);
+        String dataStream = randomAlphaOfLength(5);
+        TimeValue defaultRetention = randomTimeValue(2, 100, TimeUnit.DAYS);
+
+        DataStreamFactoryRetention factoryRetention;
+        ClusterState before = ClusterState.builder(
+            DataStreamTestHelper.getClusterStateWithDataStreams(List.of(new Tuple<>(dataStream, 2)), List.of())
+        ).build();
+        if (randomBoolean()) {
+            factoryRetention = DataStreamFactoryRetention.emptyFactoryRetention();
+            before = ClusterState.builder(before)
+                .putCustom(DataStreamGlobalRetention.TYPE, new DataStreamGlobalRetention(defaultRetention, null))
+                .build();
+        } else {
+            factoryRetention = getDefaultFactoryRetention(defaultRetention);
+        }
+
+        MetadataDataStreamsService metadataDataStreamsService = new MetadataDataStreamsService(
+            mock(ClusterService.class),
+            mock(IndicesService.class),
+            new DataStreamGlobalRetentionResolver(factoryRetention)
+        );
+
+        ClusterState after = metadataDataStreamsService.updateDataLifecycle(before, List.of(dataStream), DataStreamLifecycle.DEFAULT);
+        DataStream updatedDataStream = after.metadata().dataStreams().get(dataStream);
+        assertNotNull(updatedDataStream);
+        assertThat(updatedDataStream.getLifecycle(), equalTo(DataStreamLifecycle.DEFAULT));
+        Map<String, List<String>> responseHeaders = threadContext.getResponseHeaders();
+        assertThat(responseHeaders.size(), is(1));
+        assertThat(
+            responseHeaders.get("Warning").get(0),
+            containsString(
+                "Not providing a retention is not allowed for this project. The default retention of ["
+                    + defaultRetention.getStringRep()
+                    + "] will be applied."
+            )
+        );
+    }
+
+    public void testValidateLifecycleIndexTemplateWithWarning() {
+        ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
+        HeaderWarning.setThreadContext(threadContext);
+        TimeValue defaultRetention = randomTimeValue(2, 100, TimeUnit.DAYS);
+        MetadataIndexTemplateService.validateLifecycle(
+            Metadata.builder().build(),
+            randomAlphaOfLength(10),
+            ComposableIndexTemplate.builder()
+                .template(new Template(null, null, null, DataStreamLifecycle.DEFAULT))
+                .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate())
+                .indexPatterns(List.of(randomAlphaOfLength(10)))
+                .build(),
+            new DataStreamGlobalRetention(defaultRetention, null)
+        );
+        Map<String, List<String>> responseHeaders = threadContext.getResponseHeaders();
+        assertThat(responseHeaders.size(), is(1));
+        assertThat(
+            responseHeaders.get("Warning").get(0),
+            containsString(
+                "Not providing a retention is not allowed for this project. The default retention of ["
+                    + defaultRetention.getStringRep()
+                    + "] will be applied."
+            )
+        );
+    }
+
+    public void testValidateLifecycleInComponentTemplate() throws Exception {
+        IndicesService indicesService = mock(IndicesService.class);
+        IndexService indexService = mock(IndexService.class);
+        when(indicesService.createIndex(any(), any(), eq(false))).thenReturn(indexService);
+        when(indexService.index()).thenReturn(new Index(randomAlphaOfLength(10), randomUUID()));
+        ClusterService clusterService = mock(ClusterService.class);
+        MetadataCreateIndexService createIndexService = new MetadataCreateIndexService(
+            Settings.EMPTY,
+            clusterService,
+            indicesService,
+            null,
+            createTestShardLimitService(randomIntBetween(1, 1000)),
+            new Environment(builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()).build(), null),
+            IndexScopedSettings.DEFAULT_SCOPED_SETTINGS,
+            null,
+            xContentRegistry(),
+            EmptySystemIndices.INSTANCE,
+            true,
+            new IndexSettingProviders(Set.of())
+        );
+        TimeValue defaultRetention = randomTimeValue(2, 100, TimeUnit.DAYS);
+        DataStreamFactoryRetention factoryRetention;
+        ClusterState state;
+        if (randomBoolean()) {
+            factoryRetention = DataStreamFactoryRetention.emptyFactoryRetention();
+            state = ClusterState.builder(ClusterName.DEFAULT)
+                .putCustom(DataStreamGlobalRetention.TYPE, new DataStreamGlobalRetention(defaultRetention, null))
+                .build();
+        } else {
+            state = ClusterState.EMPTY_STATE;
+            factoryRetention = getDefaultFactoryRetention(defaultRetention);
+        }
+        MetadataIndexTemplateService metadataIndexTemplateService = new MetadataIndexTemplateService(
+            clusterService,
+            createIndexService,
+            indicesService,
+            new IndexScopedSettings(Settings.EMPTY, IndexScopedSettings.BUILT_IN_INDEX_SETTINGS),
+            xContentRegistry(),
+            EmptySystemIndices.INSTANCE,
+            new IndexSettingProviders(Set.of()),
+            new DataStreamGlobalRetentionResolver(factoryRetention)
+        );
+
+        ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
+        HeaderWarning.setThreadContext(threadContext);
+
+        Template template = new Template(
+            ComponentTemplateTests.randomSettings(),
+            null,
+            ComponentTemplateTests.randomAliases(),
+            DataStreamLifecycle.DEFAULT
+        );
+        ComponentTemplate componentTemplate = new ComponentTemplate(template, 1L, new HashMap<>());
+        state = metadataIndexTemplateService.addComponentTemplate(state, false, "foo", componentTemplate);
+
+        assertNotNull(state.metadata().componentTemplates().get("foo"));
+        assertThat(state.metadata().componentTemplates().get("foo"), equalTo(componentTemplate));
+        Map<String, List<String>> responseHeaders = threadContext.getResponseHeaders();
+        assertThat(responseHeaders.size(), is(1));
+        assertThat(
+            responseHeaders.get("Warning").get(0),
+            containsString(
+                "Not providing a retention is not allowed for this project. The default retention of ["
+                    + defaultRetention.getStringRep()
+                    + "] will be applied."
+            )
+        );
+    }
+
+    private DataStreamFactoryRetention getDefaultFactoryRetention(TimeValue defaultRetention) {
+        return new DataStreamFactoryRetention() {
+            @Override
+            public TimeValue getMaxRetention() {
+                return null;
+            }
+
+            @Override
+            public TimeValue getDefaultRetention() {
+                return defaultRetention;
+            }
+
+            @Override
+            public void init(ClusterSettings clusterSettings) {
+
+            }
+        };
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsServiceTests.java
index 3d9368ddb9fc9..9a560abe20c74 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsServiceTests.java
@@ -10,6 +10,7 @@
 
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.Tuple;
@@ -17,6 +18,7 @@
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.MapperServiceTestCase;
+import org.elasticsearch.indices.IndicesService;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -30,6 +32,7 @@
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
+import static org.mockito.Mockito.mock;
 
 public class MetadataDataStreamsServiceTests extends MapperServiceTestCase {
 
@@ -389,9 +392,14 @@ public void testUpdateLifecycle() {
         String dataStream = randomAlphaOfLength(5);
         DataStreamLifecycle lifecycle = DataStreamLifecycle.newBuilder().dataRetention(randomMillisUpToYear9999()).build();
         ClusterState before = DataStreamTestHelper.getClusterStateWithDataStreams(List.of(new Tuple<>(dataStream, 2)), List.of());
+        MetadataDataStreamsService service = new MetadataDataStreamsService(
+            mock(ClusterService.class),
+            mock(IndicesService.class),
+            new DataStreamGlobalRetentionResolver(DataStreamFactoryRetention.emptyFactoryRetention())
+        );
         {
             // Remove lifecycle
-            ClusterState after = MetadataDataStreamsService.updateDataLifecycle(before, List.of(dataStream), null);
+            ClusterState after = service.updateDataLifecycle(before, List.of(dataStream), null);
             DataStream updatedDataStream = after.metadata().dataStreams().get(dataStream);
             assertNotNull(updatedDataStream);
             assertThat(updatedDataStream.getLifecycle(), nullValue());
@@ -400,7 +408,7 @@ public void testUpdateLifecycle() {
 
         {
             // Set lifecycle
-            ClusterState after = MetadataDataStreamsService.updateDataLifecycle(before, List.of(dataStream), lifecycle);
+            ClusterState after = service.updateDataLifecycle(before, List.of(dataStream), lifecycle);
             DataStream updatedDataStream = after.metadata().dataStreams().get(dataStream);
             assertNotNull(updatedDataStream);
             assertThat(updatedDataStream.getLifecycle(), equalTo(lifecycle));
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java
index 84b6feb1dbffa..8aa57f545ac1d 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java
@@ -2483,7 +2483,8 @@ private static List<Throwable> putTemplate(NamedXContentRegistry xContentRegistr
             new IndexScopedSettings(Settings.EMPTY, IndexScopedSettings.BUILT_IN_INDEX_SETTINGS),
             xContentRegistry,
             EmptySystemIndices.INSTANCE,
-            new IndexSettingProviders(Set.of())
+            new IndexSettingProviders(Set.of()),
+            new DataStreamGlobalRetentionResolver(DataStreamFactoryRetention.emptyFactoryRetention())
         );
 
         final List<Throwable> throwables = new ArrayList<>();
@@ -2525,6 +2526,9 @@ public void onFailure(Exception e) {
     private MetadataIndexTemplateService getMetadataIndexTemplateService() {
         IndicesService indicesService = getInstanceFromNode(IndicesService.class);
         ClusterService clusterService = getInstanceFromNode(ClusterService.class);
+        DataStreamGlobalRetentionResolver dataStreamGlobalRetentionResolver = new DataStreamGlobalRetentionResolver(
+            DataStreamFactoryRetention.emptyFactoryRetention()
+        );
         MetadataCreateIndexService createIndexService = new MetadataCreateIndexService(
             Settings.EMPTY,
             clusterService,
@@ -2546,7 +2550,8 @@ private MetadataIndexTemplateService getMetadataIndexTemplateService() {
             new IndexScopedSettings(Settings.EMPTY, IndexScopedSettings.BUILT_IN_INDEX_SETTINGS),
             xContentRegistry(),
             EmptySystemIndices.INSTANCE,
-            new IndexSettingProviders(Set.of())
+            new IndexSettingProviders(Set.of()),
+            dataStreamGlobalRetentionResolver
         );
     }
 

From 638a45009cb76059dd914c70eb4aa7e777768da2 Mon Sep 17 00:00:00 2001
From: Simon Cooper <simon.cooper@elastic.co>
Date: Fri, 26 Apr 2024 09:30:46 +0100
Subject: [PATCH 42/58] Update several references to TransportVersion.toString
 to use toReleaseVersion (#107902)

---
 docs/changelog/107902.yaml                       |  5 +++++
 .../script/mustache/MultiSearchTemplateIT.java   |  4 ++--
 .../script/mustache/SearchTemplateIT.java        | 12 ++++++++----
 .../features/ResetFeatureStateRequest.java       | 14 --------------
 .../resolve/ResolveClusterActionRequest.java     |  8 ++++----
 .../resolve/ResolveClusterActionResponse.java    |  4 ++--
 .../indices/resolve/ResolveClusterInfo.java      |  4 ++--
 .../shards/IndicesShardStoresRequest.java        |  2 +-
 .../action/search/TransportSearchHelper.java     |  2 +-
 .../cluster/coordination/NodeJoinExecutor.java   |  6 +++---
 .../PublicationTransportHandler.java             |  6 +++++-
 .../io/stream/VersionCheckingStreamOutput.java   |  4 ++--
 .../java/org/elasticsearch/search/SearchHit.java |  2 +-
 .../search/builder/SearchSourceBuilder.java      | 12 ++++++++----
 .../search/dfs/DfsSearchResult.java              |  6 ++++--
 .../search/query/QuerySearchResult.java          |  2 +-
 .../elasticsearch/transport/InboundDecoder.java  |  4 ++--
 .../stream/VersionCheckingStreamOutputTests.java |  4 ++--
 .../transport/InboundDecoderTests.java           |  4 ++--
 .../FailBeforeCurrentVersionQueryBuilder.java    |  4 ++--
 .../common/validation/SourceDestValidator.java   |  7 +++----
 .../action/user/GetUserPrivilegesResponse.java   |  4 ++--
 .../core/security/authc/Authentication.java      | 12 ++++++------
 ...moteClusterMinimumVersionValidationTests.java |  8 ++++----
 .../user/GetUserPrivilegesResponseTests.java     |  4 ++--
 .../termsenum/TransportTermsEnumActionTests.java |  4 +++-
 .../ml/action/TransportStartDatafeedAction.java  |  2 +-
 .../TransportStartDatafeedActionTests.java       |  2 +-
 .../xpack/security/authc/ApiKeyService.java      | 16 ++++++++--------
 .../CrossClusterAccessAuthenticationService.java |  4 ++--
 .../security/authz/store/NativeRolesStore.java   |  4 ++--
 .../SecurityServerTransportInterceptor.java      |  2 +-
 .../xpack/security/authc/ApiKeyServiceTests.java | 12 ++++++------
 ...sClusterAccessAuthenticationServiceTests.java |  4 ++--
 .../authz/store/NativeRolesStoreTests.java       |  4 ++--
 .../SecurityServerTransportInterceptorTests.java |  2 +-
 .../blobstore/testkit/BlobAnalyzeAction.java     |  4 +++-
 .../testkit/RepositoryAnalyzeAction.java         |  6 +++---
 .../upgrades/ApiKeyBackwardsCompatibilityIT.java |  8 ++++----
 39 files changed, 113 insertions(+), 105 deletions(-)
 create mode 100644 docs/changelog/107902.yaml

diff --git a/docs/changelog/107902.yaml b/docs/changelog/107902.yaml
new file mode 100644
index 0000000000000..6b25f8c12df60
--- /dev/null
+++ b/docs/changelog/107902.yaml
@@ -0,0 +1,5 @@
+pr: 107902
+summary: Update several references to `TransportVersion.toString` to use `toReleaseVersion`
+area: Infra/Core
+type: bug
+issues: []
diff --git a/modules/lang-mustache/src/internalClusterTest/java/org/elasticsearch/script/mustache/MultiSearchTemplateIT.java b/modules/lang-mustache/src/internalClusterTest/java/org/elasticsearch/script/mustache/MultiSearchTemplateIT.java
index 0c3376c9c8a90..c01f6ef46d0c4 100644
--- a/modules/lang-mustache/src/internalClusterTest/java/org/elasticsearch/script/mustache/MultiSearchTemplateIT.java
+++ b/modules/lang-mustache/src/internalClusterTest/java/org/elasticsearch/script/mustache/MultiSearchTemplateIT.java
@@ -208,8 +208,8 @@ public void testCCSCheckCompatibility() throws Exception {
             String expectedCause = Strings.format(
                 "[fail_before_current_version] was released first in version %s, failed compatibility "
                     + "check trying to send it to node with version %s",
-                FailBeforeCurrentVersionQueryBuilder.FUTURE_VERSION,
-                TransportVersions.MINIMUM_CCS_VERSION
+                FailBeforeCurrentVersionQueryBuilder.FUTURE_VERSION.toReleaseVersion(),
+                TransportVersions.MINIMUM_CCS_VERSION.toReleaseVersion()
             );
             String actualCause = ex.getCause().getMessage();
             assertEquals(expectedCause, actualCause);
diff --git a/modules/lang-mustache/src/internalClusterTest/java/org/elasticsearch/script/mustache/SearchTemplateIT.java b/modules/lang-mustache/src/internalClusterTest/java/org/elasticsearch/script/mustache/SearchTemplateIT.java
index 510ff01cf93f7..e17fb4b26cd28 100644
--- a/modules/lang-mustache/src/internalClusterTest/java/org/elasticsearch/script/mustache/SearchTemplateIT.java
+++ b/modules/lang-mustache/src/internalClusterTest/java/org/elasticsearch/script/mustache/SearchTemplateIT.java
@@ -37,6 +37,7 @@
 import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.matchesRegex;
 
 /**
  * Full integration test of the template query plugin.
@@ -441,10 +442,13 @@ public void testCCSCheckCompatibility() throws Exception {
         );
         assertThat(primary.getMessage(), containsString("'search.check_ccs_compatibility' setting is enabled."));
 
-        String expectedCause = "[fail_before_current_version] was released first in version XXXXXXX, failed compatibility check trying to"
-            + " send it to node with version XXXXXXX";
-        String actualCause = underlying.getMessage().replaceAll("\\d{7,}", "XXXXXXX");
-        assertEquals(expectedCause, actualCause);
+        assertThat(
+            underlying.getMessage(),
+            matchesRegex(
+                "\\[fail_before_current_version] was released first in version .+,"
+                    + " failed compatibility check trying to send it to node with version .+"
+            )
+        );
     }
 
     public static void assertHitCount(SearchTemplateRequestBuilder requestBuilder, long expectedHitCount) {
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/ResetFeatureStateRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/ResetFeatureStateRequest.java
index 2c6769f5edd57..5a7bd2aee3619 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/ResetFeatureStateRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/ResetFeatureStateRequest.java
@@ -8,8 +8,6 @@
 
 package org.elasticsearch.action.admin.cluster.snapshots.features;
 
-import org.elasticsearch.TransportVersion;
-import org.elasticsearch.TransportVersions;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.support.master.MasterNodeRequest;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -20,14 +18,7 @@
 /** Request for resetting feature state */
 public class ResetFeatureStateRequest extends MasterNodeRequest<ResetFeatureStateRequest> {
 
-    private static final TransportVersion FEATURE_RESET_ON_MASTER = TransportVersions.V_7_14_0;
-
     public static ResetFeatureStateRequest fromStream(StreamInput in) throws IOException {
-        if (in.getTransportVersion().before(FEATURE_RESET_ON_MASTER)) {
-            throw new IllegalStateException(
-                "feature reset is not available in a cluster that have nodes with version before " + FEATURE_RESET_ON_MASTER
-            );
-        }
         return new ResetFeatureStateRequest(in);
     }
 
@@ -39,11 +30,6 @@ private ResetFeatureStateRequest(StreamInput in) throws IOException {
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        if (out.getTransportVersion().before(FEATURE_RESET_ON_MASTER)) {
-            throw new IllegalStateException(
-                "feature reset is not available in a cluster that have nodes with version before " + FEATURE_RESET_ON_MASTER
-            );
-        }
         super.writeTo(out);
     }
 
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveClusterActionRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveClusterActionRequest.java
index 858202d316d89..1649e4587d63c 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveClusterActionRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveClusterActionRequest.java
@@ -65,9 +65,9 @@ public ResolveClusterActionRequest(StreamInput in) throws IOException {
         if (in.getTransportVersion().before(TransportVersions.RESOLVE_CLUSTER_ENDPOINT_ADDED)) {
             throw new UnsupportedOperationException(
                 "ResolveClusterAction requires at least Transport Version "
-                    + TransportVersions.RESOLVE_CLUSTER_ENDPOINT_ADDED
+                    + TransportVersions.RESOLVE_CLUSTER_ENDPOINT_ADDED.toReleaseVersion()
                     + " but was "
-                    + in.getTransportVersion()
+                    + in.getTransportVersion().toReleaseVersion()
             );
         }
         this.names = in.readStringArray();
@@ -81,9 +81,9 @@ public void writeTo(StreamOutput out) throws IOException {
         if (out.getTransportVersion().before(TransportVersions.RESOLVE_CLUSTER_ENDPOINT_ADDED)) {
             throw new UnsupportedOperationException(
                 "ResolveClusterAction requires at least Transport Version "
-                    + TransportVersions.RESOLVE_CLUSTER_ENDPOINT_ADDED
+                    + TransportVersions.RESOLVE_CLUSTER_ENDPOINT_ADDED.toReleaseVersion()
                     + " but was "
-                    + out.getTransportVersion()
+                    + out.getTransportVersion().toReleaseVersion()
             );
         }
         out.writeStringArray(names);
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveClusterActionResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveClusterActionResponse.java
index 87439796f78f4..ee2e3d60dc56e 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveClusterActionResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveClusterActionResponse.java
@@ -47,9 +47,9 @@ public void writeTo(StreamOutput out) throws IOException {
         if (out.getTransportVersion().before(TransportVersions.RESOLVE_CLUSTER_ENDPOINT_ADDED)) {
             throw new UnsupportedOperationException(
                 "ResolveClusterAction requires at least Transport Version "
-                    + TransportVersions.RESOLVE_CLUSTER_ENDPOINT_ADDED
+                    + TransportVersions.RESOLVE_CLUSTER_ENDPOINT_ADDED.toReleaseVersion()
                     + " but was "
-                    + out.getTransportVersion()
+                    + out.getTransportVersion().toReleaseVersion()
             );
         }
         out.writeMap(infoMap, StreamOutput::writeWriteable);
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveClusterInfo.java b/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveClusterInfo.java
index 37dc99c95f10d..578b4ae547a06 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveClusterInfo.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveClusterInfo.java
@@ -68,9 +68,9 @@ public void writeTo(StreamOutput out) throws IOException {
         if (out.getTransportVersion().before(TransportVersions.RESOLVE_CLUSTER_ENDPOINT_ADDED)) {
             throw new UnsupportedOperationException(
                 "ResolveClusterAction requires at least Transport Version "
-                    + TransportVersions.RESOLVE_CLUSTER_ENDPOINT_ADDED
+                    + TransportVersions.RESOLVE_CLUSTER_ENDPOINT_ADDED.toReleaseVersion()
                     + " but was "
-                    + out.getTransportVersion()
+                    + out.getTransportVersion().toReleaseVersion()
             );
         }
         out.writeBoolean(connected);
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoresRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoresRequest.java
index 0ff478365cb53..475c9c16f149e 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoresRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoresRequest.java
@@ -75,7 +75,7 @@ public void writeTo(StreamOutput out) throws IOException {
                 "support for maxConcurrentShardRequests=["
                     + maxConcurrentShardRequests
                     + "] was added in version [8.8.0], cannot send this request using transport version ["
-                    + out.getTransportVersion()
+                    + out.getTransportVersion().toReleaseVersion()
                     + "]"
             );
         } // else just drop the value and use the default behaviour
diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportSearchHelper.java b/server/src/main/java/org/elasticsearch/action/search/TransportSearchHelper.java
index ffaecedb62bba..7cf48245c9a94 100644
--- a/server/src/main/java/org/elasticsearch/action/search/TransportSearchHelper.java
+++ b/server/src/main/java/org/elasticsearch/action/search/TransportSearchHelper.java
@@ -139,7 +139,7 @@ public static void checkCCSVersionCompatibility(Writeable writeableRequest) {
                 "["
                     + writeableRequest.getClass()
                     + "] is not compatible with version "
-                    + TransportVersions.MINIMUM_CCS_VERSION
+                    + TransportVersions.MINIMUM_CCS_VERSION.toReleaseVersion()
                     + " and the '"
                     + SearchService.CCS_VERSION_CHECK_SETTING.getKey()
                     + "' setting is enabled.",
diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/NodeJoinExecutor.java b/server/src/main/java/org/elasticsearch/cluster/coordination/NodeJoinExecutor.java
index c35ea1279ac99..2c024063e2399 100644
--- a/server/src/main/java/org/elasticsearch/cluster/coordination/NodeJoinExecutor.java
+++ b/server/src/main/java/org/elasticsearch/cluster/coordination/NodeJoinExecutor.java
@@ -341,7 +341,7 @@ public boolean runOnlyOnMaster() {
     private static void blockForbiddenVersions(TransportVersion joiningTransportVersion) {
         if (FORBIDDEN_VERSIONS.contains(joiningTransportVersion)) {
             throw new IllegalStateException(
-                "A node with transport version " + joiningTransportVersion + " is forbidden from joining this cluster"
+                "A node with transport version " + joiningTransportVersion.toReleaseVersion() + " is forbidden from joining this cluster"
             );
         }
     }
@@ -427,9 +427,9 @@ static void ensureTransportVersionBarrier(
         if (joiningCompatibilityVersions.transportVersion().before(minClusterTransportVersion)) {
             throw new IllegalStateException(
                 "node with transport version ["
-                    + joiningCompatibilityVersions.transportVersion()
+                    + joiningCompatibilityVersions.transportVersion().toReleaseVersion()
                     + "] may not join a cluster with minimum transport version ["
-                    + minClusterTransportVersion
+                    + minClusterTransportVersion.toReleaseVersion()
                     + "]"
             );
         }
diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/PublicationTransportHandler.java b/server/src/main/java/org/elasticsearch/cluster/coordination/PublicationTransportHandler.java
index 5c5c5eee17da3..d8bf85fc02b37 100644
--- a/server/src/main/java/org/elasticsearch/cluster/coordination/PublicationTransportHandler.java
+++ b/server/src/main/java/org/elasticsearch/cluster/coordination/PublicationTransportHandler.java
@@ -456,7 +456,11 @@ private void sendClusterStateDiff(DiscoveryNode destination, ActionListener<Publ
 
             final ReleasableBytesReference bytes = serializedDiffs.get(connection.getTransportVersion());
             assert bytes != null
-                : "failed to find serialized diff for node " + destination + " of version [" + connection.getTransportVersion() + "]";
+                : "failed to find serialized diff for node "
+                    + destination
+                    + " of version ["
+                    + connection.getTransportVersion().toReleaseVersion()
+                    + "]";
 
             // acquire a ref to the context just in case we need to try again with the full cluster state
             if (tryIncRef() == false) {
diff --git a/server/src/main/java/org/elasticsearch/common/io/stream/VersionCheckingStreamOutput.java b/server/src/main/java/org/elasticsearch/common/io/stream/VersionCheckingStreamOutput.java
index 42fb7f4a6afe4..5f81f7bd97719 100644
--- a/server/src/main/java/org/elasticsearch/common/io/stream/VersionCheckingStreamOutput.java
+++ b/server/src/main/java/org/elasticsearch/common/io/stream/VersionCheckingStreamOutput.java
@@ -67,9 +67,9 @@ private void checkVersionCompatibility(VersionedNamedWriteable namedWriteable) {
                 "["
                     + namedWriteable.getWriteableName()
                     + "] was released first in version "
-                    + namedWriteable.getMinimalSupportedVersion()
+                    + namedWriteable.getMinimalSupportedVersion().toReleaseVersion()
                     + ", failed compatibility check trying to send it to node with version "
-                    + getTransportVersion()
+                    + getTransportVersion().toReleaseVersion()
             );
         }
     }
diff --git a/server/src/main/java/org/elasticsearch/search/SearchHit.java b/server/src/main/java/org/elasticsearch/search/SearchHit.java
index 62a62fb93a4a0..92875071dd041 100644
--- a/server/src/main/java/org/elasticsearch/search/SearchHit.java
+++ b/server/src/main/java/org/elasticsearch/search/SearchHit.java
@@ -307,7 +307,7 @@ public void writeTo(StreamOutput out) throws IOException {
         if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_8_0)) {
             out.writeVInt(rank);
         } else if (rank != NO_RANK) {
-            throw new IllegalArgumentException("cannot serialize [rank] to version [" + out.getTransportVersion() + "]");
+            throw new IllegalArgumentException("cannot serialize [rank] to version [" + out.getTransportVersion().toReleaseVersion() + "]");
         }
         out.writeOptionalText(id);
         if (out.getTransportVersion().before(TransportVersions.V_8_0_0)) {
diff --git a/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java b/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java
index 741a0e680b522..cb2c53a97fbc3 100644
--- a/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java
@@ -299,7 +299,9 @@ public void writeTo(StreamOutput out) throws IOException {
         if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_9_X)) {
             out.writeCollection(subSearchSourceBuilders);
         } else if (out.getTransportVersion().before(TransportVersions.V_8_4_0) && subSearchSourceBuilders.size() >= 2) {
-            throw new IllegalArgumentException("cannot serialize [sub_searches] to version [" + out.getTransportVersion() + "]");
+            throw new IllegalArgumentException(
+                "cannot serialize [sub_searches] to version [" + out.getTransportVersion().toReleaseVersion() + "]"
+            );
         } else {
             out.writeOptionalNamedWriteable(query());
         }
@@ -346,8 +348,10 @@ public void writeTo(StreamOutput out) throws IOException {
             if (out.getTransportVersion().before(TransportVersions.V_8_7_0)) {
                 if (knnSearch.size() > 1) {
                     throw new IllegalArgumentException(
-                        "Versions before 8070099 don't support multiple [knn] search clauses and search was sent to ["
-                            + out.getTransportVersion()
+                        "Versions before ["
+                            + TransportVersions.V_8_7_0.toReleaseVersion()
+                            + "] don't support multiple [knn] search clauses and search was sent to ["
+                            + out.getTransportVersion().toReleaseVersion()
                             + "]"
                     );
                 }
@@ -359,7 +363,7 @@ public void writeTo(StreamOutput out) throws IOException {
         if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_8_0)) {
             out.writeOptionalNamedWriteable(rankBuilder);
         } else if (rankBuilder != null) {
-            throw new IllegalArgumentException("cannot serialize [rank] to version [" + out.getTransportVersion() + "]");
+            throw new IllegalArgumentException("cannot serialize [rank] to version [" + out.getTransportVersion().toReleaseVersion() + "]");
         }
     }
 
diff --git a/server/src/main/java/org/elasticsearch/search/dfs/DfsSearchResult.java b/server/src/main/java/org/elasticsearch/search/dfs/DfsSearchResult.java
index b9cbd5baa1ebc..b76912e8b742a 100644
--- a/server/src/main/java/org/elasticsearch/search/dfs/DfsSearchResult.java
+++ b/server/src/main/java/org/elasticsearch/search/dfs/DfsSearchResult.java
@@ -141,8 +141,10 @@ public void writeTo(StreamOutput out) throws IOException {
                 if (knnResults != null && knnResults.size() > 1) {
                     throw new IllegalArgumentException(
                         "Cannot serialize multiple KNN results to nodes using previous transport version ["
-                            + out.getTransportVersion()
-                            + "], minimum required transport version is [8070099]"
+                            + out.getTransportVersion().toReleaseVersion()
+                            + "], minimum required transport version is ["
+                            + TransportVersions.V_8_7_0.toReleaseVersion()
+                            + "]"
                     );
                 }
                 out.writeOptionalWriteable(knnResults == null || knnResults.isEmpty() ? null : knnResults.get(0));
diff --git a/server/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java b/server/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java
index 816b956251dfe..90b4885e65c01 100644
--- a/server/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java
+++ b/server/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java
@@ -456,7 +456,7 @@ public void writeToNoId(StreamOutput out) throws IOException {
         if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_8_0)) {
             out.writeOptionalNamedWriteable(rankShardResult);
         } else if (rankShardResult != null) {
-            throw new IllegalArgumentException("cannot serialize [rank] to version [" + out.getTransportVersion() + "]");
+            throw new IllegalArgumentException("cannot serialize [rank] to version [" + out.getTransportVersion().toReleaseVersion() + "]");
         }
     }
 
diff --git a/server/src/main/java/org/elasticsearch/transport/InboundDecoder.java b/server/src/main/java/org/elasticsearch/transport/InboundDecoder.java
index 056af07b13912..2c498597779f5 100644
--- a/server/src/main/java/org/elasticsearch/transport/InboundDecoder.java
+++ b/server/src/main/java/org/elasticsearch/transport/InboundDecoder.java
@@ -246,9 +246,9 @@ static void checkVersionCompatibility(TransportVersion remoteVersion) {
         if (TransportVersion.isCompatible(remoteVersion) == false) {
             throw new IllegalStateException(
                 "Received message from unsupported version: ["
-                    + remoteVersion
+                    + remoteVersion.toReleaseVersion()
                     + "] minimal compatible version is: ["
-                    + TransportVersions.MINIMUM_COMPATIBLE
+                    + TransportVersions.MINIMUM_COMPATIBLE.toReleaseVersion()
                     + "]"
             );
         }
diff --git a/server/src/test/java/org/elasticsearch/common/io/stream/VersionCheckingStreamOutputTests.java b/server/src/test/java/org/elasticsearch/common/io/stream/VersionCheckingStreamOutputTests.java
index bee63e72a3a0a..7be3a8e716ac5 100644
--- a/server/src/test/java/org/elasticsearch/common/io/stream/VersionCheckingStreamOutputTests.java
+++ b/server/src/test/java/org/elasticsearch/common/io/stream/VersionCheckingStreamOutputTests.java
@@ -49,9 +49,9 @@ public void testCheckVersionCompatibility() throws IOException {
             );
             assertEquals(
                 "[test_writable] was released first in version "
-                    + TransportVersion.current()
+                    + TransportVersion.current().toReleaseVersion()
                     + ", failed compatibility check trying to send it to node with version "
-                    + streamVersion,
+                    + streamVersion.toReleaseVersion(),
                 e.getMessage()
             );
         }
diff --git a/server/src/test/java/org/elasticsearch/transport/InboundDecoderTests.java b/server/src/test/java/org/elasticsearch/transport/InboundDecoderTests.java
index ffcd3a3386f1a..8eba0277e4901 100644
--- a/server/src/test/java/org/elasticsearch/transport/InboundDecoderTests.java
+++ b/server/src/test/java/org/elasticsearch/transport/InboundDecoderTests.java
@@ -477,9 +477,9 @@ public void testCheckVersionCompatibility() {
         } catch (IllegalStateException expected) {
             assertEquals(
                 "Received message from unsupported version: ["
-                    + invalid
+                    + invalid.toReleaseVersion()
                     + "] minimal compatible version is: ["
-                    + TransportVersions.MINIMUM_COMPATIBLE
+                    + TransportVersions.MINIMUM_COMPATIBLE.toReleaseVersion()
                     + "]",
                 expected.getMessage()
             );
diff --git a/test/framework/src/main/java/org/elasticsearch/search/FailBeforeCurrentVersionQueryBuilder.java b/test/framework/src/main/java/org/elasticsearch/search/FailBeforeCurrentVersionQueryBuilder.java
index 6c08ff43033e6..f1cb62ff41ebd 100644
--- a/test/framework/src/main/java/org/elasticsearch/search/FailBeforeCurrentVersionQueryBuilder.java
+++ b/test/framework/src/main/java/org/elasticsearch/search/FailBeforeCurrentVersionQueryBuilder.java
@@ -22,7 +22,7 @@
 public class FailBeforeCurrentVersionQueryBuilder extends DummyQueryBuilder {
 
     public static final String NAME = "fail_before_current_version";
-    public static final int FUTURE_VERSION = TransportVersion.current().id() + 11_111;
+    public static final TransportVersion FUTURE_VERSION = TransportVersion.fromId(TransportVersion.current().id() + 11_111);
 
     public FailBeforeCurrentVersionQueryBuilder(StreamInput in) throws IOException {
         super(in);
@@ -49,6 +49,6 @@ protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) throws
     public TransportVersion getMinimalSupportedVersion() {
         // this is what causes the failure - it always reports a version in the future, so it is never compatible with
         // current or minimum CCS TransportVersion
-        return new TransportVersion(FUTURE_VERSION);
+        return FUTURE_VERSION;
     }
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/common/validation/SourceDestValidator.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/common/validation/SourceDestValidator.java
index 9962f14ec3736..700158712707a 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/common/validation/SourceDestValidator.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/common/validation/SourceDestValidator.java
@@ -71,8 +71,7 @@ public final class SourceDestValidator {
         + "alias [{0}], license is not active";
     public static final String REMOTE_SOURCE_INDICES_NOT_SUPPORTED = "remote source indices are not supported";
     public static final String REMOTE_CLUSTERS_TRANSPORT_TOO_OLD =
-        "remote clusters are expected to run at least transport version [{0}] (reason: [{1}]),"
-            + " but the following clusters were too old: [{2}]";
+        "remote clusters are expected to run at least version [{0}] (reason: [{1}])," + " but the following clusters were too old: [{2}]";
     public static final String PIPELINE_MISSING = "Pipeline with id [{0}] could not be found";
 
     private final IndexNameExpressionResolver indexNameExpressionResolver;
@@ -491,12 +490,12 @@ public void validate(Context context, ActionListener<Context> listener) {
             if (oldRemoteClusterVersions.isEmpty() == false) {
                 context.addValidationError(
                     REMOTE_CLUSTERS_TRANSPORT_TOO_OLD,
-                    minExpectedVersion,
+                    minExpectedVersion.toReleaseVersion(),
                     reason,
                     oldRemoteClusterVersions.entrySet()
                         .stream()
                         .sorted(comparingByKey())  // sort to have a deterministic order among clusters in the resulting string
-                        .map(e -> e.getKey() + " (" + e.getValue() + ")")
+                        .map(e -> e.getKey() + " (" + e.getValue().toReleaseVersion() + ")")
                         .collect(joining(", "))
                 );
             }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/GetUserPrivilegesResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/GetUserPrivilegesResponse.java
index 2cd37df4ef15e..9f62513e1b69f 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/GetUserPrivilegesResponse.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/GetUserPrivilegesResponse.java
@@ -111,9 +111,9 @@ public void writeTo(StreamOutput out) throws IOException {
         } else if (hasRemoteIndicesPrivileges()) {
             throw new IllegalArgumentException(
                 "versions of Elasticsearch before ["
-                    + TransportVersions.V_8_8_0
+                    + TransportVersions.V_8_8_0.toReleaseVersion()
                     + "] can't handle remote indices privileges and attempted to send to ["
-                    + out.getTransportVersion()
+                    + out.getTransportVersion().toReleaseVersion()
                     + "]"
             );
         }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Authentication.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Authentication.java
index 6a06be1b63b77..6ef2441011e6a 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Authentication.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Authentication.java
@@ -226,9 +226,9 @@ public Authentication maybeRewriteForOlderVersion(TransportVersion olderVersion)
         if (isCrossClusterAccess() && olderVersion.before(TRANSPORT_VERSION_ADVANCED_REMOTE_CLUSTER_SECURITY)) {
             throw new IllegalArgumentException(
                 "versions of Elasticsearch before ["
-                    + TRANSPORT_VERSION_ADVANCED_REMOTE_CLUSTER_SECURITY
+                    + TRANSPORT_VERSION_ADVANCED_REMOTE_CLUSTER_SECURITY.toReleaseVersion()
                     + "] can't handle cross cluster access authentication and attempted to rewrite for ["
-                    + olderVersion
+                    + olderVersion.toReleaseVersion()
                     + "]"
             );
         }
@@ -576,9 +576,9 @@ private static void doWriteTo(Subject effectiveSubject, Subject authenticatingSu
         if (isCrossClusterAccess && out.getTransportVersion().before(TRANSPORT_VERSION_ADVANCED_REMOTE_CLUSTER_SECURITY)) {
             throw new IllegalArgumentException(
                 "versions of Elasticsearch before ["
-                    + TRANSPORT_VERSION_ADVANCED_REMOTE_CLUSTER_SECURITY
+                    + TRANSPORT_VERSION_ADVANCED_REMOTE_CLUSTER_SECURITY.toReleaseVersion()
                     + "] can't handle cross cluster access authentication and attempted to send to ["
-                    + out.getTransportVersion()
+                    + out.getTransportVersion().toReleaseVersion()
                     + "]"
             );
         }
@@ -1368,9 +1368,9 @@ static Map<String, Object> maybeRewriteMetadataForCrossClusterAccessAuthenticati
                 () -> "Cross cluster access authentication has authentication field in metadata ["
                     + authenticationFromMetadata
                     + "] that may require a rewrite from version ["
-                    + effectiveSubjectVersion
+                    + effectiveSubjectVersion.toReleaseVersion()
                     + "] to ["
-                    + olderVersion
+                    + olderVersion.toReleaseVersion()
                     + "]"
             );
             final Map<String, Object> rewrittenMetadata = new HashMap<>(metadata);
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/common/validation/RemoteClusterMinimumVersionValidationTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/common/validation/RemoteClusterMinimumVersionValidationTests.java
index 57166d996d124..299279ee13f1b 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/common/validation/RemoteClusterMinimumVersionValidationTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/common/validation/RemoteClusterMinimumVersionValidationTests.java
@@ -82,8 +82,8 @@ public void testValidate_OneRemoteClusterVersionTooLow() {
                 ctx -> assertThat(
                     ctx.getValidationException().validationErrors(),
                     contains(
-                        "remote clusters are expected to run at least transport version [7110099] (reason: [some reason]), "
-                            + "but the following clusters were too old: [cluster-A (7100099)]"
+                        "remote clusters are expected to run at least version [7.11.0] (reason: [some reason]), "
+                            + "but the following clusters were too old: [cluster-A (7.10.0)]"
                     )
                 )
             )
@@ -100,8 +100,8 @@ public void testValidate_TwoRemoteClusterVersionsTooLow() {
                 ctx -> assertThat(
                     ctx.getValidationException().validationErrors(),
                     contains(
-                        "remote clusters are expected to run at least transport version [7120099] (reason: [some reason]), "
-                            + "but the following clusters were too old: [cluster-A (7100099), cluster-B (7110099)]"
+                        "remote clusters are expected to run at least version [7.12.0] (reason: [some reason]), "
+                            + "but the following clusters were too old: [cluster-A (7.10.0), cluster-B (7.11.0)]"
                     )
                 )
             )
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/user/GetUserPrivilegesResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/user/GetUserPrivilegesResponseTests.java
index 9556d09186311..1cf61fac174a5 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/user/GetUserPrivilegesResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/user/GetUserPrivilegesResponseTests.java
@@ -100,9 +100,9 @@ public void testSerializationWithRemoteIndicesThrowsOnUnsupportedVersions() thro
                 ex.getMessage(),
                 containsString(
                     "versions of Elasticsearch before ["
-                        + TransportVersions.V_8_8_0
+                        + TransportVersions.V_8_8_0.toReleaseVersion()
                         + "] can't handle remote indices privileges and attempted to send to ["
-                        + version
+                        + version.toReleaseVersion()
                         + "]"
                 )
             );
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/termsenum/TransportTermsEnumActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/termsenum/TransportTermsEnumActionTests.java
index cb06dffead14f..975d08eb45277 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/termsenum/TransportTermsEnumActionTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/termsenum/TransportTermsEnumActionTests.java
@@ -84,7 +84,9 @@ public TransportVersion getMinimalSupportedVersion() {
         assertThat(
             ex.getCause().getCause().getMessage(),
             containsString(
-                "was released first in version " + version + ", failed compatibility check trying to send it to node with version"
+                "was released first in version "
+                    + version.toReleaseVersion()
+                    + ", failed compatibility check trying to send it to node with version"
             )
         );
     }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDatafeedAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDatafeedAction.java
index 2067bae048561..ab0ada00b8aaf 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDatafeedAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDatafeedAction.java
@@ -318,7 +318,7 @@ static void checkRemoteConfigVersions(
         throw ExceptionsHelper.badRequestException(
             Messages.getMessage(
                 REMOTE_CLUSTERS_TRANSPORT_TOO_OLD,
-                minVersion.toString(),
+                minVersion.toReleaseVersion(),
                 reason,
                 Strings.collectionToCommaDelimitedString(clustersTooOld)
             )
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportStartDatafeedActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportStartDatafeedActionTests.java
index a2d5003118536..8fd1082e0df5c 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportStartDatafeedActionTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportStartDatafeedActionTests.java
@@ -138,7 +138,7 @@ public void testRemoteClusterVersionCheck() {
         assertThat(
             ex.getMessage(),
             containsString(
-                "remote clusters are expected to run at least transport version [7110099] (reason: [runtime_mappings]), "
+                "remote clusters are expected to run at least version [7.11.0] (reason: [runtime_mappings]), "
                     + "but the following clusters were too old: [old_cluster_1]"
             )
         );
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java
index e4436d4fabe71..1a5b1ab39cd83 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java
@@ -322,8 +322,8 @@ && hasRemoteIndices(request.getRoleDescriptors())) {
                 // Creating API keys with roles which define remote indices privileges is not allowed in a mixed cluster.
                 listener.onFailure(
                     new IllegalArgumentException(
-                        "all nodes must have transport version ["
-                            + TRANSPORT_VERSION_ADVANCED_REMOTE_CLUSTER_SECURITY
+                        "all nodes must have version ["
+                            + TRANSPORT_VERSION_ADVANCED_REMOTE_CLUSTER_SECURITY.toReleaseVersion()
                             + "] or higher to support remote indices privileges for API keys"
                     )
                 );
@@ -333,8 +333,8 @@ && hasRemoteIndices(request.getRoleDescriptors())) {
                 && request.getType() == ApiKey.Type.CROSS_CLUSTER) {
                 listener.onFailure(
                     new IllegalArgumentException(
-                        "all nodes must have transport version ["
-                            + TRANSPORT_VERSION_ADVANCED_REMOTE_CLUSTER_SECURITY
+                        "all nodes must have version ["
+                            + TRANSPORT_VERSION_ADVANCED_REMOTE_CLUSTER_SECURITY.toReleaseVersion()
                             + "] or higher to support creating cross cluster API keys"
                     )
                 );
@@ -381,8 +381,8 @@ private static IllegalArgumentException validateWorkflowsRestrictionConstraints(
             // creating/updating API keys with restrictions is not allowed in a mixed cluster.
             if (transportVersion.before(WORKFLOWS_RESTRICTION_VERSION)) {
                 return new IllegalArgumentException(
-                    "all nodes must have transport version ["
-                        + WORKFLOWS_RESTRICTION_VERSION
+                    "all nodes must have version ["
+                        + WORKFLOWS_RESTRICTION_VERSION.toReleaseVersion()
                         + "] or higher to support restrictions for API keys"
                 );
             }
@@ -492,8 +492,8 @@ public void updateApiKeys(
             // Updating API keys with roles which define remote indices privileges is not allowed in a mixed cluster.
             listener.onFailure(
                 new IllegalArgumentException(
-                    "all nodes must have transport version ["
-                        + TRANSPORT_VERSION_ADVANCED_REMOTE_CLUSTER_SECURITY
+                    "all nodes must have version ["
+                        + TRANSPORT_VERSION_ADVANCED_REMOTE_CLUSTER_SECURITY.toReleaseVersion()
                         + "] or higher to support remote indices privileges for API keys"
                 )
             );
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/CrossClusterAccessAuthenticationService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/CrossClusterAccessAuthenticationService.java
index efbe4258c60f3..16f67c9077311 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/CrossClusterAccessAuthenticationService.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/CrossClusterAccessAuthenticationService.java
@@ -79,8 +79,8 @@ public void authenticate(final String action, final TransportRequest request, fi
             withRequestProcessingFailure(
                 authcContext,
                 new IllegalArgumentException(
-                    "all nodes must have transport version ["
-                        + TRANSPORT_VERSION_ADVANCED_REMOTE_CLUSTER_SECURITY
+                    "all nodes must have version ["
+                        + TRANSPORT_VERSION_ADVANCED_REMOTE_CLUSTER_SECURITY.toReleaseVersion()
                         + "] or higher to support cross cluster requests through the dedicated remote cluster port"
                 ),
                 listener
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStore.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStore.java
index a9d8859c798fa..41269ea049d66 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStore.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStore.java
@@ -259,8 +259,8 @@ public void putRole(final PutRoleRequest request, final RoleDescriptor role, fin
             && clusterService.state().getMinTransportVersion().before(TRANSPORT_VERSION_ADVANCED_REMOTE_CLUSTER_SECURITY)) {
                 listener.onFailure(
                     new IllegalStateException(
-                        "all nodes must have transport version ["
-                            + TRANSPORT_VERSION_ADVANCED_REMOTE_CLUSTER_SECURITY
+                        "all nodes must have version ["
+                            + TRANSPORT_VERSION_ADVANCED_REMOTE_CLUSTER_SECURITY.toReleaseVersion()
                             + "] or higher to support remote indices privileges"
                     )
                 );
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/SecurityServerTransportInterceptor.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/SecurityServerTransportInterceptor.java
index 462b41a519460..1a68887646731 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/SecurityServerTransportInterceptor.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/SecurityServerTransportInterceptor.java
@@ -318,7 +318,7 @@ private <T extends TransportResponse> void sendWithCrossClusterAccessHeaders(
                         "Settings for remote cluster ["
                             + remoteClusterAlias
                             + "] indicate cross cluster access headers should be sent but target cluster version ["
-                            + connection.getTransportVersion()
+                            + connection.getTransportVersion().toReleaseVersion()
                             + "] does not support receiving them"
                     );
                 }
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java
index 269031804f7e3..0cb7a270099ad 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java
@@ -2710,8 +2710,8 @@ public void testCreateCrossClusterApiKeyMinVersionConstraint() {
         assertThat(
             e.getMessage(),
             containsString(
-                "all nodes must have transport version ["
-                    + TRANSPORT_VERSION_ADVANCED_REMOTE_CLUSTER_SECURITY
+                "all nodes must have version ["
+                    + TRANSPORT_VERSION_ADVANCED_REMOTE_CLUSTER_SECURITY.toReleaseVersion()
                     + "] or higher to support creating cross cluster API keys"
             )
         );
@@ -2856,8 +2856,8 @@ public void testCreateOrUpdateApiKeyWithWorkflowsRestrictionForUnsupportedVersio
         assertThat(
             e1.getMessage(),
             containsString(
-                "all nodes must have transport version ["
-                    + WORKFLOWS_RESTRICTION_VERSION
+                "all nodes must have version ["
+                    + WORKFLOWS_RESTRICTION_VERSION.toReleaseVersion()
                     + "] or higher to support restrictions for API keys"
             )
         );
@@ -2874,8 +2874,8 @@ public void testCreateOrUpdateApiKeyWithWorkflowsRestrictionForUnsupportedVersio
         assertThat(
             e2.getMessage(),
             containsString(
-                "all nodes must have transport version ["
-                    + WORKFLOWS_RESTRICTION_VERSION
+                "all nodes must have version ["
+                    + WORKFLOWS_RESTRICTION_VERSION.toReleaseVersion()
                     + "] or higher to support restrictions for API keys"
             )
         );
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/CrossClusterAccessAuthenticationServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/CrossClusterAccessAuthenticationServiceTests.java
index 6e03aff9a842e..3bb776e0f726c 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/CrossClusterAccessAuthenticationServiceTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/CrossClusterAccessAuthenticationServiceTests.java
@@ -108,8 +108,8 @@ public void testAuthenticateThrowsOnUnsupportedMinVersions() throws IOException
         assertThat(
             actual.getCause().getCause().getMessage(),
             equalTo(
-                "all nodes must have transport version ["
-                    + TRANSPORT_VERSION_ADVANCED_REMOTE_CLUSTER_SECURITY
+                "all nodes must have version ["
+                    + TRANSPORT_VERSION_ADVANCED_REMOTE_CLUSTER_SECURITY.toReleaseVersion()
                     + "] or higher to support cross cluster requests through the dedicated remote cluster port"
             )
         );
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStoreTests.java
index 49d5a67b7d20e..ad1ab132bb321 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStoreTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStoreTests.java
@@ -435,8 +435,8 @@ void innerPutRole(final PutRoleRequest request, final RoleDescriptor role, final
         assertThat(
             e.getMessage(),
             containsString(
-                "all nodes must have transport version ["
-                    + TRANSPORT_VERSION_ADVANCED_REMOTE_CLUSTER_SECURITY
+                "all nodes must have version ["
+                    + TRANSPORT_VERSION_ADVANCED_REMOTE_CLUSTER_SECURITY.toReleaseVersion()
                     + "] or higher to support remote indices privileges"
             )
         );
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/SecurityServerTransportInterceptorTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/SecurityServerTransportInterceptorTests.java
index 6d5ba44ccabf7..2d8307eae8ba6 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/SecurityServerTransportInterceptorTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/SecurityServerTransportInterceptorTests.java
@@ -1005,7 +1005,7 @@ public TransportResponse read(StreamInput in) {
                 "Settings for remote cluster ["
                     + remoteClusterAlias
                     + "] indicate cross cluster access headers should be sent but target cluster version ["
-                    + connection.getTransportVersion()
+                    + connection.getTransportVersion().toReleaseVersion()
                     + "] does not support receiving them"
             )
         );
diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/BlobAnalyzeAction.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/BlobAnalyzeAction.java
index d9c85eb37aaa0..aa0cf3e3cfc1b 100644
--- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/BlobAnalyzeAction.java
+++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/BlobAnalyzeAction.java
@@ -716,7 +716,9 @@ public void writeTo(StreamOutput out) throws IOException {
             if (out.getTransportVersion().onOrAfter(TransportVersions.V_7_14_0)) {
                 out.writeBoolean(abortWrite);
             } else if (abortWrite) {
-                throw new IllegalStateException("cannot send abortWrite request on transport version [" + out.getTransportVersion() + "]");
+                throw new IllegalStateException(
+                    "cannot send abortWrite request on transport version [" + out.getTransportVersion().toReleaseVersion() + "]"
+                );
             }
         }
 
diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalyzeAction.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalyzeAction.java
index 51fe98b86b536..7b82b69a682fa 100644
--- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalyzeAction.java
+++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalyzeAction.java
@@ -949,8 +949,8 @@ public void writeTo(StreamOutput out) throws IOException {
                 out.writeVInt(registerOperationCount);
             } else if (registerOperationCount != concurrency) {
                 throw new IllegalArgumentException(
-                    "cannot send request with registerOperationCount != concurrency on transport version ["
-                        + out.getTransportVersion()
+                    "cannot send request with registerOperationCount != concurrency to version ["
+                        + out.getTransportVersion().toReleaseVersion()
                         + "]"
                 );
             }
@@ -965,7 +965,7 @@ public void writeTo(StreamOutput out) throws IOException {
                 out.writeBoolean(abortWritePermitted);
             } else if (abortWritePermitted) {
                 throw new IllegalArgumentException(
-                    "cannot send abortWritePermitted request on transport version [" + out.getTransportVersion() + "]"
+                    "cannot send abortWritePermitted request to version [" + out.getTransportVersion().toReleaseVersion() + "]"
                 );
             }
         }
diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/ApiKeyBackwardsCompatibilityIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/ApiKeyBackwardsCompatibilityIT.java
index 2bce06543f67c..fb0b8115ae17b 100644
--- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/ApiKeyBackwardsCompatibilityIT.java
+++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/ApiKeyBackwardsCompatibilityIT.java
@@ -167,8 +167,8 @@ public void testCreatingAndUpdatingApiKeys() throws Exception {
                     assertThat(
                         e.getMessage(),
                         containsString(
-                            "all nodes must have transport version ["
-                                + TRANSPORT_VERSION_ADVANCED_REMOTE_CLUSTER_SECURITY
+                            "all nodes must have version ["
+                                + TRANSPORT_VERSION_ADVANCED_REMOTE_CLUSTER_SECURITY.toReleaseVersion()
                                 + "] or higher to support remote indices privileges for API keys"
                         )
                     );
@@ -179,8 +179,8 @@ public void testCreatingAndUpdatingApiKeys() throws Exception {
                     assertThat(
                         e.getMessage(),
                         containsString(
-                            "all nodes must have transport version ["
-                                + TRANSPORT_VERSION_ADVANCED_REMOTE_CLUSTER_SECURITY
+                            "all nodes must have version ["
+                                + TRANSPORT_VERSION_ADVANCED_REMOTE_CLUSTER_SECURITY.toReleaseVersion()
                                 + "] or higher to support remote indices privileges for API keys"
                         )
                     );

From 9a7cd05da0237ce3a341831a2d7808bdc7fce99d Mon Sep 17 00:00:00 2001
From: Yang Wang <yang.wang@elastic.co>
Date: Fri, 26 Apr 2024 18:38:44 +1000
Subject: [PATCH 43/58] Unmute AzureStorageCleanupThirdPartyTests after
 credentials rotation (#107928)

Resolves: #107617, #107720, #107753
---
 .../repositories/azure/AzureStorageCleanupThirdPartyTests.java  | 2 --
 1 file changed, 2 deletions(-)

diff --git a/modules/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureStorageCleanupThirdPartyTests.java b/modules/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureStorageCleanupThirdPartyTests.java
index c7dbc7022e1a8..052b558a05a38 100644
--- a/modules/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureStorageCleanupThirdPartyTests.java
+++ b/modules/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureStorageCleanupThirdPartyTests.java
@@ -14,7 +14,6 @@
 import com.azure.storage.blob.BlobServiceClient;
 import com.azure.storage.blob.models.BlobStorageException;
 
-import org.apache.lucene.tests.util.LuceneTestCase;
 import org.elasticsearch.action.ActionRunnable;
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
@@ -41,7 +40,6 @@
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.not;
 
-@LuceneTestCase.AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/107720")
 public class AzureStorageCleanupThirdPartyTests extends AbstractThirdPartyRepositoryTestCase {
     private static final boolean USE_FIXTURE = Booleans.parseBoolean(System.getProperty("test.azure.fixture", "true"));
 

From 529fa61c77f942cbf2411559bc643d42900f74b1 Mon Sep 17 00:00:00 2001
From: Simon Cooper <simon.cooper@elastic.co>
Date: Fri, 26 Apr 2024 09:56:43 +0100
Subject: [PATCH 44/58] Missed a couple of test references to transportversions

---
 .../core/security/authc/AuthenticationSerializationTests.java | 4 ++--
 .../xpack/core/security/authc/AuthenticationTests.java        | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/AuthenticationSerializationTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/AuthenticationSerializationTests.java
index b098b686f6c2c..11a8f894d7cf7 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/AuthenticationSerializationTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/AuthenticationSerializationTests.java
@@ -95,9 +95,9 @@ public void testWriteToWithCrossClusterAccessThrowsOnUnsupportedVersion() throws
                 ex.getMessage(),
                 containsString(
                     "versions of Elasticsearch before ["
-                        + RemoteClusterPortSettings.TRANSPORT_VERSION_ADVANCED_REMOTE_CLUSTER_SECURITY
+                        + RemoteClusterPortSettings.TRANSPORT_VERSION_ADVANCED_REMOTE_CLUSTER_SECURITY.toReleaseVersion()
                         + "] can't handle cross cluster access authentication and attempted to send to ["
-                        + out.getTransportVersion()
+                        + out.getTransportVersion().toReleaseVersion()
                         + "]"
                 )
             );
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/AuthenticationTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/AuthenticationTests.java
index e1a60d41ca212..1e33a7f54394b 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/AuthenticationTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/AuthenticationTests.java
@@ -858,9 +858,9 @@ public void testMaybeRewriteForOlderVersionWithCrossClusterAccessThrowsOnUnsuppo
                 ex.getMessage(),
                 containsString(
                     "versions of Elasticsearch before ["
-                        + RemoteClusterPortSettings.TRANSPORT_VERSION_ADVANCED_REMOTE_CLUSTER_SECURITY
+                        + RemoteClusterPortSettings.TRANSPORT_VERSION_ADVANCED_REMOTE_CLUSTER_SECURITY.toReleaseVersion()
                         + "] can't handle cross cluster access authentication and attempted to rewrite for ["
-                        + version
+                        + version.toReleaseVersion()
                         + "]"
                 )
             );

From 3e0568a102de0d95f60402164195159803e2a6f6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Johannes=20Fred=C3=A9n?=
 <109296772+jfreden@users.noreply.github.com>
Date: Fri, 26 Apr 2024 10:58:10 +0200
Subject: [PATCH 45/58] Remove ES version from security index (#107814)

---
 .../support/SecurityIndexManager.java         | 123 +++++++-----------
 .../support/SecuritySystemIndices.java        |  19 ++-
 .../authc/AuthenticationServiceTests.java     |   1 -
 .../authc/esnative/NativeRealmTests.java      |   1 -
 .../mapper/NativeRoleMappingStoreTests.java   |   1 -
 .../authz/store/CompositeRolesStoreTests.java |   1 -
 .../store/NativePrivilegeStoreTests.java      |   1 -
 .../authz/store/NativeRolesStoreTests.java    |  11 ++
 .../CacheInvalidatorRegistryTests.java        |   6 +-
 .../support/SecurityIndexManagerTests.java    | 117 +++++++++++++----
 10 files changed, 172 insertions(+), 109 deletions(-)

diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java
index 62bb20322f185..773573d02e45a 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java
@@ -9,11 +9,9 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.ElasticsearchStatusException;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.ResourceAlreadyExistsException;
-import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.UnavailableShardsException;
 import org.elasticsearch.action.admin.indices.alias.Alias;
@@ -44,17 +42,15 @@
 import org.elasticsearch.xcontent.XContentType;
 
 import java.time.Instant;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
-import java.util.Set;
 import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.function.BiConsumer;
 import java.util.function.Consumer;
-import java.util.function.Predicate;
 
 import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_FORMAT_SETTING;
+import static org.elasticsearch.indices.SystemIndexDescriptor.VERSION_META_KEY;
 import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin;
 import static org.elasticsearch.xpack.security.support.SecurityIndexManager.State.UNRECOVERED_STATE;
 
@@ -214,6 +210,19 @@ public void removeStateListener(BiConsumer<State, State> listener) {
         stateChangeListeners.remove(listener);
     }
 
+    /**
+     * Get the minimum security index mapping version in the cluster
+     */
+    private SystemIndexDescriptor.MappingsVersion getMinSecurityIndexMappingVersion(ClusterState clusterState) {
+        var minClusterVersion = clusterState.getMinSystemIndexMappingVersions().get(systemIndexDescriptor.getPrimaryIndex());
+        // Can be null in mixed clusters. This indicates that the cluster state and index needs to be updated with the latest mapping
+        // version from the index descriptor
+        if (minClusterVersion == null) {
+            return systemIndexDescriptor.getMappingsVersion();
+        }
+        return minClusterVersion;
+    }
+
     @Override
     public void clusterChanged(ClusterChangedEvent event) {
         if (event.state().blocks().hasGlobalBlock(GatewayService.STATE_NOT_RECOVERED_BLOCK)) {
@@ -231,7 +240,7 @@ public void clusterChanged(ClusterChangedEvent event) {
         final boolean indexAvailableForWrite = available.v1();
         final boolean indexAvailableForSearch = available.v2();
         final boolean mappingIsUpToDate = indexMetadata == null || checkIndexMappingUpToDate(event.state());
-        final Version mappingVersion = oldestIndexMappingVersion(event.state());
+        final SystemIndexDescriptor.MappingsVersion mappingVersion = getMinSecurityIndexMappingVersion(event.state());
         final String concreteIndexName = indexMetadata == null
             ? systemIndexDescriptor.getPrimaryIndex()
             : indexMetadata.getIndex().getName();
@@ -261,7 +270,6 @@ public void clusterChanged(ClusterChangedEvent event) {
             concreteIndexName,
             indexHealth,
             indexState,
-            event.state().nodes().getSmallestNonClientNodeVersion(),
             indexUUID
         );
         this.state = newState;
@@ -317,51 +325,43 @@ private Tuple<Boolean, Boolean> checkIndexAvailable(ClusterState state) {
         return new Tuple<>(allPrimaryShards, searchShards);
     }
 
+    /**
+     * Detect if the mapping in the security index is outdated. If it's outdated it means that whatever is in cluster state is more recent.
+     * There could be several nodes on different ES versions (mixed cluster) supporting different mapping versions, so only return false if
+     * min version in the cluster is more recent than what's in the security index.
+     */
     private boolean checkIndexMappingUpToDate(ClusterState clusterState) {
-        final Version minimumNonClientNodeVersion = clusterState.nodes().getSmallestNonClientNodeVersion();
-        final SystemIndexDescriptor descriptor = systemIndexDescriptor.getDescriptorCompatibleWith(minimumNonClientNodeVersion);
+        // Get descriptor compatible with the min version in the cluster
+        final SystemIndexDescriptor descriptor = systemIndexDescriptor.getDescriptorCompatibleWith(
+            getMinSecurityIndexMappingVersion(clusterState)
+        );
         if (descriptor == null) {
             return false;
         }
-
-        /*
-         * The method reference looks wrong here, but it's just counter-intuitive. It expands to:
-         *
-         *     mappingVersion -> descriptor.getMappingVersion().onOrBefore(mappingVersion)
-         *
-         * ...which is true if the mappings have been updated.
-         */
-        return checkIndexMappingVersionMatches(clusterState, descriptor.getMappingsNodeVersion()::onOrBefore);
-    }
-
-    private boolean checkIndexMappingVersionMatches(ClusterState clusterState, Predicate<Version> predicate) {
-        return checkIndexMappingVersionMatches(this.systemIndexDescriptor.getAliasName(), clusterState, logger, predicate);
-    }
-
-    public static boolean checkIndexMappingVersionMatches(
-        String indexName,
-        ClusterState clusterState,
-        Logger logger,
-        Predicate<Version> predicate
-    ) {
-        return loadIndexMappingVersions(indexName, clusterState, logger).stream().allMatch(predicate);
+        return descriptor.getMappingsVersion().version() <= loadIndexMappingVersion(systemIndexDescriptor.getAliasName(), clusterState);
     }
 
-    private Version oldestIndexMappingVersion(ClusterState clusterState) {
-        final Set<Version> versions = loadIndexMappingVersions(systemIndexDescriptor.getAliasName(), clusterState, logger);
-        return versions.stream().min(Version::compareTo).orElse(null);
-    }
-
-    private static Set<Version> loadIndexMappingVersions(String aliasName, ClusterState clusterState, Logger logger) {
-        Set<Version> versions = new HashSet<>();
+    private static int loadIndexMappingVersion(String aliasName, ClusterState clusterState) {
         IndexMetadata indexMetadata = resolveConcreteIndex(aliasName, clusterState.metadata());
         if (indexMetadata != null) {
             MappingMetadata mappingMetadata = indexMetadata.mapping();
             if (mappingMetadata != null) {
-                versions.add(readMappingVersion(aliasName, mappingMetadata, logger));
+                return readMappingVersion(aliasName, mappingMetadata);
             }
         }
-        return versions;
+        return 0;
+    }
+
+    private static int readMappingVersion(String indexName, MappingMetadata mappingMetadata) {
+        @SuppressWarnings("unchecked")
+        Map<String, Object> meta = (Map<String, Object>) mappingMetadata.sourceAsMap().get("_meta");
+        if (meta == null) {
+            logger.info("Missing _meta field in mapping [{}] of index [{}]", mappingMetadata.type(), indexName);
+            throw new IllegalStateException("Cannot read managed_index_mappings_version string in index " + indexName);
+        }
+        // If null, no value has been set in the index yet, so return 0 to trigger put mapping
+        final Integer value = (Integer) meta.get(VERSION_META_KEY);
+        return value == null ? 0 : value;
     }
 
     /**
@@ -380,21 +380,6 @@ private static IndexMetadata resolveConcreteIndex(final String indexOrAliasName,
         return null;
     }
 
-    private static Version readMappingVersion(String indexName, MappingMetadata mappingMetadata, Logger logger) {
-        try {
-            @SuppressWarnings("unchecked")
-            Map<String, Object> meta = (Map<String, Object>) mappingMetadata.sourceAsMap().get("_meta");
-            if (meta == null) {
-                logger.info("Missing _meta field in mapping [{}] of index [{}]", mappingMetadata.type(), indexName);
-                throw new IllegalStateException("Cannot read security-version string in index " + indexName);
-            }
-            return Version.fromString((String) meta.get(SECURITY_VERSION_STRING));
-        } catch (ElasticsearchParseException e) {
-            logger.error(() -> "Cannot parse the mapping for index [" + indexName + "]", e);
-            throw new ElasticsearchException("Cannot parse the mapping for index [{}]", e, indexName);
-        }
-    }
-
     /**
      * Validates that the index is up to date and does not need to be migrated. If it is not, the
      * consumer is called with an exception. If the index is up to date, the runnable will
@@ -440,12 +425,10 @@ public void prepareIndexIfNeededThenExecute(final Consumer<Exception> consumer,
                 );
             } else if (state.indexExists() == false) {
                 assert state.concreteIndexName != null;
-                final SystemIndexDescriptor descriptorForVersion = systemIndexDescriptor.getDescriptorCompatibleWith(
-                    state.minimumNodeVersion
-                );
+                final SystemIndexDescriptor descriptorForVersion = systemIndexDescriptor.getDescriptorCompatibleWith(state.mappingVersion);
 
                 if (descriptorForVersion == null) {
-                    final String error = systemIndexDescriptor.getMinimumNodeVersionMessage("create index");
+                    final String error = systemIndexDescriptor.getMinimumMappingsVersionMessage("create index");
                     consumer.accept(new IllegalStateException(error));
                 } else {
                     logger.info(
@@ -491,11 +474,10 @@ public void onFailure(Exception e) {
                     );
                 }
             } else if (state.mappingUpToDate == false) {
-                final SystemIndexDescriptor descriptorForVersion = systemIndexDescriptor.getDescriptorCompatibleWith(
-                    state.minimumNodeVersion
-                );
+                final SystemIndexDescriptor descriptorForVersion = systemIndexDescriptor.getDescriptorCompatibleWith(state.mappingVersion);
+
                 if (descriptorForVersion == null) {
-                    final String error = systemIndexDescriptor.getMinimumNodeVersionMessage("updating mapping");
+                    final String error = systemIndexDescriptor.getMinimumMappingsVersionMessage("updating mapping");
                     consumer.accept(new IllegalStateException(error));
                 } else {
                     logger.info(
@@ -549,17 +531,16 @@ public static boolean isIndexDeleted(State previousState, State currentState) {
      * State of the security index.
      */
     public static class State {
-        public static final State UNRECOVERED_STATE = new State(null, false, false, false, false, null, null, null, null, null, null);
+        public static final State UNRECOVERED_STATE = new State(null, false, false, false, false, null, null, null, null, null);
         public final Instant creationTime;
         public final boolean isIndexUpToDate;
         public final boolean indexAvailableForSearch;
         public final boolean indexAvailableForWrite;
         public final boolean mappingUpToDate;
-        public final Version mappingVersion;
+        public final SystemIndexDescriptor.MappingsVersion mappingVersion;
         public final String concreteIndexName;
         public final ClusterHealthStatus indexHealth;
         public final IndexMetadata.State indexState;
-        public final Version minimumNodeVersion;
         public final String indexUUID;
 
         public State(
@@ -568,11 +549,10 @@ public State(
             boolean indexAvailableForSearch,
             boolean indexAvailableForWrite,
             boolean mappingUpToDate,
-            Version mappingVersion,
+            SystemIndexDescriptor.MappingsVersion mappingVersion,
             String concreteIndexName,
             ClusterHealthStatus indexHealth,
             IndexMetadata.State indexState,
-            Version minimumNodeVersion,
             String indexUUID
         ) {
             this.creationTime = creationTime;
@@ -584,7 +564,6 @@ public State(
             this.concreteIndexName = concreteIndexName;
             this.indexHealth = indexHealth;
             this.indexState = indexState;
-            this.minimumNodeVersion = minimumNodeVersion;
             this.indexUUID = indexUUID;
         }
 
@@ -601,8 +580,7 @@ public boolean equals(Object o) {
                 && Objects.equals(mappingVersion, state.mappingVersion)
                 && Objects.equals(concreteIndexName, state.concreteIndexName)
                 && indexHealth == state.indexHealth
-                && indexState == state.indexState
-                && Objects.equals(minimumNodeVersion, state.minimumNodeVersion);
+                && indexState == state.indexState;
         }
 
         public boolean indexExists() {
@@ -619,8 +597,7 @@ public int hashCode() {
                 mappingUpToDate,
                 mappingVersion,
                 concreteIndexName,
-                indexHealth,
-                minimumNodeVersion
+                indexHealth
             );
         }
     }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecuritySystemIndices.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecuritySystemIndices.java
index 6c21b1f275f24..3e46a370c6e92 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecuritySystemIndices.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecuritySystemIndices.java
@@ -38,7 +38,7 @@
 public class SecuritySystemIndices {
 
     public static final int INTERNAL_MAIN_INDEX_FORMAT = 6;
-    private static final int INTERNAL_MAIN_INDEX_MAPPINGS_FORMAT = 1;
+    public static final int INTERNAL_MAIN_INDEX_MAPPINGS_FORMAT = 1;
     private static final int INTERNAL_TOKENS_INDEX_FORMAT = 7;
     private static final int INTERNAL_TOKENS_INDEX_MAPPINGS_FORMAT = 1;
     private static final int INTERNAL_PROFILE_INDEX_FORMAT = 8;
@@ -54,6 +54,15 @@ public class SecuritySystemIndices {
     public static final Version VERSION_SECURITY_PROFILE_ORIGIN = Version.V_8_3_0;
     public static final NodeFeature SECURITY_PROFILE_ORIGIN_FEATURE = new NodeFeature("security.security_profile_origin");
 
+    /**
+     * Security managed index mappings used to be updated based on the product version. They are now updated based on per-index mappings
+     * versions. However, older nodes will still look for a product version in the mappings metadata, so we have to put <em>something</em>
+     * in that field that will allow the older node to realise that the mappings are ahead of what it knows about. The easiest solution is
+     * to hardcode 8.14.0 in this field, because any node from 8.14.0 onwards should be using per-index mappings versions to determine
+     * whether mappings are up-to-date.
+     */
+    public static final String BWC_MAPPINGS_VERSION = "8.14.0";
+
     private static final Logger logger = LogManager.getLogger(SecuritySystemIndices.class);
 
     private final SystemIndexDescriptor mainDescriptor;
@@ -119,7 +128,7 @@ private SystemIndexDescriptor getSecurityMainIndexDescriptor() {
             .setSettings(getMainIndexSettings())
             .setAliasName(SECURITY_MAIN_ALIAS)
             .setIndexFormat(INTERNAL_MAIN_INDEX_FORMAT)
-            .setVersionMetaKey("security-version")
+            .setVersionMetaKey(SECURITY_VERSION_STRING)
             .setOrigin(SECURITY_ORIGIN)
             .setThreadPools(ExecutorNames.CRITICAL_SYSTEM_INDEX_THREAD_POOLS)
             .build();
@@ -146,7 +155,7 @@ private XContentBuilder getMainIndexMappings() {
             builder.startObject();
             {
                 builder.startObject("_meta");
-                builder.field(SECURITY_VERSION_STRING, Version.CURRENT.toString());
+                builder.field(SECURITY_VERSION_STRING, BWC_MAPPINGS_VERSION); // Only needed for BWC with pre-8.15.0 nodes
                 builder.field(SystemIndexDescriptor.VERSION_META_KEY, INTERNAL_MAIN_INDEX_MAPPINGS_FORMAT);
                 builder.endObject();
 
@@ -632,7 +641,7 @@ private static XContentBuilder getTokenIndexMappings() {
             builder.startObject();
             {
                 builder.startObject("_meta");
-                builder.field(SECURITY_VERSION_STRING, Version.CURRENT);
+                builder.field(SECURITY_VERSION_STRING, BWC_MAPPINGS_VERSION); // Only needed for BWC with pre-8.15.0 nodes
                 builder.field(SystemIndexDescriptor.VERSION_META_KEY, INTERNAL_TOKENS_INDEX_MAPPINGS_FORMAT);
                 builder.endObject();
 
@@ -844,7 +853,7 @@ private XContentBuilder getProfileIndexMappings(int mappingsVersion) {
             builder.startObject();
             {
                 builder.startObject("_meta");
-                builder.field(SECURITY_VERSION_STRING, Version.CURRENT.toString());
+                builder.field(SECURITY_VERSION_STRING, BWC_MAPPINGS_VERSION); // Only needed for BWC with pre-8.15.0 nodes
                 builder.field(SystemIndexDescriptor.VERSION_META_KEY, mappingsVersion);
                 builder.endObject();
 
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java
index 3c6f7462c0bb4..57b656dc0ddde 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java
@@ -2525,7 +2525,6 @@ private SecurityIndexManager.State dummyState(ClusterHealthStatus indexStatus) {
             concreteSecurityIndexName,
             indexStatus,
             IndexMetadata.State.OPEN,
-            null,
             "my_uuid"
         );
     }
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmTests.java
index b9cc599609ea1..e127f70ac83a8 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmTests.java
@@ -42,7 +42,6 @@ private SecurityIndexManager.State dummyState(ClusterHealthStatus indexStatus) {
             concreteSecurityIndexName,
             indexStatus,
             IndexMetadata.State.OPEN,
-            null,
             "my_uuid"
         );
     }
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java
index 4add4fd37fff5..b47610797a832 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java
@@ -415,7 +415,6 @@ private SecurityIndexManager.State indexState(boolean isUpToDate, ClusterHealthS
             concreteSecurityIndexName,
             healthStatus,
             IndexMetadata.State.OPEN,
-            null,
             "my_uuid"
         );
     }
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java
index 3445ca05656b2..23d1f4854c23a 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java
@@ -1531,7 +1531,6 @@ public SecurityIndexManager.State dummyIndexState(boolean isIndexUpToDate, Clust
             concreteSecurityIndexName,
             healthStatus,
             IndexMetadata.State.OPEN,
-            null,
             "my_uuid"
         );
     }
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreTests.java
index ed1b5e6c7668b..2d02117b9728f 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreTests.java
@@ -796,7 +796,6 @@ private SecurityIndexManager.State dummyState(
             concreteSecurityIndexName,
             healthStatus,
             IndexMetadata.State.OPEN,
-            null,
             "my_uuid"
         );
     }
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStoreTests.java
index ad1ab132bb321..124c72a34ce00 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStoreTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStoreTests.java
@@ -27,6 +27,7 @@
 import org.elasticsearch.cluster.routing.UnassignedInfo;
 import org.elasticsearch.cluster.routing.UnassignedInfo.Reason;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.cluster.version.CompatibilityVersions;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.settings.Settings;
@@ -35,6 +36,7 @@
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.indices.SystemIndexDescriptor;
 import org.elasticsearch.license.MockLicenseState;
 import org.elasticsearch.license.TestUtils;
 import org.elasticsearch.license.XPackLicenseState;
@@ -65,6 +67,7 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicBoolean;
 
@@ -482,6 +485,7 @@ private ClusterState getClusterStateWithSecurityIndex() {
         }
 
         Index index = metadata.index(securityIndexName).getIndex();
+
         ShardRouting shardRouting = ShardRouting.newUnassigned(
             new ShardId(index, 0),
             true,
@@ -506,6 +510,13 @@ private ClusterState getClusterStateWithSecurityIndex() {
         ClusterState clusterState = ClusterState.builder(new ClusterName(NativeRolesStoreTests.class.getName()))
             .metadata(metadata)
             .routingTable(routingTable)
+            .putCompatibilityVersions(
+                "test",
+                new CompatibilityVersions(
+                    TransportVersion.current(),
+                    Map.of(".security-7", new SystemIndexDescriptor.MappingsVersion(1, 0))
+                )
+            )
             .build();
 
         return clusterState;
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/CacheInvalidatorRegistryTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/CacheInvalidatorRegistryTests.java
index 89d667de56c37..8849edca70d68 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/CacheInvalidatorRegistryTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/CacheInvalidatorRegistryTests.java
@@ -7,9 +7,9 @@
 
 package org.elasticsearch.xpack.security.support;
 
-import org.elasticsearch.Version;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.indices.SystemIndexDescriptor;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.security.support.CacheInvalidatorRegistry.CacheInvalidator;
 import org.junit.Before;
@@ -18,6 +18,7 @@
 import java.util.List;
 import java.util.Set;
 
+import static org.elasticsearch.xpack.security.support.SecuritySystemIndices.INTERNAL_MAIN_INDEX_MAPPINGS_FORMAT;
 import static org.hamcrest.Matchers.containsString;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.Mockito.mock;
@@ -60,11 +61,10 @@ public void testSecurityIndexStateChangeWillInvalidateAllRegisteredInvalidators(
             true,
             true,
             true,
-            Version.CURRENT,
+            new SystemIndexDescriptor.MappingsVersion(INTERNAL_MAIN_INDEX_MAPPINGS_FORMAT, 0),
             ".security",
             ClusterHealthStatus.GREEN,
             IndexMetadata.State.OPEN,
-            null,
             "my_uuid"
         );
 
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java
index c8f86957f84a3..2abeeb3fa040b 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java
@@ -7,7 +7,7 @@
 package org.elasticsearch.xpack.security.support;
 
 import org.elasticsearch.ElasticsearchStatusException;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionResponse;
@@ -33,6 +33,7 @@
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.UnassignedInfo;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.cluster.version.CompatibilityVersions;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.settings.Settings;
@@ -56,11 +57,13 @@
 import java.io.IOException;
 import java.io.UncheckedIOException;
 import java.util.HashSet;
+import java.util.Map;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.BiConsumer;
 
 import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder;
+import static org.elasticsearch.xpack.security.support.SecuritySystemIndices.INTERNAL_MAIN_INDEX_MAPPINGS_FORMAT;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;
@@ -75,15 +78,17 @@
 import static org.mockito.Mockito.when;
 
 public class SecurityIndexManagerTests extends ESTestCase {
-
     private static final ClusterName CLUSTER_NAME = new ClusterName("security-index-manager-tests");
     private static final ClusterState EMPTY_CLUSTER_STATE = new ClusterState.Builder(CLUSTER_NAME).build();
     private SystemIndexDescriptor descriptorSpy;
+    private ThreadPool threadPool;
     private SecurityIndexManager manager;
 
+    private int putMappingRequestCount = 0;
+
     @Before
     public void setUpManager() {
-        final ThreadPool threadPool = mock(ThreadPool.class);
+        threadPool = mock(ThreadPool.class);
         when(threadPool.getThreadContext()).thenReturn(new ThreadContext(Settings.EMPTY));
         when(threadPool.generic()).thenReturn(EsExecutors.DIRECT_EXECUTOR_SERVICE);
 
@@ -97,6 +102,7 @@ protected <Request extends ActionRequest, Response extends ActionResponse> void
                 ActionListener<Response> listener
             ) {
                 if (request instanceof PutMappingRequest) {
+                    putMappingRequestCount++;
                     listener.onResponse((Response) AcknowledgedResponse.of(true));
                 }
             }
@@ -383,7 +389,7 @@ public void testCanUpdateIndexMappings() {
 
         // Ensure that the mappings for the index are out-of-date, so that the security index manager will
         // attempt to update them.
-        String previousVersion = getPreviousVersion(Version.CURRENT);
+        int previousVersion = INTERNAL_MAIN_INDEX_MAPPINGS_FORMAT - 1;
 
         // State recovered with index, with mappings with a prior version
         ClusterState.Builder clusterStateBuilder = createClusterState(
@@ -394,31 +400,31 @@ public void testCanUpdateIndexMappings() {
             getMappings(previousVersion)
         );
         manager.clusterChanged(event(markShardsAvailable(clusterStateBuilder)));
-
         manager.prepareIndexIfNeededThenExecute(prepareException::set, () -> prepareRunnableCalled.set(true));
 
         assertThat(prepareRunnableCalled.get(), is(true));
         assertThat(prepareException.get(), nullValue());
+        // Verify that the client to send put mapping was used
+        assertThat(putMappingRequestCount, equalTo(1));
     }
 
     /**
      * Check that the security index manager will refuse to update mappings on an index
-     * if the corresponding {@link SystemIndexDescriptor} requires a higher node version
+     * if the corresponding {@link SystemIndexDescriptor} requires a higher mapping version
      * that the cluster's current minimum version.
      */
-    public void testCannotUpdateIndexMappingsWhenMinNodeVersionTooLow() {
+    public void testCannotUpdateIndexMappingsWhenMinMappingVersionTooLow() {
         final AtomicBoolean prepareRunnableCalled = new AtomicBoolean(false);
         final AtomicReference<Exception> prepareException = new AtomicReference<>(null);
 
         // Hard-code a failure here.
-        doReturn("Nope").when(descriptorSpy).getMinimumNodeVersionMessage(anyString());
-        doReturn(null).when(descriptorSpy).getDescriptorCompatibleWith(eq(Version.CURRENT));
+        doReturn("Nope").when(descriptorSpy).getMinimumMappingsVersionMessage(anyString());
+        doReturn(null).when(descriptorSpy).getDescriptorCompatibleWith(eq(new SystemIndexDescriptor.MappingsVersion(1, 0)));
 
         // Ensure that the mappings for the index are out-of-date, so that the security index manager will
         // attempt to update them.
-        String previousVersion = getPreviousVersion(Version.CURRENT);
+        int previousVersion = INTERNAL_MAIN_INDEX_MAPPINGS_FORMAT - 1;
 
-        // State recovered with index, with mappings with a prior version
         ClusterState.Builder clusterStateBuilder = createClusterState(
             TestRestrictedIndices.INTERNAL_SECURITY_MAIN_INDEX_7,
             SecuritySystemIndices.SECURITY_MAIN_ALIAS,
@@ -435,6 +441,55 @@ public void testCannotUpdateIndexMappingsWhenMinNodeVersionTooLow() {
         assertThat(exception, not(nullValue()));
         assertThat(exception, instanceOf(IllegalStateException.class));
         assertThat(exception.getMessage(), equalTo("Nope"));
+        // Verify that the client to send put mapping was never used
+        assertThat(putMappingRequestCount, equalTo(0));
+    }
+
+    /**
+     * Check that the security index manager will not update mappings on an index if the mapping version wasn't bumped
+     */
+    public void testNoUpdateWhenIndexMappingsVersionNotBumped() {
+        final AtomicBoolean prepareRunnableCalled = new AtomicBoolean(false);
+        final AtomicReference<Exception> prepareException = new AtomicReference<>(null);
+
+        ClusterState.Builder clusterStateBuilder = createClusterState(
+            TestRestrictedIndices.INTERNAL_SECURITY_MAIN_INDEX_7,
+            SecuritySystemIndices.SECURITY_MAIN_ALIAS,
+            SecuritySystemIndices.INTERNAL_MAIN_INDEX_FORMAT,
+            IndexMetadata.State.OPEN,
+            getMappings(INTERNAL_MAIN_INDEX_MAPPINGS_FORMAT)
+        );
+        manager.clusterChanged(event(markShardsAvailable(clusterStateBuilder)));
+        manager.prepareIndexIfNeededThenExecute(prepareException::set, () -> prepareRunnableCalled.set(true));
+
+        assertThat(prepareRunnableCalled.get(), is(true));
+        assertThat(prepareException.get(), is(nullValue()));
+        // Verify that the client to send put mapping was never used
+        assertThat(putMappingRequestCount, equalTo(0));
+    }
+
+    /**
+     * Check that the security index manager will not update mappings on an index if there is no mapping version in cluster state
+     */
+    public void testNoUpdateWhenNoIndexMappingsVersionInClusterState() {
+        final AtomicBoolean prepareRunnableCalled = new AtomicBoolean(false);
+        final AtomicReference<Exception> prepareException = new AtomicReference<>(null);
+
+        ClusterState.Builder clusterStateBuilder = createClusterState(
+            TestRestrictedIndices.INTERNAL_SECURITY_MAIN_INDEX_7,
+            SecuritySystemIndices.SECURITY_MAIN_ALIAS,
+            SecuritySystemIndices.INTERNAL_MAIN_INDEX_FORMAT,
+            IndexMetadata.State.OPEN,
+            getMappings(INTERNAL_MAIN_INDEX_MAPPINGS_FORMAT),
+            Map.of()
+        );
+        manager.clusterChanged(event(markShardsAvailable(clusterStateBuilder)));
+        manager.prepareIndexIfNeededThenExecute(prepareException::set, () -> prepareRunnableCalled.set(true));
+
+        assertThat(prepareRunnableCalled.get(), is(true));
+        assertThat(prepareException.get(), is(nullValue()));
+        // Verify that the client to send put mapping was never used
+        assertThat(putMappingRequestCount, equalTo(0));
     }
 
     public void testListenerNotCalledBeforeStateNotRecovered() {
@@ -566,13 +621,33 @@ private static ClusterState.Builder createClusterState(
         int format,
         IndexMetadata.State state,
         String mappings
+    ) {
+        return createClusterState(
+            indexName,
+            aliasName,
+            format,
+            state,
+            mappings,
+            Map.of(indexName, new SystemIndexDescriptor.MappingsVersion(1, 0))
+        );
+    }
+
+    private static ClusterState.Builder createClusterState(
+        String indexName,
+        String aliasName,
+        int format,
+        IndexMetadata.State state,
+        String mappings,
+        Map<String, SystemIndexDescriptor.MappingsVersion> compatibilityVersions
     ) {
         IndexMetadata.Builder indexMeta = getIndexMetadata(indexName, aliasName, format, state, mappings);
 
         Metadata.Builder metadataBuilder = new Metadata.Builder();
         metadataBuilder.put(indexMeta);
 
-        return ClusterState.builder(state()).metadata(metadataBuilder.build());
+        return ClusterState.builder(state())
+            .metadata(metadataBuilder.build())
+            .putCompatibilityVersions("test", new CompatibilityVersions(TransportVersion.current(), compatibilityVersions));
     }
 
     private ClusterState markShardsAvailable(ClusterState.Builder clusterStateBuilder) {
@@ -614,17 +689,21 @@ private static IndexMetadata.Builder getIndexMetadata(
     }
 
     private static String getMappings() {
-        return getMappings(Version.CURRENT.toString());
+        return getMappings(INTERNAL_MAIN_INDEX_MAPPINGS_FORMAT);
     }
 
-    private static String getMappings(String version) {
+    private static String getMappings(Integer version) {
         try {
             final XContentBuilder builder = jsonBuilder();
 
             builder.startObject();
             {
                 builder.startObject("_meta");
-                builder.field("security-version", version);
+                if (version != null) {
+                    builder.field(SystemIndexDescriptor.VERSION_META_KEY, version);
+                }
+                // This is expected to be ignored
+                builder.field("security-version", "8.13.0");
                 builder.endObject();
 
                 builder.field("dynamic", "strict");
@@ -643,12 +722,4 @@ private static String getMappings(String version) {
             throw new UncheckedIOException("Failed to build index mappings", e);
         }
     }
-
-    private String getPreviousVersion(Version version) {
-        if (version.minor == 0) {
-            return version.major - 1 + ".99.0";
-        }
-
-        return version.major + "." + (version.minor - 1) + ".0";
-    }
 }

From d6046219ba6f444ae5d4b8ed4eb8912948d38121 Mon Sep 17 00:00:00 2001
From: Henning Andersen <33268011+henningandersen@users.noreply.github.com>
Date: Fri, 26 Apr 2024 11:56:05 +0200
Subject: [PATCH 46/58] Utility methods for rounding to aligned size (#107908)

Add more utility methods for rounding to aligned size and use
these in computeRange and toPageAlignedSize.
---
 .../blobcache/BlobCacheUtils.java             | 36 +++++++++---
 .../blobcache/BlobCacheUtilsTests.java        | 58 ++++++++++++++++++-
 2 files changed, 83 insertions(+), 11 deletions(-)

diff --git a/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/BlobCacheUtils.java b/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/BlobCacheUtils.java
index ff273d99d3c41..aa28b9ed60c12 100644
--- a/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/BlobCacheUtils.java
+++ b/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/BlobCacheUtils.java
@@ -33,14 +33,32 @@ public static int toIntBytes(long l) {
     }
 
     /**
-     * Rounds the length up so that it is aligned on the next page size (defined by SharedBytes.PAGE_SIZE). For example
+     * Round down the size to the nearest aligned size &lt;= size.
      */
-    public static long toPageAlignedSize(long length) {
-        int remainder = (int) (length % SharedBytes.PAGE_SIZE);
-        if (remainder > 0L) {
-            return length + (SharedBytes.PAGE_SIZE - remainder);
+    public static long roundDownToAlignedSize(long size, long alignment) {
+        assert size >= 0;
+        assert alignment > 0;
+        return size / alignment * alignment;
+    }
+
+    /**
+     * Round up the size to the nearest aligned size &gt;= size
+     */
+    public static long roundUpToAlignedSize(long size, long alignment) {
+        assert size >= 0;
+        if (size == 0) {
+            return 0;
         }
-        return length;
+        assert alignment > 0;
+        return (((size - 1) / alignment) + 1) * alignment;
+    }
+
+    /**
+     * Rounds the length up so that it is aligned on the next page size (defined by SharedBytes.PAGE_SIZE).
+     */
+    public static long toPageAlignedSize(long length) {
+        int alignment = SharedBytes.PAGE_SIZE;
+        return roundUpToAlignedSize(length, alignment);
     }
 
     public static void throwEOF(long channelPos, long len) throws EOFException {
@@ -58,13 +76,13 @@ public static void ensureSeek(long pos, IndexInput input) throws IOException {
 
     public static ByteRange computeRange(long rangeSize, long position, long size, long blobLength) {
         return ByteRange.of(
-            (position / rangeSize) * rangeSize,
-            Math.min((((position + size - 1) / rangeSize) + 1) * rangeSize, blobLength)
+            roundDownToAlignedSize(position, rangeSize),
+            Math.min(roundUpToAlignedSize(position + size, rangeSize), blobLength)
         );
     }
 
     public static ByteRange computeRange(long rangeSize, long position, long size) {
-        return ByteRange.of((position / rangeSize) * rangeSize, (((position + size - 1) / rangeSize) + 1) * rangeSize);
+        return ByteRange.of(roundDownToAlignedSize(position, rangeSize), roundUpToAlignedSize(position + size, rangeSize));
     }
 
     public static void ensureSlice(String sliceName, long sliceOffset, long sliceLength, IndexInput input) {
diff --git a/x-pack/plugin/blob-cache/src/test/java/org/elasticsearch/blobcache/BlobCacheUtilsTests.java b/x-pack/plugin/blob-cache/src/test/java/org/elasticsearch/blobcache/BlobCacheUtilsTests.java
index 2b615dad05655..9c21ec6c7efd0 100644
--- a/x-pack/plugin/blob-cache/src/test/java/org/elasticsearch/blobcache/BlobCacheUtilsTests.java
+++ b/x-pack/plugin/blob-cache/src/test/java/org/elasticsearch/blobcache/BlobCacheUtilsTests.java
@@ -6,14 +6,16 @@
  */
 package org.elasticsearch.blobcache;
 
+import org.elasticsearch.blobcache.common.ByteRange;
 import org.elasticsearch.blobcache.shared.SharedBytes;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.test.ESTestCase;
-import org.hamcrest.Matchers;
 
 import java.io.EOFException;
 import java.nio.ByteBuffer;
 
+import static org.hamcrest.Matchers.equalTo;
+
 public class BlobCacheUtilsTests extends ESTestCase {
 
     public void testReadSafeThrows() {
@@ -25,6 +27,58 @@ public void testReadSafeThrows() {
     public void testToPageAlignedSize() {
         long value = randomLongBetween(0, Long.MAX_VALUE - SharedBytes.PAGE_SIZE);
         long expected = ((value - 1) / SharedBytes.PAGE_SIZE + 1) * SharedBytes.PAGE_SIZE;
-        assertThat(BlobCacheUtils.toPageAlignedSize(value), Matchers.equalTo(expected));
+        assertThat(BlobCacheUtils.toPageAlignedSize(value), equalTo(expected));
+        assertThat(BlobCacheUtils.toPageAlignedSize(value), equalTo(roundUpUsingRemainder(value, SharedBytes.PAGE_SIZE)));
+    }
+
+    public void testRoundUpToAlignment() {
+        assertThat(BlobCacheUtils.roundUpToAlignedSize(8, 4), equalTo(8L));
+        assertThat(BlobCacheUtils.roundUpToAlignedSize(9, 4), equalTo(12L));
+        assertThat(BlobCacheUtils.roundUpToAlignedSize(between(1, 4), 4), equalTo(4L));
+        long alignment = randomLongBetween(1, Long.MAX_VALUE / 2);
+        assertThat(BlobCacheUtils.roundUpToAlignedSize(0, alignment), equalTo(0L));
+        long value = randomLongBetween(0, Long.MAX_VALUE - alignment);
+        assertThat(BlobCacheUtils.roundUpToAlignedSize(value, alignment), equalTo(roundUpUsingRemainder(value, alignment)));
+    }
+
+    public void testRoundDownToAlignment() {
+        assertThat(BlobCacheUtils.roundDownToAlignedSize(8, 4), equalTo(8L));
+        assertThat(BlobCacheUtils.roundDownToAlignedSize(9, 4), equalTo(8L));
+        assertThat(BlobCacheUtils.roundDownToAlignedSize(between(0, 3), 4), equalTo(0L));
+        long alignment = randomLongBetween(1, Long.MAX_VALUE / 2);
+        long value = randomLongBetween(0, Long.MAX_VALUE);
+        assertThat(BlobCacheUtils.roundDownToAlignedSize(value, alignment), equalTo(roundDownUsingRemainder(value, alignment)));
+    }
+
+    public void testComputeRange() {
+        assertThat(BlobCacheUtils.computeRange(8, 8, 8), equalTo(ByteRange.of(8, 16)));
+        assertThat(BlobCacheUtils.computeRange(8, 9, 8), equalTo(ByteRange.of(8, 24)));
+        assertThat(BlobCacheUtils.computeRange(8, 8, 9), equalTo(ByteRange.of(8, 24)));
+
+        long large = randomLongBetween(24, 64);
+        assertThat(BlobCacheUtils.computeRange(8, 8, 8, large), equalTo(ByteRange.of(8, 16)));
+        assertThat(BlobCacheUtils.computeRange(8, 9, 8, large), equalTo(ByteRange.of(8, 24)));
+        assertThat(BlobCacheUtils.computeRange(8, 8, 9, large), equalTo(ByteRange.of(8, 24)));
+
+        long small = randomLongBetween(8, 16);
+        assertThat(BlobCacheUtils.computeRange(8, 8, 8, small), equalTo(ByteRange.of(8, small)));
+        assertThat(BlobCacheUtils.computeRange(8, 9, 8, small), equalTo(ByteRange.of(8, small)));
+        assertThat(BlobCacheUtils.computeRange(8, 8, 9, small), equalTo(ByteRange.of(8, small)));
+    }
+
+    private static long roundUpUsingRemainder(long value, long alignment) {
+        long remainder = value % alignment;
+        if (remainder > 0L) {
+            return value + (alignment - remainder);
+        }
+        return value;
+    }
+
+    private static long roundDownUsingRemainder(long value, long alignment) {
+        long remainder = value % alignment;
+        if (remainder > 0L) {
+            return value - remainder;
+        }
+        return value;
     }
 }

From af0c9566e5a8a037722e5cecca36173202994ae0 Mon Sep 17 00:00:00 2001
From: Panagiotis Bailis <pmpailis@gmail.com>
Date: Fri, 26 Apr 2024 14:18:41 +0300
Subject: [PATCH 47/58] Adding new RankFeature phase (#107099)

In this PR we add a new search phase, in-between query and fetch, that is responsible for applying any reranking needed.

The idea is to trim down the query phase results down to rank_window_size, reach out to the shards to extract any additional feature data if needed, and then use this information to rerank the top results, trim them down to size and pass them to fetch phase.
---
 .../action/search/FetchSearchPhase.java       | 22 +++---
 .../action/search/RankFeaturePhase.java       | 77 +++++++++++++++++++
 .../SearchDfsQueryThenFetchAsyncAction.java   |  2 +-
 .../SearchQueryThenFetchAsyncAction.java      |  2 +-
 .../action/search/FetchSearchPhaseTests.java  | 25 ++++--
 5 files changed, 110 insertions(+), 18 deletions(-)
 create mode 100644 server/src/main/java/org/elasticsearch/action/search/RankFeaturePhase.java

diff --git a/server/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java b/server/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java
index 1f06158951392..569e5aec6eca3 100644
--- a/server/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java
+++ b/server/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java
@@ -33,15 +33,21 @@ final class FetchSearchPhase extends SearchPhase {
     private final BiFunction<SearchResponseSections, AtomicArray<SearchPhaseResult>, SearchPhase> nextPhaseFactory;
     private final SearchPhaseContext context;
     private final Logger logger;
-    private final SearchPhaseResults<SearchPhaseResult> resultConsumer;
     private final SearchProgressListener progressListener;
     private final AggregatedDfs aggregatedDfs;
+    private final SearchPhaseController.ReducedQueryPhase reducedQueryPhase;
 
-    FetchSearchPhase(SearchPhaseResults<SearchPhaseResult> resultConsumer, AggregatedDfs aggregatedDfs, SearchPhaseContext context) {
+    FetchSearchPhase(
+        SearchPhaseResults<SearchPhaseResult> resultConsumer,
+        AggregatedDfs aggregatedDfs,
+        SearchPhaseContext context,
+        SearchPhaseController.ReducedQueryPhase reducedQueryPhase
+    ) {
         this(
             resultConsumer,
             aggregatedDfs,
             context,
+            reducedQueryPhase,
             (response, queryPhaseResults) -> new ExpandSearchPhase(
                 context,
                 response.hits,
@@ -54,6 +60,7 @@ final class FetchSearchPhase extends SearchPhase {
         SearchPhaseResults<SearchPhaseResult> resultConsumer,
         AggregatedDfs aggregatedDfs,
         SearchPhaseContext context,
+        SearchPhaseController.ReducedQueryPhase reducedQueryPhase,
         BiFunction<SearchResponseSections, AtomicArray<SearchPhaseResult>, SearchPhase> nextPhaseFactory
     ) {
         super("fetch");
@@ -72,18 +79,16 @@ final class FetchSearchPhase extends SearchPhase {
         this.nextPhaseFactory = nextPhaseFactory;
         this.context = context;
         this.logger = context.getLogger();
-        this.resultConsumer = resultConsumer;
         this.progressListener = context.getTask().getProgressListener();
+        this.reducedQueryPhase = reducedQueryPhase;
     }
 
     @Override
     public void run() {
         context.execute(new AbstractRunnable() {
+
             @Override
-            protected void doRun() throws Exception {
-                // we do the heavy lifting in this inner run method where we reduce aggs etc. that's why we fork this phase
-                // off immediately instead of forking when we send back the response to the user since there we only need
-                // to merge together the fetched results which is a linear operation.
+            protected void doRun() {
                 innerRun();
             }
 
@@ -94,9 +99,8 @@ public void onFailure(Exception e) {
         });
     }
 
-    private void innerRun() throws Exception {
+    private void innerRun() {
         final int numShards = context.getNumShards();
-        final SearchPhaseController.ReducedQueryPhase reducedQueryPhase = resultConsumer.reduce();
         // Usually when there is a single shard, we force the search type QUERY_THEN_FETCH. But when there's kNN, we might
         // still use DFS_QUERY_THEN_FETCH, which does not perform the "query and fetch" optimization during the query phase.
         final boolean queryAndFetchOptimization = queryResults.length() == 1
diff --git a/server/src/main/java/org/elasticsearch/action/search/RankFeaturePhase.java b/server/src/main/java/org/elasticsearch/action/search/RankFeaturePhase.java
new file mode 100644
index 0000000000000..a18d2c6418542
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/action/search/RankFeaturePhase.java
@@ -0,0 +1,77 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+package org.elasticsearch.action.search;
+
+import org.elasticsearch.common.util.concurrent.AbstractRunnable;
+import org.elasticsearch.search.SearchPhaseResult;
+import org.elasticsearch.search.dfs.AggregatedDfs;
+
+/**
+ * This search phase is responsible for executing any re-ranking needed for the given search request, iff that is applicable.
+ * It starts by retrieving {code num_shards * window_size} results from the query phase and reduces them to a global list of
+ * the top {@code window_size} results. It then reaches out to the shards to extract the needed feature data,
+ * and finally passes all this information to the appropriate {@code RankFeatureRankCoordinatorContext} which is responsible for reranking
+ * the results. If no rank query is specified, it proceeds directly to the next phase (FetchSearchPhase) by first reducing the results.
+ */
+public final class RankFeaturePhase extends SearchPhase {
+
+    private final SearchPhaseContext context;
+    private final SearchPhaseResults<SearchPhaseResult> queryPhaseResults;
+    private final SearchPhaseResults<SearchPhaseResult> rankPhaseResults;
+
+    private final AggregatedDfs aggregatedDfs;
+
+    RankFeaturePhase(SearchPhaseResults<SearchPhaseResult> queryPhaseResults, AggregatedDfs aggregatedDfs, SearchPhaseContext context) {
+        super("rank-feature");
+        if (context.getNumShards() != queryPhaseResults.getNumShards()) {
+            throw new IllegalStateException(
+                "number of shards must match the length of the query results but doesn't:"
+                    + context.getNumShards()
+                    + "!="
+                    + queryPhaseResults.getNumShards()
+            );
+        }
+        this.context = context;
+        this.queryPhaseResults = queryPhaseResults;
+        this.aggregatedDfs = aggregatedDfs;
+        this.rankPhaseResults = new ArraySearchPhaseResults<>(context.getNumShards());
+        context.addReleasable(rankPhaseResults);
+    }
+
+    @Override
+    public void run() {
+        context.execute(new AbstractRunnable() {
+            @Override
+            protected void doRun() throws Exception {
+                // we need to reduce the results at this point instead of fetch phase, so we fork this process similarly to how
+                // was set up at FetchSearchPhase.
+
+                // we do the heavy lifting in this inner run method where we reduce aggs etc
+                innerRun();
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                context.onPhaseFailure(RankFeaturePhase.this, "", e);
+            }
+        });
+    }
+
+    private void innerRun() throws Exception {
+        // other than running reduce, this is currently close to a no-op
+        SearchPhaseController.ReducedQueryPhase reducedQueryPhase = queryPhaseResults.reduce();
+        moveToNextPhase(queryPhaseResults, reducedQueryPhase);
+    }
+
+    private void moveToNextPhase(
+        SearchPhaseResults<SearchPhaseResult> phaseResults,
+        SearchPhaseController.ReducedQueryPhase reducedQueryPhase
+    ) {
+        context.executeNextPhase(this, new FetchSearchPhase(phaseResults, aggregatedDfs, context, reducedQueryPhase));
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchDfsQueryThenFetchAsyncAction.java b/server/src/main/java/org/elasticsearch/action/search/SearchDfsQueryThenFetchAsyncAction.java
index fcc848384866a..f0dca04efe374 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchDfsQueryThenFetchAsyncAction.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchDfsQueryThenFetchAsyncAction.java
@@ -100,7 +100,7 @@ protected SearchPhase getNextPhase(final SearchPhaseResults<DfsSearchResult> res
             aggregatedDfs,
             mergedKnnResults,
             queryPhaseResultConsumer,
-            (queryResults) -> new FetchSearchPhase(queryResults, aggregatedDfs, context),
+            (queryResults) -> new RankFeaturePhase(queryResults, aggregatedDfs, context),
             context
         );
     }
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncAction.java b/server/src/main/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncAction.java
index 3ad7c52567d14..4720653c29381 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncAction.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncAction.java
@@ -122,7 +122,7 @@ && getRequest().scroll() == null
 
     @Override
     protected SearchPhase getNextPhase(final SearchPhaseResults<SearchPhaseResult> results, SearchPhaseContext context) {
-        return new FetchSearchPhase(results, null, this);
+        return new RankFeaturePhase(results, null, this);
     }
 
     private ShardSearchRequest rewriteShardSearchRequest(ShardSearchRequest request) {
diff --git a/server/src/test/java/org/elasticsearch/action/search/FetchSearchPhaseTests.java b/server/src/test/java/org/elasticsearch/action/search/FetchSearchPhaseTests.java
index a2c5bed51f5e7..7b7061c0e1bc6 100644
--- a/server/src/test/java/org/elasticsearch/action/search/FetchSearchPhaseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/FetchSearchPhaseTests.java
@@ -47,7 +47,7 @@
 public class FetchSearchPhaseTests extends ESTestCase {
     private static final long FETCH_PROFILE_TIME = 555;
 
-    public void testShortcutQueryAndFetchOptimization() {
+    public void testShortcutQueryAndFetchOptimization() throws Exception {
         SearchPhaseController controller = new SearchPhaseController((t, s) -> InternalAggregationTestCase.emptyReduceContextBuilder());
         MockSearchPhaseContext mockSearchPhaseContext = new MockSearchPhaseContext(1);
         try (
@@ -99,11 +99,12 @@ public void testShortcutQueryAndFetchOptimization() {
             } else {
                 numHits = 0;
             }
-
+            SearchPhaseController.ReducedQueryPhase reducedQueryPhase = results.reduce();
             FetchSearchPhase phase = new FetchSearchPhase(
                 results,
                 null,
                 mockSearchPhaseContext,
+                reducedQueryPhase,
                 (searchResponse, scrollId) -> new SearchPhase("test") {
                     @Override
                     public void run() {
@@ -141,7 +142,7 @@ private void assertProfiles(boolean profiled, int totalShards, SearchResponse se
         }
     }
 
-    public void testFetchTwoDocument() {
+    public void testFetchTwoDocument() throws Exception {
         MockSearchPhaseContext mockSearchPhaseContext = new MockSearchPhaseContext(2);
         SearchPhaseController controller = new SearchPhaseController((t, s) -> InternalAggregationTestCase.emptyReduceContextBuilder());
         try (
@@ -231,10 +232,12 @@ public void sendExecuteFetch(
                     }
                 }
             };
+            SearchPhaseController.ReducedQueryPhase reducedQueryPhase = results.reduce();
             FetchSearchPhase phase = new FetchSearchPhase(
                 results,
                 null,
                 mockSearchPhaseContext,
+                reducedQueryPhase,
                 (searchResponse, scrollId) -> new SearchPhase("test") {
                     @Override
                     public void run() {
@@ -262,7 +265,7 @@ public void run() {
         }
     }
 
-    public void testFailFetchOneDoc() {
+    public void testFailFetchOneDoc() throws Exception {
         MockSearchPhaseContext mockSearchPhaseContext = new MockSearchPhaseContext(2);
         SearchPhaseController controller = new SearchPhaseController((t, s) -> InternalAggregationTestCase.emptyReduceContextBuilder());
         try (
@@ -343,10 +346,12 @@ public void sendExecuteFetch(
                     }
                 }
             };
+            SearchPhaseController.ReducedQueryPhase reducedQueryPhase = results.reduce();
             FetchSearchPhase phase = new FetchSearchPhase(
                 results,
                 null,
                 mockSearchPhaseContext,
+                reducedQueryPhase,
                 (searchResponse, scrollId) -> new SearchPhase("test") {
                     @Override
                     public void run() {
@@ -390,7 +395,7 @@ public void run() {
         }
     }
 
-    public void testFetchDocsConcurrently() throws InterruptedException {
+    public void testFetchDocsConcurrently() throws Exception {
         int resultSetSize = randomIntBetween(0, 100);
         // we use at least 2 hits otherwise this is subject to single shard optimization and we trip an assert...
         int numHits = randomIntBetween(2, 100); // also numshards --> 1 hit per shard
@@ -454,10 +459,12 @@ public void sendExecuteFetch(
                 }
             };
             CountDownLatch latch = new CountDownLatch(1);
+            SearchPhaseController.ReducedQueryPhase reducedQueryPhase = results.reduce();
             FetchSearchPhase phase = new FetchSearchPhase(
                 results,
                 null,
                 mockSearchPhaseContext,
+                reducedQueryPhase,
                 (searchResponse, scrollId) -> new SearchPhase("test") {
                     @Override
                     public void run() {
@@ -509,7 +516,7 @@ public void run() {
         }
     }
 
-    public void testExceptionFailsPhase() {
+    public void testExceptionFailsPhase() throws Exception {
         MockSearchPhaseContext mockSearchPhaseContext = new MockSearchPhaseContext(2);
         SearchPhaseController controller = new SearchPhaseController((t, s) -> InternalAggregationTestCase.emptyReduceContextBuilder());
         try (
@@ -600,10 +607,12 @@ public void sendExecuteFetch(
                     }
                 }
             };
+            SearchPhaseController.ReducedQueryPhase reducedQueryPhase = results.reduce();
             FetchSearchPhase phase = new FetchSearchPhase(
                 results,
                 null,
                 mockSearchPhaseContext,
+                reducedQueryPhase,
                 (searchResponse, scrollId) -> new SearchPhase("test") {
                     @Override
                     public void run() {
@@ -624,7 +633,7 @@ public void run() {
         }
     }
 
-    public void testCleanupIrrelevantContexts() { // contexts that are not fetched should be cleaned up
+    public void testCleanupIrrelevantContexts() throws Exception { // contexts that are not fetched should be cleaned up
         MockSearchPhaseContext mockSearchPhaseContext = new MockSearchPhaseContext(2);
         SearchPhaseController controller = new SearchPhaseController((t, s) -> InternalAggregationTestCase.emptyReduceContextBuilder());
         try (
@@ -705,10 +714,12 @@ public void sendExecuteFetch(
                     }
                 }
             };
+            SearchPhaseController.ReducedQueryPhase reducedQueryPhase = results.reduce();
             FetchSearchPhase phase = new FetchSearchPhase(
                 results,
                 null,
                 mockSearchPhaseContext,
+                reducedQueryPhase,
                 (searchResponse, scrollId) -> new SearchPhase("test") {
                     @Override
                     public void run() {

From 3183e6d6c93e0c22e5aaa77160d5afa36b1f62fd Mon Sep 17 00:00:00 2001
From: Kostas Krikellas <131142368+kkrik-es@users.noreply.github.com>
Date: Fri, 26 Apr 2024 15:35:31 +0300
Subject: [PATCH 48/58] Add ignored field values to synthetic source (#107567)

* Add ignored field values to synthetic source

* Update docs/changelog/107567.yaml

* initialize map

* yaml fix

* add node feature

* add comments

* small fixes

* missing cluster feature in yaml

* constants for chars, stored fields

* remove duplicate method

* throw exception on parse failure

* remove Base64 encoding

* add assert on IgnoredValuesFieldMapper::write

* changes from review

* simplify logic

* add comment

* rename classes

* rename _ignored_values to _ignored_source

* rename _ignored_values to _ignored_source
---
 docs/changelog/107567.yaml                    |   5 +
 .../indices.create/20_synthetic_source.yml    | 151 +++++++
 .../test/nodes.stats/11_indices_metrics.yml   |  51 +--
 server/src/main/java/module-info.java         |   1 +
 .../index/mapper/DocumentParserContext.java   |  33 ++
 .../mapper/IgnoreMalformedStoredValues.java   | 141 +------
 .../mapper/IgnoredSourceFieldMapper.java      | 133 ++++++
 .../index/mapper/MapperFeatures.java          |  24 ++
 .../index/mapper/ObjectMapper.java            |  20 +-
 .../index/mapper/SourceLoader.java            |  17 +
 .../index/mapper/XContentDataHelper.java      | 399 ++++++++++++++++++
 .../elasticsearch/indices/IndicesModule.java  |   2 +
 ...lasticsearch.features.FeatureSpecification |   1 +
 .../mapper/DocCountFieldMapperTests.java      |   5 +-
 .../index/mapper/DocumentMapperTests.java     |   2 +
 .../mapper/IgnoredSourceFieldMapperTests.java | 148 +++++++
 .../index/mapper/XContentDataHelperTests.java |  90 ++++
 .../indices/IndicesModuleTests.java           |   2 +
 18 files changed, 1059 insertions(+), 166 deletions(-)
 create mode 100644 docs/changelog/107567.yaml
 create mode 100644 server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java
 create mode 100644 server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java
 create mode 100644 server/src/main/java/org/elasticsearch/index/mapper/XContentDataHelper.java
 create mode 100644 server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java
 create mode 100644 server/src/test/java/org/elasticsearch/index/mapper/XContentDataHelperTests.java

diff --git a/docs/changelog/107567.yaml b/docs/changelog/107567.yaml
new file mode 100644
index 0000000000000..558b5b570b1fb
--- /dev/null
+++ b/docs/changelog/107567.yaml
@@ -0,0 +1,5 @@
+pr: 107567
+summary: Add ignored field values to synthetic source
+area: Mapping
+type: enhancement
+issues: []
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml
index 62a4e240a5b5d..39787366c0cc9 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml
@@ -36,3 +36,154 @@ nested is disabled:
                 properties:
                   foo:
                     type: keyword
+
+---
+object with unmapped fields:
+  - requires:
+      cluster_features: ["mapper.track_ignored_source"]
+      reason: requires tracking ignored values
+
+  - do:
+      indices.create:
+        index: test
+        body:
+          settings:
+            index:
+              mapping:
+                total_fields:
+                  ignore_dynamic_beyond_limit: true
+                  limit: 1
+
+          mappings:
+            _source:
+              mode: synthetic
+            properties:
+              name:
+                type: keyword
+
+  - do:
+      bulk:
+        index: test
+        refresh: true
+        body:
+          - '{ "create": { } }'
+          - '{ "name": "aaaa", "some_string": "AaAa", "some_int": 1000, "some_double": 123.456789, "some_bool": true, "a.very.deeply.nested.field": "AAAA" }'
+          - '{ "create": { } }'
+          - '{ "name": "bbbb", "some_string": "BbBb", "some_int": 2000, "some_double": 321.987654, "some_bool": false, "a.very.deeply.nested.field": "BBBB" }'
+
+  - do:
+      search:
+        index: test
+
+  - match: { hits.total.value: 2 }
+  - match: { hits.hits.0._source.name: aaaa }
+  - match: { hits.hits.0._source.some_string: AaAa }
+  - match: { hits.hits.0._source.some_int: 1000 }
+  - match: { hits.hits.0._source.some_double: 123.456789 }
+  - match: { hits.hits.0._source.a.very.deeply.nested.field: AAAA }
+  - match: { hits.hits.0._source.some_bool: true }
+  - match: { hits.hits.1._source.name: bbbb }
+  - match: { hits.hits.1._source.some_string: BbBb }
+  - match: { hits.hits.1._source.some_int: 2000 }
+  - match: { hits.hits.1._source.some_double: 321.987654 }
+  - match: { hits.hits.1._source.a.very.deeply.nested.field: BBBB }
+
+
+---
+nested object with unmapped fields:
+  - requires:
+      cluster_features: ["mapper.track_ignored_source"]
+      reason: requires tracking ignored values
+
+  - do:
+      indices.create:
+        index: test
+        body:
+          settings:
+            index:
+              mapping:
+                total_fields:
+                  ignore_dynamic_beyond_limit: true
+                  limit: 3
+
+          mappings:
+            _source:
+              mode: synthetic
+            properties:
+              path:
+                properties:
+                  to:
+                    properties:
+                      name:
+                        type: keyword
+
+  - do:
+      bulk:
+        index: test
+        refresh: true
+        body:
+          - '{ "create": { } }'
+          - '{ "path.to.name": "aaaa", "path.to.surname": "AaAa", "path.some.other.name": "AaAaAa"  }'
+          - '{ "create": { } }'
+          - '{ "path.to.name": "bbbb", "path.to.surname": "BbBb", "path.some.other.name": "BbBbBb"  }'
+
+  - do:
+      search:
+        index: test
+
+  - match: { hits.total.value: 2 }
+  - match: { hits.hits.0._source.path.to.name: aaaa }
+  - match: { hits.hits.0._source.path.to.surname: AaAa }
+  - match: { hits.hits.0._source.path.some.other.name: AaAaAa }
+  - match: { hits.hits.1._source.path.to.name: bbbb }
+  - match: { hits.hits.1._source.path.to.surname: BbBb }
+  - match: { hits.hits.1._source.path.some.other.name: BbBbBb }
+
+
+---
+empty object with unmapped fields:
+  - requires:
+      cluster_features: ["mapper.track_ignored_source"]
+      reason: requires tracking ignored values
+
+  - do:
+      indices.create:
+        index: test
+        body:
+          settings:
+            index:
+              mapping:
+                total_fields:
+                  ignore_dynamic_beyond_limit: true
+                  limit: 3
+
+          mappings:
+            _source:
+              mode: synthetic
+            properties:
+              path:
+                properties:
+                  to:
+                    properties:
+                      name:
+                        type: keyword
+
+  - do:
+      bulk:
+        index: test
+        refresh: true
+        body:
+          - '{ "create": { } }'
+          - '{ "path.to.surname": "AaAa", "path.some.other.name": "AaAaAa"  }'
+          - '{ "create": { } }'
+          - '{ "path.to.surname": "BbBb", "path.some.other.name": "BbBbBb"  }'
+
+  - do:
+      search:
+        index: test
+
+  - match: { hits.total.value: 2 }
+  - match: { hits.hits.0._source.path.to.surname: AaAa }
+  - match: { hits.hits.0._source.path.some.other.name: AaAaAa }
+  - match: { hits.hits.1._source.path.to.surname: BbBb }
+  - match: { hits.hits.1._source.path.some.other.name: BbBbBb }
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/nodes.stats/11_indices_metrics.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/nodes.stats/11_indices_metrics.yml
index 1a7da98af9129..ac0f8aec4f3d0 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/nodes.stats/11_indices_metrics.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/nodes.stats/11_indices_metrics.yml
@@ -417,8 +417,8 @@
 
   - requires:
       test_runner_features: [arbitrary_key]
-      cluster_features: ["gte_v8.5.0"]
-      reason:  "mappings added in version 8.5.0"
+      cluster_features: ["mapper.track_ignored_source"]
+      reason:  "_ignored_source added to mappings"
 
   - do:
       indices.create:
@@ -476,35 +476,36 @@
   # 4. _field_names
   # 5. _id
   # 6. _ignored
-  # 7. _index
-  # 8. _nested_path
-  # 9. _routing
-  # 10. _seq_no
-  # 11. _source
-  # 12. _tier
-  # 13. _version
-  # 14. @timestamp
-  # 15. authors.age
-  # 16. authors.company
-  # 17. authors.company.keyword
-  # 18. authors.name.last_name
-  # 19. authors.name.first_name
-  # 20. authors.name.full_name
-  # 21. link
-  # 22. title
-  # 23. url
+  # 7. _ignored_source
+  # 8. _index
+  # 9. _nested_path
+  # 10. _routing
+  # 11. _seq_no
+  # 12. _source
+  # 13. _tier
+  # 14. _version
+  # 15. @timestamp
+  # 16. authors.age
+  # 17. authors.company
+  # 18. authors.company.keyword
+  # 19. authors.name.last_name
+  # 20. authors.name.first_name
+  # 21. authors.name.full_name
+  # 22. link
+  # 23. title
+  # 24. url
   # Object mappers:
-  # 24. authors
-  # 25. authors.name
+  # 25. authors
+  # 26. authors.name
   # Runtime field mappers:
-  # 26. a_source_field
+  # 27. a_source_field
 
-  - gte: { nodes.$node_id.indices.mappings.total_count: 26 }
+  - gte: { nodes.$node_id.indices.mappings.total_count: 27 }
   - is_true: nodes.$node_id.indices.mappings.total_estimated_overhead
   - gte: { nodes.$node_id.indices.mappings.total_estimated_overhead_in_bytes: 26624 }
-  - match: { nodes.$node_id.indices.indices.index1.mappings.total_count: 26 }
+  - match: { nodes.$node_id.indices.indices.index1.mappings.total_count: 27 }
   - is_true: nodes.$node_id.indices.indices.index1.mappings.total_estimated_overhead
-  - match: { nodes.$node_id.indices.indices.index1.mappings.total_estimated_overhead_in_bytes: 26624 }
+  - match: { nodes.$node_id.indices.indices.index1.mappings.total_estimated_overhead_in_bytes: 27648 }
 
 ---
 "indices mappings does not exist in shards level":
diff --git a/server/src/main/java/module-info.java b/server/src/main/java/module-info.java
index 7560e2ff1a7d6..475158c7a8709 100644
--- a/server/src/main/java/module-info.java
+++ b/server/src/main/java/module-info.java
@@ -426,6 +426,7 @@
             org.elasticsearch.rest.RestFeatures,
             org.elasticsearch.indices.IndicesFeatures,
             org.elasticsearch.action.admin.cluster.allocation.AllocationStatsFeatures,
+            org.elasticsearch.index.mapper.MapperFeatures,
             org.elasticsearch.search.retriever.RetrieversFeatures;
 
     uses org.elasticsearch.plugins.internal.SettingsExtension;
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java
index a42477bed2146..de1266ae3a7ee 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java
@@ -104,6 +104,7 @@ public int get() {
     private final MappingParserContext mappingParserContext;
     private final SourceToParse sourceToParse;
     private final Set<String> ignoredFields;
+    private final List<IgnoredSourceFieldMapper.NameValue> ignoredFieldValues;
     private final Map<String, List<Mapper>> dynamicMappers;
     private final DynamicMapperSize dynamicMappersSize;
     private final Map<String, ObjectMapper> dynamicObjectMappers;
@@ -122,6 +123,7 @@ private DocumentParserContext(
         MappingParserContext mappingParserContext,
         SourceToParse sourceToParse,
         Set<String> ignoreFields,
+        List<IgnoredSourceFieldMapper.NameValue> ignoredFieldValues,
         Map<String, List<Mapper>> dynamicMappers,
         Map<String, ObjectMapper> dynamicObjectMappers,
         Map<String, List<RuntimeField>> dynamicRuntimeFields,
@@ -139,6 +141,7 @@ private DocumentParserContext(
         this.mappingParserContext = mappingParserContext;
         this.sourceToParse = sourceToParse;
         this.ignoredFields = ignoreFields;
+        this.ignoredFieldValues = ignoredFieldValues;
         this.dynamicMappers = dynamicMappers;
         this.dynamicObjectMappers = dynamicObjectMappers;
         this.dynamicRuntimeFields = dynamicRuntimeFields;
@@ -159,6 +162,7 @@ private DocumentParserContext(ObjectMapper parent, ObjectMapper.Dynamic dynamic,
             in.mappingParserContext,
             in.sourceToParse,
             in.ignoredFields,
+            in.ignoredFieldValues,
             in.dynamicMappers,
             in.dynamicObjectMappers,
             in.dynamicRuntimeFields,
@@ -186,6 +190,7 @@ protected DocumentParserContext(
             mappingParserContext,
             source,
             new HashSet<>(),
+            new ArrayList<>(),
             new HashMap<>(),
             new HashMap<>(),
             new HashMap<>(),
@@ -251,6 +256,20 @@ public final Collection<String> getIgnoredFields() {
         return Collections.unmodifiableCollection(ignoredFields);
     }
 
+    /**
+     * Add the given ignored values to the corresponding list.
+     */
+    public final void addIgnoredField(IgnoredSourceFieldMapper.NameValue values) {
+        ignoredFieldValues.add(values);
+    }
+
+    /**
+     * Return the collection of values for fields that have been ignored so far.
+     */
+    public final Collection<IgnoredSourceFieldMapper.NameValue> getIgnoredFieldValues() {
+        return Collections.unmodifiableCollection(ignoredFieldValues);
+    }
+
     /**
      * Add the given {@code field} to the _field_names field
      *
@@ -345,6 +364,20 @@ public final boolean addDynamicMapper(Mapper mapper) {
             int additionalFieldsToAdd = getNewFieldsSize() + mapperSize;
             if (indexSettings().isIgnoreDynamicFieldsBeyondLimit()) {
                 if (mappingLookup.exceedsLimit(indexSettings().getMappingTotalFieldsLimit(), additionalFieldsToAdd)) {
+                    if (indexSettings().getMode().isSyntheticSourceEnabled() || mappingLookup.isSourceSynthetic()) {
+                        try {
+                            int parentOffset = parent() instanceof RootObjectMapper ? 0 : parent().fullPath().length() + 1;
+                            addIgnoredField(
+                                new IgnoredSourceFieldMapper.NameValue(
+                                    mapper.name(),
+                                    parentOffset,
+                                    XContentDataHelper.encodeToken(parser())
+                                )
+                            );
+                        } catch (IOException e) {
+                            throw new IllegalArgumentException("failed to parse field [" + mapper.name() + " ]", e);
+                        }
+                    }
                     addIgnoredField(mapper.name());
                     return false;
                 }
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IgnoreMalformedStoredValues.java b/server/src/main/java/org/elasticsearch/index/mapper/IgnoreMalformedStoredValues.java
index b7990648539c1..52f4048e9b230 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/IgnoreMalformedStoredValues.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/IgnoreMalformedStoredValues.java
@@ -10,17 +10,10 @@
 
 import org.apache.lucene.document.StoredField;
 import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.BytesRefIterator;
-import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.util.ByteUtils;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentParser;
-import org.elasticsearch.xcontent.XContentParserConfiguration;
-import org.elasticsearch.xcontent.XContentType;
 
 import java.io.IOException;
-import java.math.BigDecimal;
-import java.math.BigInteger;
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Stream;
@@ -32,38 +25,8 @@
  * {@code _source}.
  */
 public abstract class IgnoreMalformedStoredValues {
-    /**
-     * Build a {@link StoredField} for the value on which the parser is
-     * currently positioned.
-     * <p>
-     * We try to use {@link StoredField}'s native types for fields where
-     * possible but we have to preserve more type information than
-     * stored fields support, so we encode all of those into stored fields'
-     * {@code byte[]} type and then encode type information in the first byte.
-     * </p>
-     */
-    public static StoredField storedField(String fieldName, XContentParser parser) throws IOException {
-        String name = name(fieldName);
-        return switch (parser.currentToken()) {
-            case VALUE_STRING -> new StoredField(name, parser.text());
-            case VALUE_NUMBER -> switch (parser.numberType()) {
-                case INT -> new StoredField(name, parser.intValue());
-                case LONG -> new StoredField(name, parser.longValue());
-                case DOUBLE -> new StoredField(name, parser.doubleValue());
-                case FLOAT -> new StoredField(name, parser.floatValue());
-                case BIG_INTEGER -> new StoredField(name, encode((BigInteger) parser.numberValue()));
-                case BIG_DECIMAL -> new StoredField(name, encode((BigDecimal) parser.numberValue()));
-            };
-            case VALUE_BOOLEAN -> new StoredField(name, new byte[] { parser.booleanValue() ? (byte) 't' : (byte) 'f' });
-            case VALUE_EMBEDDED_OBJECT -> new StoredField(name, encode(parser.binaryValue()));
-            case START_OBJECT, START_ARRAY -> {
-                try (XContentBuilder builder = XContentBuilder.builder(parser.contentType().xContent())) {
-                    builder.copyCurrentStructure(parser);
-                    yield new StoredField(name, encode(builder));
-                }
-            }
-            default -> throw new IllegalArgumentException("synthetic _source doesn't support malformed objects");
-        };
+    public static StoredField storedField(String name, XContentParser parser) throws IOException {
+        return XContentDataHelper.storedField(name(name), parser);
     }
 
     /**
@@ -136,114 +99,16 @@ public int count() {
         public void write(XContentBuilder b) throws IOException {
             for (Object v : values) {
                 if (v instanceof BytesRef r) {
-                    decodeAndWrite(b, r);
+                    XContentDataHelper.decodeAndWrite(b, r);
                 } else {
                     b.value(v);
                 }
             }
             values = emptyList();
         }
-
-        private static void decodeAndWrite(XContentBuilder b, BytesRef r) throws IOException {
-            switch (r.bytes[r.offset]) {
-                case 'b':
-                    b.value(r.bytes, r.offset + 1, r.length - 1);
-                    return;
-                case 'c':
-                    decodeAndWriteXContent(b, XContentType.CBOR, r);
-                    return;
-                case 'd':
-                    if (r.length < 5) {
-                        throw new IllegalArgumentException("Can't decode " + r);
-                    }
-                    int scale = ByteUtils.readIntLE(r.bytes, r.offset + 1);
-                    b.value(new BigDecimal(new BigInteger(r.bytes, r.offset + 5, r.length - 5), scale));
-                    return;
-                case 'f':
-                    if (r.length != 1) {
-                        throw new IllegalArgumentException("Can't decode " + r);
-                    }
-                    b.value(false);
-                    return;
-                case 'i':
-                    b.value(new BigInteger(r.bytes, r.offset + 1, r.length - 1));
-                    return;
-                case 'j':
-                    decodeAndWriteXContent(b, XContentType.JSON, r);
-                    return;
-                case 's':
-                    decodeAndWriteXContent(b, XContentType.SMILE, r);
-                    return;
-                case 't':
-                    if (r.length != 1) {
-                        throw new IllegalArgumentException("Can't decode " + r);
-                    }
-                    b.value(true);
-                    return;
-                case 'y':
-                    decodeAndWriteXContent(b, XContentType.YAML, r);
-                    return;
-                default:
-                    throw new IllegalArgumentException("Can't decode " + r);
-            }
-        }
-
-        private static void decodeAndWriteXContent(XContentBuilder b, XContentType type, BytesRef r) throws IOException {
-            try (
-                XContentParser parser = type.xContent().createParser(XContentParserConfiguration.EMPTY, r.bytes, r.offset + 1, r.length - 1)
-            ) {
-                b.copyCurrentStructure(parser);
-            }
-        }
     }
 
     private static String name(String fieldName) {
         return fieldName + "._ignore_malformed";
     }
-
-    private static byte[] encode(BigInteger n) {
-        byte[] twosCompliment = n.toByteArray();
-        byte[] encoded = new byte[1 + twosCompliment.length];
-        encoded[0] = 'i';
-        System.arraycopy(twosCompliment, 0, encoded, 1, twosCompliment.length);
-        return encoded;
-    }
-
-    private static byte[] encode(BigDecimal n) {
-        byte[] twosCompliment = n.unscaledValue().toByteArray();
-        byte[] encoded = new byte[5 + twosCompliment.length];
-        encoded[0] = 'd';
-        ByteUtils.writeIntLE(n.scale(), encoded, 1);
-        System.arraycopy(twosCompliment, 0, encoded, 5, twosCompliment.length);
-        return encoded;
-    }
-
-    private static byte[] encode(byte[] b) {
-        byte[] encoded = new byte[1 + b.length];
-        encoded[0] = 'b';
-        System.arraycopy(b, 0, encoded, 1, b.length);
-        return encoded;
-    }
-
-    private static byte[] encode(XContentBuilder builder) throws IOException {
-        BytesReference b = BytesReference.bytes(builder);
-        byte[] encoded = new byte[1 + b.length()];
-        encoded[0] = switch (builder.contentType()) {
-            case JSON -> 'j';
-            case SMILE -> 's';
-            case YAML -> 'y';
-            case CBOR -> 'c';
-            default -> throw new IllegalArgumentException("unsupported type " + builder.contentType());
-        };
-
-        int position = 1;
-        BytesRefIterator itr = b.iterator();
-        BytesRef ref;
-        while ((ref = itr.next()) != null) {
-            System.arraycopy(ref.bytes, ref.offset, encoded, position, ref.length);
-            position += ref.length;
-        }
-        assert position == encoded.length;
-        return encoded;
-    }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java
new file mode 100644
index 0000000000000..1daa7d1d674e3
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java
@@ -0,0 +1,133 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.index.mapper;
+
+import org.apache.lucene.document.StoredField;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.util.ByteUtils;
+import org.elasticsearch.features.NodeFeature;
+import org.elasticsearch.index.query.SearchExecutionContext;
+
+import java.nio.charset.StandardCharsets;
+import java.util.Collections;
+
+/**
+
+ * Mapper for the {@code _ignored_source} field.
+ *
+ * A field mapper that records fields that have been ignored, along with their values. It's intended for use
+ * in indexes with synthetic source to reconstruct the latter, taking into account fields that got ignored during
+ * indexing.
+ *
+ * This overlaps with {@link IgnoredFieldMapper} that tracks just the ignored field names. It's worth evaluating
+ * if we can replace it for all use cases to avoid duplication, assuming that the storage tradeoff is favorable.
+ */
+public class IgnoredSourceFieldMapper extends MetadataFieldMapper {
+
+    // This factor is used to combine two offsets within the same integer:
+    // - the offset of the end of the parent field within the field name (N / PARENT_OFFSET_IN_NAME_OFFSET)
+    // - the offset of the field value within the encoding string containing the offset (first 4 bytes), the field name and value
+    // (N % PARENT_OFFSET_IN_NAME_OFFSET)
+    private static final int PARENT_OFFSET_IN_NAME_OFFSET = 1 << 16;
+
+    public static final String NAME = "_ignored_source";
+
+    public static final IgnoredSourceFieldMapper INSTANCE = new IgnoredSourceFieldMapper();
+
+    public static final TypeParser PARSER = new FixedTypeParser(context -> INSTANCE);
+
+    static final NodeFeature TRACK_IGNORED_SOURCE = new NodeFeature("mapper.track_ignored_source");
+
+    /*
+     * Container for the ignored field data:
+     *  - the full name
+     *  - the offset in the full name indicating the end of the substring matching
+     *    the full name of the parent field
+     *  - the value, encoded as a byte array
+     */
+    public record NameValue(String name, int parentOffset, BytesRef value) {
+        String getParentFieldName() {
+            // _doc corresponds to the root object
+            return (parentOffset == 0) ? "_doc" : name.substring(0, parentOffset - 1);
+        }
+
+        String getFieldName() {
+            return parentOffset() == 0 ? name() : name().substring(parentOffset());
+        }
+    }
+
+    static final class IgnoredValuesFieldMapperType extends StringFieldType {
+
+        private static final IgnoredValuesFieldMapperType INSTANCE = new IgnoredValuesFieldMapperType();
+
+        private IgnoredValuesFieldMapperType() {
+            super(NAME, false, true, false, TextSearchInfo.NONE, Collections.emptyMap());
+        }
+
+        @Override
+        public String typeName() {
+            return NAME;
+        }
+
+        @Override
+        public ValueFetcher valueFetcher(SearchExecutionContext context, String format) {
+            return new StoredValueFetcher(context.lookup(), NAME);
+        }
+    }
+
+    private IgnoredSourceFieldMapper() {
+        super(IgnoredValuesFieldMapperType.INSTANCE);
+    }
+
+    @Override
+    protected String contentType() {
+        return NAME;
+    }
+
+    @Override
+    public void postParse(DocumentParserContext context) {
+        // Ignored values are only expected in synthetic mode.
+        assert context.getIgnoredFieldValues().isEmpty()
+            || context.indexSettings().getMode().isSyntheticSourceEnabled()
+            || context.mappingLookup().isSourceSynthetic();
+        for (NameValue nameValue : context.getIgnoredFieldValues()) {
+            context.doc().add(new StoredField(NAME, encode(nameValue)));
+        }
+    }
+
+    static byte[] encode(NameValue values) {
+        assert values.parentOffset < PARENT_OFFSET_IN_NAME_OFFSET;
+        assert values.parentOffset * (long) PARENT_OFFSET_IN_NAME_OFFSET < Integer.MAX_VALUE;
+
+        byte[] nameBytes = values.name.getBytes(StandardCharsets.UTF_8);
+        byte[] bytes = new byte[4 + nameBytes.length + values.value.length];
+        ByteUtils.writeIntLE(values.name.length() + PARENT_OFFSET_IN_NAME_OFFSET * values.parentOffset, bytes, 0);
+        System.arraycopy(nameBytes, 0, bytes, 4, nameBytes.length);
+        System.arraycopy(values.value.bytes, values.value.offset, bytes, 4 + nameBytes.length, values.value.length);
+        return bytes;
+    }
+
+    static NameValue decode(Object field) {
+        byte[] bytes = ((BytesRef) field).bytes;
+        int encodedSize = ByteUtils.readIntLE(bytes, 0);
+        int nameSize = encodedSize % PARENT_OFFSET_IN_NAME_OFFSET;
+        int parentOffset = encodedSize / PARENT_OFFSET_IN_NAME_OFFSET;
+        String name = new String(bytes, 4, nameSize, StandardCharsets.UTF_8);
+        BytesRef value = new BytesRef(bytes, 4 + nameSize, bytes.length - nameSize - 4);
+        return new NameValue(name, parentOffset, value);
+    }
+
+    // This mapper doesn't contribute to source directly as it has no access to the object structure. Instead, its contents
+    // are loaded by SourceLoader and passed to object mappers that, in turn, write their ignore fields at the appropriate level.
+    @Override
+    public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
+        return SourceLoader.SyntheticFieldLoader.NOTHING;
+    }
+
+}
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java
new file mode 100644
index 0000000000000..dc189aecab01c
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java
@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.index.mapper;
+
+import org.elasticsearch.features.FeatureSpecification;
+import org.elasticsearch.features.NodeFeature;
+
+import java.util.Set;
+
+/**
+ * Spec for mapper-related features.
+ */
+public class MapperFeatures implements FeatureSpecification {
+    @Override
+    public Set<NodeFeature> getFeatures() {
+        return Set.of(IgnoredSourceFieldMapper.TRACK_IGNORED_SOURCE);
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java
index ba396e9a72d30..dca6af2489910 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java
@@ -730,6 +730,7 @@ public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
     private class SyntheticSourceFieldLoader implements SourceLoader.SyntheticFieldLoader {
         private final List<SourceLoader.SyntheticFieldLoader> fields;
         private boolean hasValue;
+        private List<IgnoredSourceFieldMapper.NameValue> ignoredValues;
 
         private SyntheticSourceFieldLoader(List<SourceLoader.SyntheticFieldLoader> fields) {
             this.fields = fields;
@@ -793,8 +794,25 @@ public void write(XContentBuilder b) throws IOException {
                     field.write(b);
                 }
             }
-            b.endObject();
             hasValue = false;
+            if (ignoredValues != null) {
+                for (IgnoredSourceFieldMapper.NameValue ignored : ignoredValues) {
+                    b.field(ignored.getFieldName());
+                    XContentDataHelper.decodeAndWrite(b, ignored.value());
+                }
+                ignoredValues = null;
+            }
+            b.endObject();
+        }
+
+        @Override
+        public boolean setIgnoredValues(Map<String, List<IgnoredSourceFieldMapper.NameValue>> objectsWithIgnoredFields) {
+            ignoredValues = objectsWithIgnoredFields.get(name());
+            hasValue |= ignoredValues != null;
+            for (SourceLoader.SyntheticFieldLoader loader : fields) {
+                hasValue |= loader.setIgnoredValues(objectsWithIgnoredFields);
+            }
+            return this.ignoredValues != null;
         }
     }
 
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java
index c07821f3c9ae7..f37f494cb8865 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java
@@ -17,6 +17,8 @@
 
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -89,6 +91,7 @@ public Synthetic(Mapping mapping) {
                 .storedFieldLoaders()
                 .map(Map.Entry::getKey)
                 .collect(Collectors.toSet());
+            this.requiredStoredFields.add(IgnoredSourceFieldMapper.NAME);
         }
 
         @Override
@@ -122,12 +125,22 @@ private SyntheticLeaf(SyntheticFieldLoader loader, SyntheticFieldLoader.DocValue
 
             @Override
             public Source source(LeafStoredFieldLoader storedFieldLoader, int docId) throws IOException {
+                // Maps the names of existing objects to lists of ignored fields they contain.
+                Map<String, List<IgnoredSourceFieldMapper.NameValue>> objectsWithIgnoredFields = new HashMap<>();
+
                 for (Map.Entry<String, List<Object>> e : storedFieldLoader.storedFields().entrySet()) {
                     SyntheticFieldLoader.StoredFieldLoader loader = storedFieldLoaders.get(e.getKey());
                     if (loader != null) {
                         loader.load(e.getValue());
                     }
+                    if (IgnoredSourceFieldMapper.NAME.equals(e.getKey())) {
+                        for (Object value : e.getValue()) {
+                            IgnoredSourceFieldMapper.NameValue nameValue = IgnoredSourceFieldMapper.decode(value);
+                            objectsWithIgnoredFields.computeIfAbsent(nameValue.getParentFieldName(), k -> new ArrayList<>()).add(nameValue);
+                        }
+                    }
                 }
+                loader.setIgnoredValues(objectsWithIgnoredFields);
                 if (docValuesLoader != null) {
                     docValuesLoader.advanceToDoc(docId);
                 }
@@ -224,6 +237,10 @@ public void write(XContentBuilder b) {}
          */
         void write(XContentBuilder b) throws IOException;
 
+        default boolean setIgnoredValues(Map<String, List<IgnoredSourceFieldMapper.NameValue>> objectsWithIgnoredFields) {
+            return false;
+        }
+
         /**
          * Sync for stored field values.
          */
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/XContentDataHelper.java b/server/src/main/java/org/elasticsearch/index/mapper/XContentDataHelper.java
new file mode 100644
index 0000000000000..c41fbd5057227
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/index/mapper/XContentDataHelper.java
@@ -0,0 +1,399 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.index.mapper;
+
+import org.apache.lucene.document.StoredField;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.BytesRefIterator;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.util.ByteUtils;
+import org.elasticsearch.core.CheckedFunction;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentParser;
+import org.elasticsearch.xcontent.XContentParserConfiguration;
+import org.elasticsearch.xcontent.XContentType;
+
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+
+/**
+ * Helper class for processing field data of any type, as provided by the {@link XContentParser}.
+ */
+final class XContentDataHelper {
+    /**
+     * Build a {@link StoredField} for the value on which the parser is
+     * currently positioned.
+     * <p>
+     * We try to use {@link StoredField}'s native types for fields where
+     * possible but we have to preserve more type information than
+     * stored fields support, so we encode all of those into stored fields'
+     * {@code byte[]} type and then encode type information in the first byte.
+     * </p>
+     */
+    static StoredField storedField(String name, XContentParser parser) throws IOException {
+        return (StoredField) processToken(parser, typeUtils -> typeUtils.buildStoredField(name, parser));
+    }
+
+    /**
+     * Build a {@link BytesRef} wrapping a byte array containing an encoded form
+     * the value on which the parser is currently positioned.
+     */
+    static BytesRef encodeToken(XContentParser parser) throws IOException {
+        return new BytesRef((byte[]) processToken(parser, (typeUtils) -> typeUtils.encode(parser)));
+    }
+
+    /**
+     * Decode the value in the passed {@link BytesRef} and add it as a value to the
+     * passed build. The assumption is that the passed value has encoded using the function
+     * {@link #encodeToken(XContentParser)} above.
+     */
+    static void decodeAndWrite(XContentBuilder b, BytesRef r) throws IOException {
+        switch ((char) r.bytes[r.offset]) {
+            case BINARY_ENCODING -> TypeUtils.EMBEDDED_OBJECT.decodeAndWrite(b, r);
+            case CBOR_OBJECT_ENCODING, JSON_OBJECT_ENCODING, YAML_OBJECT_ENCODING, SMILE_OBJECT_ENCODING -> {
+                TypeUtils.START.decodeAndWrite(b, r);
+            }
+            case BIG_DECIMAL_ENCODING -> TypeUtils.BIG_DECIMAL.decodeAndWrite(b, r);
+            case FALSE_ENCODING, TRUE_ENCODING -> TypeUtils.BOOLEAN.decodeAndWrite(b, r);
+            case BIG_INTEGER_ENCODING -> TypeUtils.BIG_INTEGER.decodeAndWrite(b, r);
+            case STRING_ENCODING -> TypeUtils.STRING.decodeAndWrite(b, r);
+            case INTEGER_ENCODING -> TypeUtils.INTEGER.decodeAndWrite(b, r);
+            case LONG_ENCODING -> TypeUtils.LONG.decodeAndWrite(b, r);
+            case DOUBLE_ENCODING -> TypeUtils.DOUBLE.decodeAndWrite(b, r);
+            case FLOAT_ENCODING -> TypeUtils.FLOAT.decodeAndWrite(b, r);
+            default -> throw new IllegalArgumentException("Can't decode " + r);
+        }
+    }
+
+    private static Object processToken(XContentParser parser, CheckedFunction<TypeUtils, Object, IOException> visitor) throws IOException {
+        return switch (parser.currentToken()) {
+            case VALUE_STRING -> visitor.apply(TypeUtils.STRING);
+            case VALUE_NUMBER -> switch (parser.numberType()) {
+                case INT -> visitor.apply(TypeUtils.INTEGER);
+                case LONG -> visitor.apply(TypeUtils.LONG);
+                case DOUBLE -> visitor.apply(TypeUtils.DOUBLE);
+                case FLOAT -> visitor.apply(TypeUtils.FLOAT);
+                case BIG_INTEGER -> visitor.apply(TypeUtils.BIG_INTEGER);
+                case BIG_DECIMAL -> visitor.apply(TypeUtils.BIG_DECIMAL);
+            };
+            case VALUE_BOOLEAN -> visitor.apply(TypeUtils.BOOLEAN);
+            case VALUE_EMBEDDED_OBJECT -> visitor.apply(TypeUtils.EMBEDDED_OBJECT);
+            case START_OBJECT, START_ARRAY -> visitor.apply(TypeUtils.START);
+            default -> throw new IllegalArgumentException("synthetic _source doesn't support malformed objects");
+        };
+    }
+
+    private static final char STRING_ENCODING = 'S';
+    private static final char INTEGER_ENCODING = 'I';
+    private static final char LONG_ENCODING = 'L';
+    private static final char DOUBLE_ENCODING = 'D';
+    private static final char FLOAT_ENCODING = 'F';
+    private static final char BIG_INTEGER_ENCODING = 'i';
+    private static final char BIG_DECIMAL_ENCODING = 'd';
+    private static final char FALSE_ENCODING = 'f';
+    private static final char TRUE_ENCODING = 't';
+    private static final char BINARY_ENCODING = 'b';
+    private static final char CBOR_OBJECT_ENCODING = 'c';
+    private static final char JSON_OBJECT_ENCODING = 'j';
+    private static final char YAML_OBJECT_ENCODING = 'y';
+    private static final char SMILE_OBJECT_ENCODING = 's';
+
+    private enum TypeUtils {
+        STRING(STRING_ENCODING) {
+            @Override
+            StoredField buildStoredField(String name, XContentParser parser) throws IOException {
+                return new StoredField(name, parser.text());
+            }
+
+            @Override
+            byte[] encode(XContentParser parser) throws IOException {
+                byte[] text = parser.text().getBytes(StandardCharsets.UTF_8);
+                byte[] bytes = new byte[text.length + 1];
+                bytes[0] = getEncoding();
+                System.arraycopy(text, 0, bytes, 1, text.length);
+                assertValidEncoding(bytes);
+                return bytes;
+            }
+
+            @Override
+            void decodeAndWrite(XContentBuilder b, BytesRef r) throws IOException {
+                b.value(new BytesRef(r.bytes, r.offset + 1, r.length - 1).utf8ToString());
+            }
+        },
+        INTEGER(INTEGER_ENCODING) {
+            @Override
+            StoredField buildStoredField(String name, XContentParser parser) throws IOException {
+                return new StoredField(name, parser.intValue());
+            }
+
+            @Override
+            byte[] encode(XContentParser parser) throws IOException {
+                byte[] bytes = new byte[5];
+                bytes[0] = getEncoding();
+                ByteUtils.writeIntLE(parser.intValue(), bytes, 1);
+                assertValidEncoding(bytes);
+                return bytes;
+            }
+
+            @Override
+            void decodeAndWrite(XContentBuilder b, BytesRef r) throws IOException {
+                b.value(ByteUtils.readIntLE(r.bytes, 1 + r.offset));
+            }
+        },
+        LONG(LONG_ENCODING) {
+            @Override
+            StoredField buildStoredField(String name, XContentParser parser) throws IOException {
+                return new StoredField(name, parser.longValue());
+            }
+
+            @Override
+            byte[] encode(XContentParser parser) throws IOException {
+                byte[] bytes = new byte[9];
+                bytes[0] = getEncoding();
+                ByteUtils.writeLongLE(parser.longValue(), bytes, 1);
+                assertValidEncoding(bytes);
+                return bytes;
+            }
+
+            @Override
+            void decodeAndWrite(XContentBuilder b, BytesRef r) throws IOException {
+                b.value(ByteUtils.readLongLE(r.bytes, 1 + r.offset));
+            }
+        },
+        DOUBLE(DOUBLE_ENCODING) {
+            @Override
+            StoredField buildStoredField(String name, XContentParser parser) throws IOException {
+                return new StoredField(name, parser.doubleValue());
+            }
+
+            @Override
+            byte[] encode(XContentParser parser) throws IOException {
+                byte[] bytes = new byte[9];
+                bytes[0] = getEncoding();
+                ByteUtils.writeDoubleLE(parser.doubleValue(), bytes, 1);
+                assertValidEncoding(bytes);
+                return bytes;
+            }
+
+            @Override
+            void decodeAndWrite(XContentBuilder b, BytesRef r) throws IOException {
+                b.value(ByteUtils.readDoubleLE(r.bytes, 1 + r.offset));
+            }
+        },
+        FLOAT(FLOAT_ENCODING) {
+            @Override
+            StoredField buildStoredField(String name, XContentParser parser) throws IOException {
+                return new StoredField(name, parser.floatValue());
+            }
+
+            @Override
+            byte[] encode(XContentParser parser) throws IOException {
+                byte[] bytes = new byte[5];
+                bytes[0] = getEncoding();
+                ByteUtils.writeFloatLE(parser.floatValue(), bytes, 1);
+                assertValidEncoding(bytes);
+                return bytes;
+            }
+
+            @Override
+            void decodeAndWrite(XContentBuilder b, BytesRef r) throws IOException {
+                b.value(ByteUtils.readFloatLE(r.bytes, 1 + r.offset));
+            }
+        },
+        BIG_INTEGER(BIG_INTEGER_ENCODING) {
+            @Override
+            StoredField buildStoredField(String name, XContentParser parser) throws IOException {
+                return new StoredField(name, encode(parser));
+            }
+
+            @Override
+            byte[] encode(XContentParser parser) throws IOException {
+                byte[] bytes = encode((BigInteger) parser.numberValue(), getEncoding());
+                assertValidEncoding(bytes);
+                return bytes;
+            }
+
+            @Override
+            void decodeAndWrite(XContentBuilder b, BytesRef r) throws IOException {
+                b.value(new BigInteger(r.bytes, r.offset + 1, r.length - 1));
+            }
+        },
+        BIG_DECIMAL(BIG_DECIMAL_ENCODING) {
+            @Override
+            StoredField buildStoredField(String name, XContentParser parser) throws IOException {
+                return new StoredField(name, encode(parser));
+            }
+
+            @Override
+            byte[] encode(XContentParser parser) throws IOException {
+                byte[] bytes = encode((BigDecimal) parser.numberValue(), getEncoding());
+                assertValidEncoding(bytes);
+                return bytes;
+            }
+
+            @Override
+            void decodeAndWrite(XContentBuilder b, BytesRef r) throws IOException {
+                if (r.length < 5) {
+                    throw new IllegalArgumentException("Can't decode " + r);
+                }
+                int scale = ByteUtils.readIntLE(r.bytes, r.offset + 1);
+                b.value(new BigDecimal(new BigInteger(r.bytes, r.offset + 5, r.length - 5), scale));
+            }
+        },
+        BOOLEAN(new Character[] { TRUE_ENCODING, FALSE_ENCODING }) {
+            @Override
+            StoredField buildStoredField(String name, XContentParser parser) throws IOException {
+                return new StoredField(name, encode(parser));
+            }
+
+            @Override
+            byte[] encode(XContentParser parser) throws IOException {
+                byte[] bytes = new byte[] { parser.booleanValue() ? (byte) 't' : (byte) 'f' };
+                assertValidEncoding(bytes);
+                return bytes;
+            }
+
+            @Override
+            void decodeAndWrite(XContentBuilder b, BytesRef r) throws IOException {
+                if (r.length != 1) {
+                    throw new IllegalArgumentException("Can't decode " + r);
+                }
+                assert r.bytes[r.offset] == 't' || r.bytes[r.offset] == 'f' : r.bytes[r.offset];
+                b.value(r.bytes[r.offset] == 't');
+            }
+        },
+        EMBEDDED_OBJECT(BINARY_ENCODING) {
+            @Override
+            StoredField buildStoredField(String name, XContentParser parser) throws IOException {
+                return new StoredField(name, encode(parser.binaryValue()));
+            }
+
+            @Override
+            byte[] encode(XContentParser parser) throws IOException {
+                byte[] bytes = encode(parser.binaryValue());
+                assertValidEncoding(bytes);
+                return bytes;
+            }
+
+            @Override
+            void decodeAndWrite(XContentBuilder b, BytesRef r) throws IOException {
+                b.value(r.bytes, r.offset + 1, r.length - 1);
+            }
+        },
+        START(new Character[] { CBOR_OBJECT_ENCODING, JSON_OBJECT_ENCODING, YAML_OBJECT_ENCODING, SMILE_OBJECT_ENCODING }) {
+            @Override
+            StoredField buildStoredField(String name, XContentParser parser) throws IOException {
+                return new StoredField(name, encode(parser));
+            }
+
+            @Override
+            byte[] encode(XContentParser parser) throws IOException {
+                try (XContentBuilder builder = XContentBuilder.builder(parser.contentType().xContent())) {
+                    builder.copyCurrentStructure(parser);
+                    byte[] bytes = encode(builder);
+                    assertValidEncoding(bytes);
+                    return bytes;
+                }
+            }
+
+            @Override
+            void decodeAndWrite(XContentBuilder b, BytesRef r) throws IOException {
+                switch ((char) r.bytes[r.offset]) {
+                    case CBOR_OBJECT_ENCODING -> decodeAndWriteXContent(b, XContentType.CBOR, r);
+                    case JSON_OBJECT_ENCODING -> decodeAndWriteXContent(b, XContentType.JSON, r);
+                    case SMILE_OBJECT_ENCODING -> decodeAndWriteXContent(b, XContentType.SMILE, r);
+                    case YAML_OBJECT_ENCODING -> decodeAndWriteXContent(b, XContentType.YAML, r);
+                    default -> throw new IllegalArgumentException("Can't decode " + r);
+                }
+            }
+        };
+
+        TypeUtils(char encoding) {
+            this.encoding = new Character[] { encoding };
+        }
+
+        TypeUtils(Character[] encoding) {
+            this.encoding = encoding;
+        }
+
+        byte getEncoding() {
+            assert encoding.length == 1;
+            return (byte) encoding[0].charValue();
+        }
+
+        void assertValidEncoding(byte[] encodedValue) {
+            assert Arrays.asList(encoding).contains((char) encodedValue[0]);
+        }
+
+        final Character[] encoding;
+
+        abstract StoredField buildStoredField(String name, XContentParser parser) throws IOException;
+
+        abstract byte[] encode(XContentParser parser) throws IOException;
+
+        abstract void decodeAndWrite(XContentBuilder b, BytesRef r) throws IOException;
+
+        static byte[] encode(BigInteger n, Byte encoding) throws IOException {
+            byte[] twosCompliment = n.toByteArray();
+            byte[] encoded = new byte[1 + twosCompliment.length];
+            encoded[0] = encoding;
+            System.arraycopy(twosCompliment, 0, encoded, 1, twosCompliment.length);
+            return encoded;
+        }
+
+        static byte[] encode(BigDecimal n, Byte encoding) {
+            byte[] twosCompliment = n.unscaledValue().toByteArray();
+            byte[] encoded = new byte[5 + twosCompliment.length];
+            encoded[0] = 'd';
+            ByteUtils.writeIntLE(n.scale(), encoded, 1);
+            System.arraycopy(twosCompliment, 0, encoded, 5, twosCompliment.length);
+            return encoded;
+        }
+
+        static byte[] encode(byte[] b) {
+            byte[] encoded = new byte[1 + b.length];
+            encoded[0] = 'b';
+            System.arraycopy(b, 0, encoded, 1, b.length);
+            return encoded;
+        }
+
+        static byte[] encode(XContentBuilder builder) throws IOException {
+            BytesReference b = BytesReference.bytes(builder);
+            byte[] encoded = new byte[1 + b.length()];
+            encoded[0] = switch (builder.contentType()) {
+                case JSON -> JSON_OBJECT_ENCODING;
+                case SMILE -> SMILE_OBJECT_ENCODING;
+                case YAML -> YAML_OBJECT_ENCODING;
+                case CBOR -> CBOR_OBJECT_ENCODING;
+                default -> throw new IllegalArgumentException("unsupported type " + builder.contentType());
+            };
+
+            int position = 1;
+            BytesRefIterator itr = b.iterator();
+            BytesRef ref;
+            while ((ref = itr.next()) != null) {
+                System.arraycopy(ref.bytes, ref.offset, encoded, position, ref.length);
+                position += ref.length;
+            }
+            assert position == encoded.length;
+            return encoded;
+        }
+
+        static void decodeAndWriteXContent(XContentBuilder b, XContentType type, BytesRef r) throws IOException {
+            try (
+                XContentParser parser = type.xContent().createParser(XContentParserConfiguration.EMPTY, r.bytes, r.offset + 1, r.length - 1)
+            ) {
+                b.copyCurrentStructure(parser);
+            }
+        }
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesModule.java b/server/src/main/java/org/elasticsearch/indices/IndicesModule.java
index b17777fc5a91e..17e0105d59d8c 100644
--- a/server/src/main/java/org/elasticsearch/indices/IndicesModule.java
+++ b/server/src/main/java/org/elasticsearch/indices/IndicesModule.java
@@ -39,6 +39,7 @@
 import org.elasticsearch.index.mapper.GeoPointScriptFieldType;
 import org.elasticsearch.index.mapper.IdFieldMapper;
 import org.elasticsearch.index.mapper.IgnoredFieldMapper;
+import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper;
 import org.elasticsearch.index.mapper.IndexFieldMapper;
 import org.elasticsearch.index.mapper.IpFieldMapper;
 import org.elasticsearch.index.mapper.IpScriptFieldType;
@@ -258,6 +259,7 @@ private static Map<String, MetadataFieldMapper.TypeParser> initBuiltInMetadataMa
         builtInMetadataMappers.put(TimeSeriesRoutingHashFieldMapper.NAME, TimeSeriesRoutingHashFieldMapper.PARSER);
         builtInMetadataMappers.put(IndexFieldMapper.NAME, IndexFieldMapper.PARSER);
         builtInMetadataMappers.put(SourceFieldMapper.NAME, SourceFieldMapper.PARSER);
+        builtInMetadataMappers.put(IgnoredSourceFieldMapper.NAME, IgnoredSourceFieldMapper.PARSER);
         builtInMetadataMappers.put(NestedPathFieldMapper.NAME, NestedPathFieldMapper.PARSER);
         builtInMetadataMappers.put(VersionFieldMapper.NAME, VersionFieldMapper.PARSER);
         builtInMetadataMappers.put(SeqNoFieldMapper.NAME, SeqNoFieldMapper.PARSER);
diff --git a/server/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification b/server/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification
index cdb35cb9ac660..a158f91903c70 100644
--- a/server/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification
+++ b/server/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification
@@ -13,4 +13,5 @@ org.elasticsearch.cluster.metadata.MetadataFeatures
 org.elasticsearch.rest.RestFeatures
 org.elasticsearch.indices.IndicesFeatures
 org.elasticsearch.action.admin.cluster.allocation.AllocationStatsFeatures
+org.elasticsearch.index.mapper.MapperFeatures
 org.elasticsearch.search.retriever.RetrieversFeatures
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocCountFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocCountFieldMapperTests.java
index c1fd872e89f45..06e70e84bbb67 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/DocCountFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/DocCountFieldMapperTests.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.index.fieldvisitor.StoredFieldLoader;
 import org.elasticsearch.xcontent.XContentParserConfiguration;
 import org.elasticsearch.xcontent.json.JsonXContent;
+import org.hamcrest.Matchers;
 
 import java.io.IOException;
 import java.util.List;
@@ -97,7 +98,7 @@ public void testSyntheticSourceMany() throws IOException {
             }
         }, reader -> {
             SourceLoader loader = mapper.mappingLookup().newSourceLoader();
-            assertTrue(loader.requiredStoredFields().isEmpty());
+            assertThat(loader.requiredStoredFields(), Matchers.contains("_ignored_source"));
             for (LeafReaderContext leaf : reader.leaves()) {
                 int[] docIds = IntStream.range(0, leaf.reader().maxDoc()).toArray();
                 SourceLoader.Leaf sourceLoaderLeaf = loader.leaf(leaf.reader(), docIds);
@@ -129,7 +130,7 @@ public void testSyntheticSourceManyDoNotHave() throws IOException {
             }
         }, reader -> {
             SourceLoader loader = mapper.mappingLookup().newSourceLoader();
-            assertTrue(loader.requiredStoredFields().isEmpty());
+            assertThat(loader.requiredStoredFields(), Matchers.contains("_ignored_source"));
             for (LeafReaderContext leaf : reader.leaves()) {
                 int[] docIds = IntStream.range(0, leaf.reader().maxDoc()).toArray();
                 SourceLoader.Leaf sourceLoaderLeaf = loader.leaf(leaf.reader(), docIds);
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperTests.java
index 486b33d9b155a..c210fb0654683 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperTests.java
@@ -319,6 +319,7 @@ public void testEmptyDocumentMapper() {
                 .item(DocCountFieldMapper.class)
                 .item(FieldNamesFieldMapper.class)
                 .item(IgnoredFieldMapper.class)
+                .item(IgnoredSourceFieldMapper.class)
                 .item(IndexFieldMapper.class)
                 .item(NestedPathFieldMapper.class)
                 .item(ProvidedIdFieldMapper.class)
@@ -336,6 +337,7 @@ public void testEmptyDocumentMapper() {
                 .item(FieldNamesFieldMapper.CONTENT_TYPE)
                 .item(IdFieldMapper.CONTENT_TYPE)
                 .item(IgnoredFieldMapper.CONTENT_TYPE)
+                .item(IgnoredSourceFieldMapper.NAME)
                 .item(IndexFieldMapper.CONTENT_TYPE)
                 .item(NestedPathFieldMapper.NAME)
                 .item(RoutingFieldMapper.CONTENT_TYPE)
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java
new file mode 100644
index 0000000000000..a21c3993d4f2b
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java
@@ -0,0 +1,148 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.index.mapper;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.CheckedConsumer;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.hamcrest.Matchers;
+
+import java.io.IOException;
+import java.math.BigInteger;
+import java.util.Base64;
+
+public class IgnoredSourceFieldMapperTests extends MapperServiceTestCase {
+
+    private String getSyntheticSource(CheckedConsumer<XContentBuilder, IOException> build) throws IOException {
+        DocumentMapper documentMapper = createMapperService(
+            Settings.builder()
+                .put("index.mapping.total_fields.limit", 2)
+                .put("index.mapping.total_fields.ignore_dynamic_beyond_limit", true)
+                .build(),
+            syntheticSourceMapping(b -> {
+                b.startObject("foo").field("type", "keyword").endObject();
+                b.startObject("bar").field("type", "object").endObject();
+            })
+        ).documentMapper();
+        return syntheticSource(documentMapper, build);
+    }
+
+    public void testIgnoredBoolean() throws IOException {
+        boolean value = randomBoolean();
+        assertEquals("{\"my_value\":" + value + "}", getSyntheticSource(b -> b.field("my_value", value)));
+    }
+
+    public void testIgnoredString() throws IOException {
+        String value = randomAlphaOfLength(5);
+        assertEquals("{\"my_value\":\"" + value + "\"}", getSyntheticSource(b -> b.field("my_value", value)));
+    }
+
+    public void testIgnoredInt() throws IOException {
+        int value = randomInt();
+        assertEquals("{\"my_value\":" + value + "}", getSyntheticSource(b -> b.field("my_value", value)));
+    }
+
+    public void testIgnoredLong() throws IOException {
+        long value = randomLong();
+        assertEquals("{\"my_value\":" + value + "}", getSyntheticSource(b -> b.field("my_value", value)));
+    }
+
+    public void testIgnoredFloat() throws IOException {
+        float value = randomFloat();
+        assertEquals("{\"my_value\":" + value + "}", getSyntheticSource(b -> b.field("my_value", value)));
+    }
+
+    public void testIgnoredDouble() throws IOException {
+        double value = randomDouble();
+        assertEquals("{\"my_value\":" + value + "}", getSyntheticSource(b -> b.field("my_value", value)));
+    }
+
+    public void testIgnoredBigInteger() throws IOException {
+        BigInteger value = randomBigInteger();
+        assertEquals("{\"my_value\":" + value + "}", getSyntheticSource(b -> b.field("my_value", value)));
+    }
+
+    public void testIgnoredBytes() throws IOException {
+        byte[] value = randomByteArrayOfLength(10);
+        assertEquals(
+            "{\"my_value\":\"" + Base64.getEncoder().encodeToString(value) + "\"}",
+            getSyntheticSource(b -> b.field("my_value", value))
+        );
+    }
+
+    public void testIgnoredObjectBoolean() throws IOException {
+        boolean value = randomBoolean();
+        assertEquals("{\"my_value\":" + value + "}", getSyntheticSource(b -> b.field("my_value", value)));
+    }
+
+    public void testMultipleIgnoredFieldsRootObject() throws IOException {
+        boolean booleanValue = randomBoolean();
+        int intValue = randomInt();
+        String stringValue = randomAlphaOfLength(20);
+        String syntheticSource = getSyntheticSource(b -> {
+            b.field("boolean_value", booleanValue);
+            b.field("int_value", intValue);
+            b.field("string_value", stringValue);
+        });
+        assertThat(syntheticSource, Matchers.containsString("\"boolean_value\":" + booleanValue));
+        assertThat(syntheticSource, Matchers.containsString("\"int_value\":" + intValue));
+        assertThat(syntheticSource, Matchers.containsString("\"string_value\":\"" + stringValue + "\""));
+    }
+
+    public void testMultipleIgnoredFieldsSameObject() throws IOException {
+        boolean booleanValue = randomBoolean();
+        int intValue = randomInt();
+        String stringValue = randomAlphaOfLength(20);
+        String syntheticSource = getSyntheticSource(b -> {
+            b.startObject("bar");
+            {
+                b.field("boolean_value", booleanValue);
+                b.field("int_value", intValue);
+                b.field("string_value", stringValue);
+            }
+            b.endObject();
+        });
+        assertThat(syntheticSource, Matchers.containsString("{\"bar\":{"));
+        assertThat(syntheticSource, Matchers.containsString("\"boolean_value\":" + booleanValue));
+        assertThat(syntheticSource, Matchers.containsString("\"int_value\":" + intValue));
+        assertThat(syntheticSource, Matchers.containsString("\"string_value\":\"" + stringValue + "\""));
+    }
+
+    public void testMultipleIgnoredFieldsManyObjects() throws IOException {
+        boolean booleanValue = randomBoolean();
+        int intValue = randomInt();
+        String stringValue = randomAlphaOfLength(20);
+        String syntheticSource = getSyntheticSource(b -> {
+            b.field("boolean_value", booleanValue);
+            b.startObject("path");
+            {
+                b.startObject("to");
+                {
+                    b.field("int_value", intValue);
+                    b.startObject("some");
+                    {
+                        b.startObject("deeply");
+                        {
+                            b.startObject("nested");
+                            b.field("string_value", stringValue);
+                            b.endObject();
+                        }
+                        b.endObject();
+                    }
+                    b.endObject();
+                }
+                b.endObject();
+            }
+            b.endObject();
+        });
+        assertThat(syntheticSource, Matchers.containsString("\"boolean_value\":" + booleanValue));
+        assertThat(syntheticSource, Matchers.containsString("\"path\":{\"to\":{\"int_value\":" + intValue));
+        assertThat(syntheticSource, Matchers.containsString("\"some\":{\"deeply\":{\"nested\":{\"string_value\":\"" + stringValue + "\""));
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/XContentDataHelperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/XContentDataHelperTests.java
new file mode 100644
index 0000000000000..06db79c3f9fb0
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/index/mapper/XContentDataHelperTests.java
@@ -0,0 +1,90 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.index.mapper;
+
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentFactory;
+import org.elasticsearch.xcontent.XContentParser;
+import org.elasticsearch.xcontent.json.JsonXContent;
+
+import java.io.IOException;
+import java.math.BigInteger;
+import java.util.List;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class XContentDataHelperTests extends ESTestCase {
+
+    private String encodeAndDecode(String value) throws IOException {
+        XContentParser p = createParser(JsonXContent.jsonXContent, "{ \"foo\": " + value + " }");
+        assertThat(p.nextToken(), equalTo(XContentParser.Token.START_OBJECT));
+        assertThat(p.nextToken(), equalTo(XContentParser.Token.FIELD_NAME));
+        assertThat(p.currentName(), equalTo("foo"));
+        p.nextToken();
+
+        XContentBuilder builder = XContentFactory.jsonBuilder();
+        builder.humanReadable(true);
+        XContentDataHelper.decodeAndWrite(builder, XContentDataHelper.encodeToken(p));
+        return Strings.toString(builder);
+    }
+
+    public void testBoolean() throws IOException {
+        boolean b = randomBoolean();
+        assertEquals(b, Boolean.parseBoolean(encodeAndDecode(Boolean.toString(b))));
+    }
+
+    public void testString() throws IOException {
+        String s = "\"" + randomAlphaOfLength(5) + "\"";
+        assertEquals(s, encodeAndDecode(s));
+    }
+
+    public void testInt() throws IOException {
+        int i = randomInt();
+        assertEquals(i, Integer.parseInt(encodeAndDecode(Integer.toString(i))));
+    }
+
+    public void testLong() throws IOException {
+        long l = randomLong();
+        assertEquals(l, Long.parseLong(encodeAndDecode(Long.toString(l))));
+    }
+
+    public void testFloat() throws IOException {
+        float f = randomFloat();
+        assertEquals(0, Float.compare(f, Float.parseFloat(encodeAndDecode(Float.toString(f)))));
+    }
+
+    public void testDouble() throws IOException {
+        double d = randomDouble();
+        assertEquals(0, Double.compare(d, Double.parseDouble(encodeAndDecode(Double.toString(d)))));
+    }
+
+    public void testBigInteger() throws IOException {
+        BigInteger i = randomBigInteger();
+        assertEquals(i, new BigInteger(encodeAndDecode(i.toString()), 10));
+    }
+
+    public void testObject() throws IOException {
+        String object = "{\"name\":\"foo\"}";
+        XContentParser p = createParser(JsonXContent.jsonXContent, object);
+        assertThat(p.nextToken(), equalTo(XContentParser.Token.START_OBJECT));
+        XContentBuilder builder = XContentFactory.jsonBuilder();
+        builder.humanReadable(true);
+        XContentDataHelper.decodeAndWrite(builder, XContentDataHelper.encodeToken(p));
+        assertEquals(object, Strings.toString(builder));
+    }
+
+    public void testArrayInt() throws IOException {
+        String values = "["
+            + String.join(",", List.of(Integer.toString(randomInt()), Integer.toString(randomInt()), Integer.toString(randomInt())))
+            + "]";
+        assertEquals(values, encodeAndDecode(values));
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java b/server/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java
index 0216bad7cf7a3..c173a22dcdf57 100644
--- a/server/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java
+++ b/server/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.index.mapper.FieldNamesFieldMapper;
 import org.elasticsearch.index.mapper.IdFieldMapper;
 import org.elasticsearch.index.mapper.IgnoredFieldMapper;
+import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper;
 import org.elasticsearch.index.mapper.IndexFieldMapper;
 import org.elasticsearch.index.mapper.KeywordFieldMapper;
 import org.elasticsearch.index.mapper.Mapper;
@@ -85,6 +86,7 @@ public Map<String, MetadataFieldMapper.TypeParser> getMetadataMappers() {
         TimeSeriesRoutingHashFieldMapper.NAME,
         IndexFieldMapper.NAME,
         SourceFieldMapper.NAME,
+        IgnoredSourceFieldMapper.NAME,
         NestedPathFieldMapper.NAME,
         VersionFieldMapper.NAME,
         SeqNoFieldMapper.NAME,

From fccb2e7d04e17e920e145cf590cfc76ae146d409 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Fri, 26 Apr 2024 08:41:22 -0400
Subject: [PATCH 49/58] ESQL: Estimate memory usage on `Block.Builder`
 (#107923)

This adds a method to `Block.Builder` that estimates the number of bytes
that'll be used by the `Block` that it builds. It's not always accurate,
but it is directional.
---
 .../compute/data/BooleanArrayBlock.java       |  2 +-
 .../data/BooleanVectorFixedBuilder.java       |  5 ++++
 .../compute/data/BytesRefArrayBlock.java      |  2 +-
 .../compute/data/BytesRefBlockBuilder.java    |  5 ++++
 .../compute/data/DoubleArrayBlock.java        |  2 +-
 .../data/DoubleVectorFixedBuilder.java        |  5 ++++
 .../compute/data/IntArrayBlock.java           |  2 +-
 .../elasticsearch/compute/data/IntBlock.java  |  7 ------
 .../compute/data/IntBlockBuilder.java         |  5 ----
 .../compute/data/IntVectorFixedBuilder.java   |  5 ++++
 .../compute/data/LongArrayBlock.java          |  2 +-
 .../compute/data/LongVectorFixedBuilder.java  |  5 ++++
 .../compute/data/AbstractBlockBuilder.java    |  5 ++++
 .../compute/data/AbstractVectorBuilder.java   |  5 ++++
 .../org/elasticsearch/compute/data/Block.java |  7 ++++++
 .../compute/data/ConstantNullBlock.java       |  5 ++++
 .../elasticsearch/compute/data/DocBlock.java  |  5 ++++
 .../elasticsearch/compute/data/DocVector.java |  2 +-
 .../data/SingletonOrdinalsBuilder.java        | 20 +++++++++++----
 .../elasticsearch/compute/data/Vector.java    |  7 ++++++
 .../compute/data/X-ArrayBlock.java.st         |  2 +-
 .../compute/data/X-Block.java.st              |  9 -------
 .../compute/data/X-BlockBuilder.java.st       | 11 ++++----
 .../compute/data/X-VectorFixedBuilder.java.st |  5 ++++
 .../compute/data/BlockBuilderTests.java       | 11 ++++++++
 .../compute/data/TestBlockBuilder.java        | 25 +++++++++++++++++++
 26 files changed, 127 insertions(+), 39 deletions(-)

diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanArrayBlock.java
index 710eb17f72f6a..2ec68d268ae8a 100644
--- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanArrayBlock.java
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanArrayBlock.java
@@ -20,7 +20,7 @@
  */
 final class BooleanArrayBlock extends AbstractArrayBlock implements BooleanBlock {
 
-    private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(BooleanArrayBlock.class);
+    static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(BooleanArrayBlock.class);
 
     private final BooleanArrayVector vector;
 
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanVectorFixedBuilder.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanVectorFixedBuilder.java
index 5977dc5de36f0..4cc2ec17b6ad4 100644
--- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanVectorFixedBuilder.java
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanVectorFixedBuilder.java
@@ -46,6 +46,11 @@ private static long ramBytesUsed(int size) {
             );
     }
 
+    @Override
+    public long estimatedBytes() {
+        return ramBytesUsed(values.length);
+    }
+
     @Override
     public BooleanVector build() {
         if (nextIndex < 0) {
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefArrayBlock.java
index 6cc66183db2ed..8eaf07b473a3a 100644
--- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefArrayBlock.java
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefArrayBlock.java
@@ -23,7 +23,7 @@
  */
 final class BytesRefArrayBlock extends AbstractArrayBlock implements BytesRefBlock {
 
-    private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(BytesRefArrayBlock.class);
+    static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(BytesRefArrayBlock.class);
 
     private final BytesRefArrayVector vector;
 
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefBlockBuilder.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefBlockBuilder.java
index 4ef7ed4084228..49075789ed4a4 100644
--- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefBlockBuilder.java
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefBlockBuilder.java
@@ -140,6 +140,11 @@ public BytesRefBlockBuilder mvOrdering(Block.MvOrdering mvOrdering) {
         return this;
     }
 
+    @Override
+    public long estimatedBytes() {
+        return super.estimatedBytes() + BytesRefArrayBlock.BASE_RAM_BYTES_USED + values.ramBytesUsed();
+    }
+
     private BytesRefBlock buildFromBytesArray() {
         assert estimatedBytes == 0 || firstValueIndexes != null;
         final BytesRefBlock theBlock;
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleArrayBlock.java
index d872a4938a734..d545fca4fca8d 100644
--- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleArrayBlock.java
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleArrayBlock.java
@@ -20,7 +20,7 @@
  */
 final class DoubleArrayBlock extends AbstractArrayBlock implements DoubleBlock {
 
-    private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(DoubleArrayBlock.class);
+    static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(DoubleArrayBlock.class);
 
     private final DoubleArrayVector vector;
 
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleVectorFixedBuilder.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleVectorFixedBuilder.java
index c58856afa0266..42cdd0f5667ff 100644
--- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleVectorFixedBuilder.java
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleVectorFixedBuilder.java
@@ -46,6 +46,11 @@ private static long ramBytesUsed(int size) {
             );
     }
 
+    @Override
+    public long estimatedBytes() {
+        return ramBytesUsed(values.length);
+    }
+
     @Override
     public DoubleVector build() {
         if (nextIndex < 0) {
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntArrayBlock.java
index 492769d1f3d43..41c9d3b84485d 100644
--- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntArrayBlock.java
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntArrayBlock.java
@@ -20,7 +20,7 @@
  */
 final class IntArrayBlock extends AbstractArrayBlock implements IntBlock {
 
-    private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(IntArrayBlock.class);
+    static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(IntArrayBlock.class);
 
     private final IntArrayVector vector;
 
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntBlock.java
index 2747862d534b7..e9d606b51c6a1 100644
--- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntBlock.java
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntBlock.java
@@ -223,13 +223,6 @@ sealed interface Builder extends Block.Builder, BlockLoader.IntBuilder permits I
         @Override
         Builder mvOrdering(Block.MvOrdering mvOrdering);
 
-        /**
-         * An estimate of the number of bytes the {@link IntBlock} created by
-         * {@link #build} will use. This may overestimate the size but shouldn't
-         * underestimate it.
-         */
-        long estimatedBytes();
-
         @Override
         IntBlock build();
     }
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntBlockBuilder.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntBlockBuilder.java
index 886bf98f4e049..85f943004de29 100644
--- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntBlockBuilder.java
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntBlockBuilder.java
@@ -182,9 +182,4 @@ public IntBlock build() {
             throw e;
         }
     }
-
-    @Override
-    public long estimatedBytes() {
-        return estimatedBytes;
-    }
 }
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntVectorFixedBuilder.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntVectorFixedBuilder.java
index b143e9d592dc6..77e3511a5cb54 100644
--- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntVectorFixedBuilder.java
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntVectorFixedBuilder.java
@@ -46,6 +46,11 @@ private static long ramBytesUsed(int size) {
             );
     }
 
+    @Override
+    public long estimatedBytes() {
+        return ramBytesUsed(values.length);
+    }
+
     @Override
     public IntVector build() {
         if (nextIndex < 0) {
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongArrayBlock.java
index 77ae863e41ff0..56370f718bae0 100644
--- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongArrayBlock.java
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongArrayBlock.java
@@ -20,7 +20,7 @@
  */
 final class LongArrayBlock extends AbstractArrayBlock implements LongBlock {
 
-    private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(LongArrayBlock.class);
+    static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(LongArrayBlock.class);
 
     private final LongArrayVector vector;
 
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongVectorFixedBuilder.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongVectorFixedBuilder.java
index ccf87da153667..2ad259198bf1b 100644
--- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongVectorFixedBuilder.java
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongVectorFixedBuilder.java
@@ -46,6 +46,11 @@ private static long ramBytesUsed(int size) {
             );
     }
 
+    @Override
+    public long estimatedBytes() {
+        return ramBytesUsed(values.length);
+    }
+
     @Override
     public LongVector build() {
         if (nextIndex < 0) {
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/AbstractBlockBuilder.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/AbstractBlockBuilder.java
index abf3a243b7682..5fac64735155d 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/AbstractBlockBuilder.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/AbstractBlockBuilder.java
@@ -120,6 +120,11 @@ protected final void finish() {
         }
     }
 
+    @Override
+    public long estimatedBytes() {
+        return estimatedBytes;
+    }
+
     /**
      * Called during implementations of {@link Block.Builder#build} as a last step
      * to mark the Builder as closed and make sure that further closes don't double
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/AbstractVectorBuilder.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/AbstractVectorBuilder.java
index 0f86a79700b4b..7ee4ff2441f4e 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/AbstractVectorBuilder.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/AbstractVectorBuilder.java
@@ -62,6 +62,11 @@ protected final void finish() {
         }
     }
 
+    @Override
+    public long estimatedBytes() {
+        return estimatedBytes;
+    }
+
     /**
      * Called during implementations of {@link Block.Builder#build} as a last step
      * to mark the Builder as closed and make sure that further closes don't double
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/Block.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/Block.java
index 1e6422a5c31da..709ad4165170d 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/Block.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/Block.java
@@ -200,6 +200,13 @@ interface Builder extends BlockLoader.Builder, Releasable {
          */
         Builder mvOrdering(Block.MvOrdering mvOrdering);
 
+        /**
+         * An estimate of the number of bytes the {@link Block} created by
+         * {@link #build} will use. This may overestimate the size but shouldn't
+         * underestimate it.
+         */
+        long estimatedBytes();
+
         /**
          * Builds the block. This method can be called multiple times.
          */
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/ConstantNullBlock.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/ConstantNullBlock.java
index 3df75f4bc1c56..bdeb5334e0da7 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/ConstantNullBlock.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/ConstantNullBlock.java
@@ -182,6 +182,11 @@ public Block.Builder mvOrdering(MvOrdering mvOrdering) {
             return this;
         }
 
+        @Override
+        public long estimatedBytes() {
+            return BASE_RAM_BYTES_USED;
+        }
+
         @Override
         public Block build() {
             if (closed) {
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/DocBlock.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/DocBlock.java
index 2751cd31fd362..f454abe7d2cfe 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/DocBlock.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/DocBlock.java
@@ -160,6 +160,11 @@ public Block.Builder mvOrdering(MvOrdering mvOrdering) {
             return this;
         }
 
+        @Override
+        public long estimatedBytes() {
+            return DocVector.BASE_RAM_BYTES_USED + shards.estimatedBytes() + segments.estimatedBytes() + docs.estimatedBytes();
+        }
+
         @Override
         public DocBlock build() {
             // Pass null for singleSegmentNonDecreasing so we calculate it when we first need it.
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/DocVector.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/DocVector.java
index 2404217d11f95..067fddd311cc7 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/DocVector.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/DocVector.java
@@ -18,7 +18,7 @@
  */
 public final class DocVector extends AbstractVector implements Vector {
 
-    private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(DocVector.class);
+    static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(DocVector.class);
 
     /**
      * Per position memory cost to build the shard segment doc map required
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/SingletonOrdinalsBuilder.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/SingletonOrdinalsBuilder.java
index fd9dd6a479298..576bde5cdf676 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/SingletonOrdinalsBuilder.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/SingletonOrdinalsBuilder.java
@@ -151,13 +151,23 @@ BytesRefBlock buildRegularBlock() {
         }
     }
 
+    @Override
+    public long estimatedBytes() {
+        /*
+         * This is a *terrible* estimate because we have no idea how big the
+         * values in the ordinals are.
+         */
+        long overhead = shouldBuildOrdinalsBlock() ? 5 : 20;
+        return ords.length * overhead;
+    }
+
     @Override
     public BytesRefBlock build() {
-        if (ords.length >= 2 * docValues.getValueCount() && ords.length >= 32) {
-            return buildOrdinal();
-        } else {
-            return buildRegularBlock();
-        }
+        return shouldBuildOrdinalsBlock() ? buildOrdinal() : buildRegularBlock();
+    }
+
+    boolean shouldBuildOrdinalsBlock() {
+        return ords.length >= 2 * docValues.getValueCount() && ords.length >= 32;
     }
 
     @Override
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/Vector.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/Vector.java
index 84722fad93b7f..89b39569be454 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/Vector.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/Vector.java
@@ -62,6 +62,13 @@ public interface Vector extends Accountable, RefCounted, Releasable {
      * This is {@link Releasable} and should be released after building the vector or if building the vector fails.
      */
     interface Builder extends Releasable {
+        /**
+         * An estimate of the number of bytes the {@link Vector} created by
+         * {@link #build} will use. This may overestimate the size but shouldn't
+         * underestimate it.
+         */
+        long estimatedBytes();
+
         /**
          * Builds the block. This method can be called multiple times.
          */
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-ArrayBlock.java.st b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-ArrayBlock.java.st
index a7c5f10032394..9b153317c8a0e 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-ArrayBlock.java.st
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-ArrayBlock.java.st
@@ -32,7 +32,7 @@ $endif$
  */
 final class $Type$ArrayBlock extends AbstractArrayBlock implements $Type$Block {
 
-    private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance($Type$ArrayBlock.class);
+    static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance($Type$ArrayBlock.class);
 
     private final $Type$ArrayVector vector;
 
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-Block.java.st b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-Block.java.st
index b82061b85760a..331a5713fa3d1 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-Block.java.st
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-Block.java.st
@@ -277,15 +277,6 @@ $endif$
         @Override
         Builder mvOrdering(Block.MvOrdering mvOrdering);
 
-$if(int)$
-        /**
-         * An estimate of the number of bytes the {@link IntBlock} created by
-         * {@link #build} will use. This may overestimate the size but shouldn't
-         * underestimate it.
-         */
-        long estimatedBytes();
-
-$endif$
         @Override
         $Type$Block build();
     }
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-BlockBuilder.java.st b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-BlockBuilder.java.st
index 347f37cd7828d..0d3d2293a1bb1 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-BlockBuilder.java.st
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-BlockBuilder.java.st
@@ -188,6 +188,11 @@ $endif$
     }
 
 $if(BytesRef)$
+    @Override
+    public long estimatedBytes() {
+        return super.estimatedBytes() + BytesRefArrayBlock.BASE_RAM_BYTES_USED + values.ramBytesUsed();
+    }
+
     private $Type$Block buildFromBytesArray() {
         assert estimatedBytes == 0 || firstValueIndexes != null;
         final $Type$Block theBlock;
@@ -295,11 +300,5 @@ $if(BytesRef)$
     public void extraClose() {
         Releasables.closeExpectNoException(values);
     }
-$elseif(int)$
-
-    @Override
-    public long estimatedBytes() {
-        return estimatedBytes;
-    }
 $endif$
 }
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-VectorFixedBuilder.java.st b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-VectorFixedBuilder.java.st
index 43401d59095f4..af783a2435251 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-VectorFixedBuilder.java.st
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-VectorFixedBuilder.java.st
@@ -46,6 +46,11 @@ final class $Type$VectorFixedBuilder implements $Type$Vector.FixedBuilder {
             );
     }
 
+    @Override
+    public long estimatedBytes() {
+        return ramBytesUsed(values.length);
+    }
+
     @Override
     public $Type$Vector build() {
         if (nextIndex < 0) {
diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BlockBuilderTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BlockBuilderTests.java
index a48e22e9ccefa..6b5c37ee26888 100644
--- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BlockBuilderTests.java
+++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BlockBuilderTests.java
@@ -21,8 +21,11 @@
 import java.util.ArrayList;
 import java.util.List;
 
+import static org.hamcrest.Matchers.both;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.lessThan;
 
 public class BlockBuilderTests extends ESTestCase {
     @ParametersFactory
@@ -58,6 +61,10 @@ private void testAllNullsImpl(Block.Builder builder, int numEntries) {
         for (int i = 0; i < numEntries; i++) {
             builder.appendNull();
         }
+        assertThat(
+            builder.estimatedBytes(),
+            both(greaterThan(blockFactory.breaker().getUsed() - 1024)).and(lessThan(blockFactory.breaker().getUsed() + 1024))
+        );
         try (Block block = builder.build()) {
             assertThat(block.getPositionCount(), is(numEntries));
             for (int p = 0; p < numEntries; p++) {
@@ -113,6 +120,10 @@ private void testBuild(int size, boolean nullable, int maxValueCount) {
         try (Block.Builder builder = elementType.newBlockBuilder(randomBoolean() ? size : 1, blockFactory)) {
             BasicBlockTests.RandomBlock random = BasicBlockTests.randomBlock(elementType, size, nullable, 1, maxValueCount, 0, 0);
             builder.copyFrom(random.block(), 0, random.block().getPositionCount());
+            assertThat(
+                builder.estimatedBytes(),
+                both(greaterThan(blockFactory.breaker().getUsed() - 1024)).and(lessThan(blockFactory.breaker().getUsed() + 1024))
+            );
             try (Block built = builder.build()) {
                 assertThat(built, equalTo(random.block()));
                 assertThat(blockFactory.breaker().getUsed(), equalTo(built.ramBytesUsed()));
diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/TestBlockBuilder.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/TestBlockBuilder.java
index 4595b26ca27aa..33a3531481df9 100644
--- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/TestBlockBuilder.java
+++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/TestBlockBuilder.java
@@ -113,6 +113,11 @@ public TestBlockBuilder mvOrdering(Block.MvOrdering mvOrdering) {
             return this;
         }
 
+        @Override
+        public long estimatedBytes() {
+            return builder.estimatedBytes();
+        }
+
         @Override
         public IntBlock build() {
             return builder.build();
@@ -168,6 +173,11 @@ public TestBlockBuilder mvOrdering(Block.MvOrdering mvOrdering) {
             return this;
         }
 
+        @Override
+        public long estimatedBytes() {
+            return builder.estimatedBytes();
+        }
+
         @Override
         public LongBlock build() {
             return builder.build();
@@ -223,6 +233,11 @@ public TestBlockBuilder mvOrdering(Block.MvOrdering mvOrdering) {
             return this;
         }
 
+        @Override
+        public long estimatedBytes() {
+            return builder.estimatedBytes();
+        }
+
         @Override
         public DoubleBlock build() {
             return builder.build();
@@ -278,6 +293,11 @@ public TestBlockBuilder mvOrdering(Block.MvOrdering mvOrdering) {
             return this;
         }
 
+        @Override
+        public long estimatedBytes() {
+            return builder.estimatedBytes();
+        }
+
         @Override
         public BytesRefBlock build() {
             return builder.build();
@@ -336,6 +356,11 @@ public TestBlockBuilder mvOrdering(Block.MvOrdering mvOrdering) {
             return this;
         }
 
+        @Override
+        public long estimatedBytes() {
+            return builder.estimatedBytes();
+        }
+
         @Override
         public BooleanBlock build() {
             return builder.build();

From 1507c8767e27762a1fe4ac6fe57bcfbf516c5ba2 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Fri, 26 Apr 2024 08:45:27 -0400
Subject: [PATCH 50/58] ESQL: Add a `HashLookupOperator` (#107894)

Adds and operator takes a `Block` on construction, builds a `BlockHash`,
and uses it to resolve keys to row offsets. Example time!

Say we hand construct the operator with the first two columns of this
`Page`:
| a |  b |  v |
| -:| --:| --:|
| 1 | 11 | 21 |
| 2 | 12 | 22 |
| 2 | 14 | 23 |
| 2 | 11 | 24 |

If we then fire this the first two columns of this `Page` into it, we'll
get the third column:
|     a |  b |   ord |
| -----:| --:| -----:|
|     2 | 14 |     2 |
|     1 | 11 |     0 |
|     3 | 11 |  null |
| [1,2] | 11 | [0,3] |

This is the first half of the of the `Operator` side of a hash join.
The second half is looking up values from those row offsets. That'd
mean adding the `v` column like so:
|     a |  b |   ord |       v |
| -----:| --:| -----:| -------:|
|     2 | 14 |     2 |      23 |
|     1 | 11 |     0 |      21 |
|     3 | 11 |  null |    null |
| [1,2] | 11 | [0,3] | [21,24] |

And *that* is comparatively simple.

Notice that I said this is the *Operator* side of a hash join. There's
no planning or distributed execution involved. Yet. And a hash join is
something you'd distribute. This `Operator` can run on a data node or a
coordinating node. It doesn't care. It just needs an input.
---
 .../org/elasticsearch/TransportVersions.java  |   1 +
 .../aggregation/blockhash/BlockHash.java      |   7 +
 .../blockhash/PackedValuesBlockHash.java      |  78 +++--
 .../org/elasticsearch/compute/data/Page.java  |  32 ++
 ...AbstractPageMappingToIteratorOperator.java | 291 ++++++++++++++++++
 .../compute/operator/HashLookupOperator.java  | 138 +++++++++
 .../compute/operator/ProjectOperator.java     |  26 +-
 .../elasticsearch/compute/OperatorTests.java  |  70 +++++
 ...eMappingToIteratorOperatorStatusTests.java |  60 ++++
 .../operator/HashLookupOperatorTests.java     |  48 +++
 .../operator/IteratorAppendPageTests.java     | 116 +++++++
 .../operator/IteratorRemovePageTests.java     | 118 +++++++
 .../compute/operator/OperatorTestCase.java    |   8 +-
 .../xpack/esql/plugin/EsqlPlugin.java         |   2 +
 14 files changed, 923 insertions(+), 72 deletions(-)
 create mode 100644 x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/AbstractPageMappingToIteratorOperator.java
 create mode 100644 x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/HashLookupOperator.java
 create mode 100644 x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/AbstractPageMappingToIteratorOperatorStatusTests.java
 create mode 100644 x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/HashLookupOperatorTests.java
 create mode 100644 x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/IteratorAppendPageTests.java
 create mode 100644 x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/IteratorRemovePageTests.java

diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java
index 2bdb3368e1b5c..fc4323e418b72 100644
--- a/server/src/main/java/org/elasticsearch/TransportVersions.java
+++ b/server/src/main/java/org/elasticsearch/TransportVersions.java
@@ -183,6 +183,7 @@ static TransportVersion def(int id) {
     public static final TransportVersion INDEX_SEGMENTS_VECTOR_FORMATS = def(8_642_00_0);
     public static final TransportVersion ADD_RESOURCE_ALREADY_UPLOADED_EXCEPTION = def(8_643_00_0);
     public static final TransportVersion ESQL_MV_ORDERING_SORTED_ASCENDING = def(8_644_00_0);
+    public static final TransportVersion ESQL_PAGE_MAPPING_TO_ITERATOR = def(8_645_00_0);
 
     /*
      * STOP! READ THIS FIRST! No, really,
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/BlockHash.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/BlockHash.java
index 1e7ecebc16a62..431d8fe3bcd5d 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/BlockHash.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/BlockHash.java
@@ -111,6 +111,13 @@ public static BlockHash build(List<GroupSpec> groups, BlockFactory blockFactory,
         return new PackedValuesBlockHash(groups, blockFactory, emitBatchSize);
     }
 
+    /**
+     * Temporary method to build a {@link PackedValuesBlockHash}.
+     */
+    public static BlockHash buildPackedValuesBlockHash(List<GroupSpec> groups, BlockFactory blockFactory, int emitBatchSize) {
+        return new PackedValuesBlockHash(groups, blockFactory, emitBatchSize);
+    }
+
     /**
      * Creates a specialized hash table that maps a {@link Block} of the given input element type to ids.
      */
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/PackedValuesBlockHash.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/PackedValuesBlockHash.java
index 85c535faf3180..769155db5ecfa 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/PackedValuesBlockHash.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/PackedValuesBlockHash.java
@@ -9,7 +9,6 @@
 
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.BytesRefBuilder;
-import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.BitArray;
@@ -24,6 +23,7 @@
 import org.elasticsearch.compute.data.Page;
 import org.elasticsearch.compute.operator.mvdedupe.BatchEncoder;
 import org.elasticsearch.compute.operator.mvdedupe.MultivalueDedupe;
+import org.elasticsearch.core.Releasable;
 import org.elasticsearch.core.ReleasableIterator;
 import org.elasticsearch.core.Releasables;
 
@@ -65,14 +65,14 @@ final class PackedValuesBlockHash extends BlockHash {
     private final BytesRefHash bytesRefHash;
     private final int nullTrackingBytes;
     private final BytesRefBuilder bytes = new BytesRefBuilder();
-    private final Group[] groups;
+    private final List<GroupSpec> specs;
 
     PackedValuesBlockHash(List<GroupSpec> specs, BlockFactory blockFactory, int emitBatchSize) {
         super(blockFactory);
-        this.groups = specs.stream().map(Group::new).toArray(Group[]::new);
+        this.specs = specs;
         this.emitBatchSize = emitBatchSize;
         this.bytesRefHash = new BytesRefHash(1, blockFactory.bigArrays());
-        this.nullTrackingBytes = (groups.length + 7) / 8;
+        this.nullTrackingBytes = (specs.size() + 7) / 8;
         bytes.grow(nullTrackingBytes);
     }
 
@@ -90,9 +90,9 @@ void add(Page page, GroupingAggregatorFunction.AddInput addInput, int batchSize)
     /**
      * The on-heap representation of a {@code for} loop for each group key.
      */
-    private static class Group {
+    private static class Group implements Releasable {
         final GroupSpec spec;
-        BatchEncoder encoder;
+        final BatchEncoder encoder;
         int positionOffset;
         int valueOffset;
         /**
@@ -107,18 +107,25 @@ private static class Group {
         int valueCount;
         int bytesStart;
 
-        Group(GroupSpec spec) {
+        Group(GroupSpec spec, Page page, int batchSize) {
             this.spec = spec;
+            this.encoder = MultivalueDedupe.batchEncoder(page.getBlock(spec.channel()), batchSize, true);
+        }
+
+        @Override
+        public void close() {
+            encoder.close();
         }
     }
 
     class AddWork extends AbstractAddBlock {
+        final Group[] groups;
         final int positionCount;
         int position;
 
         AddWork(Page page, GroupingAggregatorFunction.AddInput addInput, int batchSize) {
             super(blockFactory, emitBatchSize, addInput);
-            initializeGroupsForPage(page, batchSize);
+            this.groups = specs.stream().map(s -> new Group(s, page, batchSize)).toArray(Group[]::new);
             this.positionCount = page.getPositionCount();
         }
 
@@ -129,7 +136,7 @@ class AddWork extends AbstractAddBlock {
          */
         void add() {
             for (position = 0; position < positionCount; position++) {
-                boolean singleEntry = startPosition();
+                boolean singleEntry = startPosition(groups);
                 if (singleEntry) {
                     addSingleEntry();
                 } else {
@@ -140,7 +147,7 @@ void add() {
         }
 
         private void addSingleEntry() {
-            fillBytesSv();
+            fillBytesSv(groups);
             ords.appendInt(Math.toIntExact(hashOrdToGroup(bytesRefHash.add(bytes.get()))));
             addedValue(position);
         }
@@ -149,13 +156,13 @@ private void addMultipleEntries() {
             ords.beginPositionEntry();
             int g = 0;
             do {
-                fillBytesMv(g);
+                fillBytesMv(groups, g);
 
                 // emit ords
                 ords.appendInt(Math.toIntExact(hashOrdToGroup(bytesRefHash.add(bytes.get()))));
                 addedValueInMultivaluePosition(position);
 
-                g = rewindKeys();
+                g = rewindKeys(groups);
             } while (g >= 0);
             ords.endPositionEntry();
             for (Group group : groups) {
@@ -165,10 +172,7 @@ private void addMultipleEntries() {
 
         @Override
         public void close() {
-            Releasables.closeExpectNoException(
-                super::close,
-                Releasables.wrap(() -> Iterators.map(Iterators.forArray(groups), g -> g.encoder))
-            );
+            Releasables.closeExpectNoException(super::close, Releasables.wrap(groups));
         }
     }
 
@@ -178,14 +182,15 @@ public ReleasableIterator<IntBlock> lookup(Page page, ByteSizeValue targetBlockS
     }
 
     class LookupWork implements ReleasableIterator<IntBlock> {
+        private final Group[] groups;
         private final long targetBytesSize;
         private final int positionCount;
         private int position;
 
         LookupWork(Page page, long targetBytesSize, int batchSize) {
+            this.groups = specs.stream().map(s -> new Group(s, page, batchSize)).toArray(Group[]::new);
             this.positionCount = page.getPositionCount();
             this.targetBytesSize = targetBytesSize;
-            initializeGroupsForPage(page, batchSize);
         }
 
         @Override
@@ -198,7 +203,7 @@ public IntBlock next() {
             int size = Math.toIntExact(Math.min(Integer.MAX_VALUE, targetBytesSize / Integer.BYTES / 2));
             try (IntBlock.Builder ords = blockFactory.newIntBlockBuilder(size)) {
                 while (position < positionCount && ords.estimatedBytes() < targetBytesSize) {
-                    boolean singleEntry = startPosition();
+                    boolean singleEntry = startPosition(groups);
                     if (singleEntry) {
                         lookupSingleEntry(ords);
                     } else {
@@ -211,7 +216,7 @@ public IntBlock next() {
         }
 
         private void lookupSingleEntry(IntBlock.Builder ords) {
-            fillBytesSv();
+            fillBytesSv(groups);
             long found = bytesRefHash.find(bytes.get());
             if (found < 0) {
                 ords.appendNull();
@@ -226,7 +231,7 @@ private void lookupMultipleEntries(IntBlock.Builder ords) {
             int g = 0;
             int count = 0;
             do {
-                fillBytesMv(g);
+                fillBytesMv(groups, g);
 
                 // emit ords
                 long found = bytesRefHash.find(bytes.get());
@@ -248,7 +253,7 @@ private void lookupMultipleEntries(IntBlock.Builder ords) {
                         }
                     }
                 }
-                g = rewindKeys();
+                g = rewindKeys(groups);
             } while (g >= 0);
             if (firstFound < 0) {
                 ords.appendNull();
@@ -265,24 +270,17 @@ private void lookupMultipleEntries(IntBlock.Builder ords) {
 
         @Override
         public void close() {
-            Releasables.closeExpectNoException(Releasables.wrap(() -> Iterators.map(Iterators.forArray(groups), g -> g.encoder)));
-        }
-    }
-
-    private void initializeGroupsForPage(Page page, int batchSize) {
-        for (Group group : groups) {
-            Block b = page.getBlock(group.spec.channel());
-            group.encoder = MultivalueDedupe.batchEncoder(b, batchSize, true);
+            Releasables.closeExpectNoException(groups);
         }
     }
 
     /**
-     * Correctly position all {@link #groups}, clear the {@link #bytes},
+     * Correctly position all {@code groups}, clear the {@link #bytes},
      * and position it past the null tracking bytes. Call this before
      * encoding a new position.
      * @return true if this position has only a single ordinal
      */
-    private boolean startPosition() {
+    private boolean startPosition(Group[] groups) {
         boolean singleEntry = true;
         for (Group g : groups) {
             /*
@@ -304,7 +302,7 @@ private boolean startPosition() {
         return singleEntry;
     }
 
-    private void fillBytesSv() {
+    private void fillBytesSv(Group[] groups) {
         for (int g = 0; g < groups.length; g++) {
             Group group = groups[g];
             assert group.writtenValues == 0;
@@ -317,7 +315,7 @@ private void fillBytesSv() {
         }
     }
 
-    private void fillBytesMv(int startingGroup) {
+    private void fillBytesMv(Group[] groups, int startingGroup) {
         for (int g = startingGroup; g < groups.length; g++) {
             Group group = groups[g];
             group.bytesStart = bytes.length();
@@ -331,7 +329,7 @@ private void fillBytesMv(int startingGroup) {
         }
     }
 
-    private int rewindKeys() {
+    private int rewindKeys(Group[] groups) {
         int g = groups.length - 1;
         Group group = groups[g];
         bytes.setLength(group.bytesStart);
@@ -350,11 +348,11 @@ private int rewindKeys() {
     @Override
     public Block[] getKeys() {
         int size = Math.toIntExact(bytesRefHash.size());
-        BatchEncoder.Decoder[] decoders = new BatchEncoder.Decoder[groups.length];
-        Block.Builder[] builders = new Block.Builder[groups.length];
+        BatchEncoder.Decoder[] decoders = new BatchEncoder.Decoder[specs.size()];
+        Block.Builder[] builders = new Block.Builder[specs.size()];
         try {
             for (int g = 0; g < builders.length; g++) {
-                ElementType elementType = groups[g].spec.elementType();
+                ElementType elementType = specs.get(g).elementType();
                 decoders[g] = BatchEncoder.decoder(elementType);
                 builders[g] = elementType.newBlockBuilder(size, blockFactory);
             }
@@ -424,12 +422,12 @@ public String toString() {
         StringBuilder b = new StringBuilder();
         b.append("PackedValuesBlockHash{groups=[");
         boolean first = true;
-        for (int i = 0; i < groups.length; i++) {
+        for (int i = 0; i < specs.size(); i++) {
             if (i > 0) {
                 b.append(", ");
             }
-            Group group = groups[i];
-            b.append(group.spec.channel()).append(':').append(group.spec.elementType());
+            GroupSpec spec = specs.get(i);
+            b.append(spec.channel()).append(':').append(spec.elementType());
         }
         b.append("], entries=").append(bytesRefHash.size());
         b.append(", size=").append(ByteSizeValue.ofBytes(bytesRefHash.ramBytesUsed()));
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/Page.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/Page.java
index bb6e10c0595d8..4d41ab27312c3 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/Page.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/Page.java
@@ -259,4 +259,36 @@ public Page shallowCopy() {
         }
         return new Page(blocks);
     }
+
+    /**
+     * Returns a new page with blocks in the containing {@link Block}s
+     * shifted around or removed. The new {@link Page} will have as
+     * many blocks as the {@code length} of the provided array. Those
+     * blocks will be set to the block at the position of the
+     * <strong>value</strong> of each entry in the parameter.
+     */
+    public Page projectBlocks(int[] blockMapping) {
+        if (blocksReleased) {
+            throw new IllegalStateException("can't read released page");
+        }
+        Block[] mapped = new Block[blockMapping.length];
+        try {
+            for (int b = 0; b < blockMapping.length; b++) {
+                if (blockMapping[b] >= blocks.length) {
+                    throw new IllegalArgumentException(
+                        "Cannot project block with index [" + blockMapping[b] + "] from a page with size [" + blocks.length + "]"
+                    );
+                }
+                mapped[b] = blocks[blockMapping[b]];
+                mapped[b].incRef();
+            }
+            Page result = new Page(false, getPositionCount(), mapped);
+            mapped = null;
+            return result;
+        } finally {
+            if (mapped != null) {
+                Releasables.close(mapped);
+            }
+        }
+    }
 }
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/AbstractPageMappingToIteratorOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/AbstractPageMappingToIteratorOperator.java
new file mode 100644
index 0000000000000..4fb4053b0c0f4
--- /dev/null
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/AbstractPageMappingToIteratorOperator.java
@@ -0,0 +1,291 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.compute.operator;
+
+import org.elasticsearch.TransportVersion;
+import org.elasticsearch.TransportVersions;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.compute.data.Block;
+import org.elasticsearch.compute.data.Page;
+import org.elasticsearch.core.ReleasableIterator;
+import org.elasticsearch.core.Releasables;
+import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.Objects;
+import java.util.stream.IntStream;
+
+/**
+ * Maps a single {@link Page} into zero or more resulting pages.
+ */
+public abstract class AbstractPageMappingToIteratorOperator implements Operator {
+    private ReleasableIterator<Page> next;
+
+    private boolean finished = false;
+
+    /**
+     * Number of milliseconds this operation has spent receiving pages.
+     */
+    private long processNanos;
+
+    /**
+     * Count of pages that have been received by this operator.
+     */
+    private int pagesReceived;
+
+    /**
+     * Count of pages that have been emitted by this operator.
+     */
+    private int pagesEmitted;
+
+    /**
+     * Build and Iterator of results for a new page.
+     */
+    protected abstract ReleasableIterator<Page> receive(Page page);
+
+    /**
+     * Append an {@link Iterator} of {@link Block}s to a {@link Page}, one
+     * after the other. It's required that the iterator emit as many
+     * <strong>positions</strong> as there were in the page.
+     */
+    public static ReleasableIterator<Page> appendBlocks(Page page, ReleasableIterator<? extends Block> toAdd) {
+        return new AppendBlocksIterator(page, toAdd);
+    }
+
+    @Override
+    public abstract String toString();
+
+    @Override
+    public final boolean needsInput() {
+        return finished == false && (next == null || next.hasNext() == false);
+    }
+
+    @Override
+    public final void addInput(Page page) {
+        if (next != null) {
+            assert next.hasNext() == false : "has pending input page";
+            next.close();
+        }
+        if (page.getPositionCount() == 0) {
+            return;
+        }
+        next = new RuntimeTrackingIterator(receive(page));
+        pagesReceived++;
+    }
+
+    @Override
+    public final void finish() {
+        finished = true;
+    }
+
+    @Override
+    public final boolean isFinished() {
+        return finished && (next == null || next.hasNext() == false);
+    }
+
+    @Override
+    public final Page getOutput() {
+        if (next == null || next.hasNext() == false) {
+            return null;
+        }
+        Page ret = next.next();
+        pagesEmitted++;
+        return ret;
+    }
+
+    @Override
+    public final AbstractPageMappingToIteratorOperator.Status status() {
+        return status(processNanos, pagesReceived, pagesEmitted);
+    }
+
+    protected AbstractPageMappingToIteratorOperator.Status status(long processNanos, int pagesReceived, int pagesEmitted) {
+        return new AbstractPageMappingToIteratorOperator.Status(processNanos, pagesReceived, pagesEmitted);
+    }
+
+    @Override
+    public void close() {
+        Releasables.closeExpectNoException(next);
+    }
+
+    private class RuntimeTrackingIterator implements ReleasableIterator<Page> {
+        private final ReleasableIterator<Page> next;
+
+        private RuntimeTrackingIterator(ReleasableIterator<Page> next) {
+            this.next = next;
+        }
+
+        @Override
+        public boolean hasNext() {
+            return next.hasNext();
+        }
+
+        @Override
+        public Page next() {
+            long start = System.nanoTime();
+            Page out = next.next();
+            processNanos += System.nanoTime() - start;
+            return out;
+        }
+
+        @Override
+        public void close() {
+            next.close();
+        }
+    }
+
+    public static class Status implements Operator.Status {
+        public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(
+            Operator.Status.class,
+            "page_mapping_to_iterator",
+            AbstractPageMappingOperator.Status::new
+        );
+
+        private final long processNanos;
+        private final int pagesReceived;
+        private final int pagesEmitted;
+
+        public Status(long processNanos, int pagesProcessed, int pagesEmitted) {
+            this.processNanos = processNanos;
+            this.pagesReceived = pagesProcessed;
+            this.pagesEmitted = pagesEmitted;
+        }
+
+        protected Status(StreamInput in) throws IOException {
+            processNanos = in.readVLong();
+            pagesReceived = in.readVInt();
+            pagesEmitted = in.readVInt();
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeVLong(processNanos);
+            out.writeVInt(pagesReceived);
+            out.writeVInt(pagesEmitted);
+        }
+
+        @Override
+        public String getWriteableName() {
+            return ENTRY.name;
+        }
+
+        public int pagesReceived() {
+            return pagesReceived;
+        }
+
+        public int pagesEmitted() {
+            return pagesEmitted;
+        }
+
+        public long processNanos() {
+            return processNanos;
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.startObject();
+            innerToXContent(builder);
+            return builder.endObject();
+        }
+
+        /**
+         * Render the body of the object for this status. Protected so subclasses
+         * can call it to render the "default" body.
+         */
+        protected final XContentBuilder innerToXContent(XContentBuilder builder) throws IOException {
+            builder.field("process_nanos", processNanos);
+            if (builder.humanReadable()) {
+                builder.field("process_time", TimeValue.timeValueNanos(processNanos));
+            }
+            builder.field("pages_received", pagesReceived);
+            return builder.field("pages_emitted", pagesEmitted);
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            AbstractPageMappingToIteratorOperator.Status status = (AbstractPageMappingToIteratorOperator.Status) o;
+            return processNanos == status.processNanos && pagesReceived == status.pagesReceived && pagesEmitted == status.pagesEmitted;
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(processNanos, pagesReceived, pagesEmitted);
+        }
+
+        @Override
+        public String toString() {
+            return Strings.toString(this);
+        }
+
+        @Override
+        public TransportVersion getMinimalSupportedVersion() {
+            return TransportVersions.ESQL_PAGE_MAPPING_TO_ITERATOR;
+        }
+    }
+
+    private static class AppendBlocksIterator implements ReleasableIterator<Page> {
+        private final Page page;
+        private final ReleasableIterator<? extends Block> next;
+
+        private int positionOffset;
+
+        protected AppendBlocksIterator(Page page, ReleasableIterator<? extends Block> next) {
+            this.page = page;
+            this.next = next;
+        }
+
+        @Override
+        public final boolean hasNext() {
+            if (next.hasNext()) {
+                assert positionOffset < page.getPositionCount();
+                return true;
+            }
+            assert positionOffset == page.getPositionCount();
+            return false;
+        }
+
+        @Override
+        public final Page next() {
+            Block read = next.next();
+            int start = positionOffset;
+            positionOffset += read.getPositionCount();
+            if (start == 0 && read.getPositionCount() == page.getPositionCount()) {
+                for (int b = 0; b < page.getBlockCount(); b++) {
+                    page.getBlock(b).incRef();
+                }
+                return page.appendBlock(read);
+            }
+            Block[] newBlocks = new Block[page.getBlockCount() + 1];
+            newBlocks[page.getBlockCount()] = read;
+            try {
+                // TODO a way to filter with a range please.
+                int[] positions = IntStream.range(start, positionOffset).toArray();
+                for (int b = 0; b < page.getBlockCount(); b++) {
+                    newBlocks[b] = page.getBlock(b).filter(positions);
+                }
+                Page result = new Page(newBlocks);
+                Arrays.fill(newBlocks, null);
+                return result;
+            } finally {
+                Releasables.closeExpectNoException(newBlocks);
+            }
+        }
+
+        @Override
+        public void close() {
+            Releasables.closeExpectNoException(page::releaseBlocks, next);
+        }
+    }
+}
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/HashLookupOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/HashLookupOperator.java
new file mode 100644
index 0000000000000..2b77003f11a4f
--- /dev/null
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/HashLookupOperator.java
@@ -0,0 +1,138 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.compute.operator;
+
+import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.compute.aggregation.GroupingAggregatorFunction;
+import org.elasticsearch.compute.aggregation.blockhash.BlockHash;
+import org.elasticsearch.compute.data.Block;
+import org.elasticsearch.compute.data.BlockFactory;
+import org.elasticsearch.compute.data.IntBlock;
+import org.elasticsearch.compute.data.IntVector;
+import org.elasticsearch.compute.data.Page;
+import org.elasticsearch.core.ReleasableIterator;
+import org.elasticsearch.core.Releasables;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+public class HashLookupOperator extends AbstractPageMappingToIteratorOperator {
+    /**
+     * Factory for {@link HashLookupOperator}. It's received {@link Block}s
+     * are never closed, so we need to build them from a non-tracking factory.
+     */
+    public static class Factory implements Operator.OperatorFactory {
+        private final Block[] keys;
+        private final int[] blockMapping;
+
+        public Factory(Block[] keys, int[] blockMapping) {
+            this.keys = keys;
+            this.blockMapping = blockMapping;
+        }
+
+        @Override
+        public Operator get(DriverContext driverContext) {
+            return new HashLookupOperator(driverContext.blockFactory(), keys, blockMapping);
+        }
+
+        @Override
+        public String describe() {
+            StringBuilder b = new StringBuilder();
+            b.append("HashLookup[keys=[");
+            for (int k = 0; k < keys.length; k++) {
+                Block key = keys[k];
+                if (k != 0) {
+                    b.append(", ");
+                }
+                b.append("{type=").append(key.elementType());
+                b.append(", positions=").append(key.getPositionCount());
+                b.append(", size=").append(ByteSizeValue.ofBytes(key.ramBytesUsed())).append("}");
+            }
+            b.append("], mapping=").append(Arrays.toString(blockMapping)).append("]");
+            return b.toString();
+        }
+    }
+
+    private final BlockHash hash;
+    private final int[] blockMapping;
+
+    public HashLookupOperator(BlockFactory blockFactory, Block[] keys, int[] blockMapping) {
+        this.blockMapping = blockMapping;
+        List<BlockHash.GroupSpec> groups = new ArrayList<>(keys.length);
+        for (int k = 0; k < keys.length; k++) {
+            groups.add(new BlockHash.GroupSpec(k, keys[k].elementType()));
+        }
+        /*
+         * Force PackedValuesBlockHash because it assigned ordinals in order
+         * of arrival. We'll figure out how to adapt other block hashes to
+         * do that soon. Soon we must figure out how to map ordinals to rows.
+         * And, probably at the same time, handle multiple rows containing
+         * the same keys.
+         */
+        this.hash = BlockHash.buildPackedValuesBlockHash(
+            groups,
+            blockFactory,
+            (int) BlockFactory.DEFAULT_MAX_BLOCK_PRIMITIVE_ARRAY_SIZE.getBytes()
+        );
+        boolean success = false;
+        try {
+            final int[] lastOrd = new int[] { -1 };
+            hash.add(new Page(keys), new GroupingAggregatorFunction.AddInput() {
+                @Override
+                public void add(int positionOffset, IntBlock groupIds) {
+                    // TODO support multiple rows with the same keys
+                    for (int p = 0; p < groupIds.getPositionCount(); p++) {
+                        int first = groupIds.getFirstValueIndex(p);
+                        int end = groupIds.getValueCount(p) + first;
+                        for (int i = first; i < end; i++) {
+                            int ord = groupIds.getInt(i);
+                            if (ord != lastOrd[0] + 1) {
+                                throw new IllegalArgumentException("found a duplicate row");
+                            }
+                            lastOrd[0] = ord;
+                        }
+                    }
+                }
+
+                @Override
+                public void add(int positionOffset, IntVector groupIds) {
+                    for (int p = 0; p < groupIds.getPositionCount(); p++) {
+                        int ord = groupIds.getInt(p);
+                        if (ord != lastOrd[0] + 1) {
+                            throw new IllegalArgumentException("found a duplicate row");
+                        }
+                        lastOrd[0] = ord;
+                    }
+                }
+            });
+            success = true;
+        } finally {
+            if (success == false) {
+                close();
+            }
+        }
+    }
+
+    @Override
+    protected ReleasableIterator<Page> receive(Page page) {
+        Page mapped = page.projectBlocks(blockMapping);
+        page.releaseBlocks();
+        return appendBlocks(mapped, hash.lookup(mapped, BlockFactory.DEFAULT_MAX_BLOCK_PRIMITIVE_ARRAY_SIZE));
+    }
+
+    @Override
+    public String toString() {
+        return "HashLookup[hash=" + hash + ", mapping=" + Arrays.toString(blockMapping) + "]";
+    }
+
+    @Override
+    public void close() {
+        Releasables.close(super::close, hash);
+    }
+}
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/ProjectOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/ProjectOperator.java
index 9b4d9d8f11a31..18bbcde41eb6b 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/ProjectOperator.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/ProjectOperator.java
@@ -7,9 +7,7 @@
 
 package org.elasticsearch.compute.operator;
 
-import org.elasticsearch.compute.data.Block;
 import org.elasticsearch.compute.data.Page;
-import org.elasticsearch.core.Releasables;
 
 import java.util.Arrays;
 import java.util.List;
@@ -31,7 +29,6 @@ public String describe() {
     }
 
     private final int[] projection;
-    private final Block[] blocks;
 
     /**
      * Creates an operator that applies the given projection, encoded as an integer list where
@@ -42,7 +39,6 @@ public String describe() {
      */
     public ProjectOperator(List<Integer> projection) {
         this.projection = projection.stream().mapToInt(Integer::intValue).toArray();
-        this.blocks = new Block[projection.size()];
     }
 
     @Override
@@ -51,29 +47,9 @@ protected Page process(Page page) {
         if (blockCount == 0) {
             return page;
         }
-        Page output = null;
         try {
-            int b = 0;
-            for (int source : projection) {
-                if (source >= blockCount) {
-                    throw new IllegalArgumentException(
-                        "Cannot project block with index [" + source + "] from a page with size [" + blockCount + "]"
-                    );
-                }
-                var block = page.getBlock(source);
-                blocks[b++] = block;
-                block.incRef();
-            }
-            int positionCount = page.getPositionCount();
-            // Use positionCount explicitly to avoid re-computing - also, if the projection is empty, there may be
-            // no more blocks left to determine the positionCount from.
-            output = new Page(positionCount, blocks);
-            return output;
+            return page.projectBlocks(projection);
         } finally {
-            if (output == null) {
-                Releasables.close(blocks);
-            }
-            Arrays.fill(blocks, null);
             page.releaseBlocks();
         }
     }
diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/OperatorTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/OperatorTests.java
index ef17bea26a14b..805f26e9ef280 100644
--- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/OperatorTests.java
+++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/OperatorTests.java
@@ -37,6 +37,7 @@
 import org.elasticsearch.compute.aggregation.blockhash.BlockHash;
 import org.elasticsearch.compute.data.Block;
 import org.elasticsearch.compute.data.BlockFactory;
+import org.elasticsearch.compute.data.BlockTestUtils;
 import org.elasticsearch.compute.data.BytesRefBlock;
 import org.elasticsearch.compute.data.DocBlock;
 import org.elasticsearch.compute.data.DocVector;
@@ -54,6 +55,7 @@
 import org.elasticsearch.compute.operator.Driver;
 import org.elasticsearch.compute.operator.DriverContext;
 import org.elasticsearch.compute.operator.HashAggregationOperator;
+import org.elasticsearch.compute.operator.HashLookupOperator;
 import org.elasticsearch.compute.operator.LimitOperator;
 import org.elasticsearch.compute.operator.Operator;
 import org.elasticsearch.compute.operator.OperatorTestCase;
@@ -71,12 +73,14 @@
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.TreeMap;
 
 import static org.elasticsearch.compute.aggregation.AggregatorMode.FINAL;
 import static org.elasticsearch.compute.aggregation.AggregatorMode.INITIAL;
@@ -324,6 +328,72 @@ public ScoreMode scoreMode() {
         return docIds;
     }
 
+    public void testHashLookup() {
+        // TODO move this to an integration test once we've plugged in the lookup
+        DriverContext driverContext = driverContext();
+        Map<Long, Integer> primeOrds = new TreeMap<>();
+        Block primesBlock;
+        try (LongBlock.Builder primes = driverContext.blockFactory().newLongBlockBuilder(30)) {
+            boolean[] sieve = new boolean[100];
+            Arrays.fill(sieve, true);
+            sieve[0] = false;
+            sieve[1] = false;
+            int prime = 2;
+            while (prime < 100) {
+                if (false == sieve[prime]) {
+                    prime++;
+                    continue;
+                }
+                primes.appendLong(prime);
+                primeOrds.put((long) prime, primeOrds.size());
+                for (int m = prime + prime; m < sieve.length; m += prime) {
+                    sieve[m] = false;
+                }
+                prime++;
+            }
+            primesBlock = primes.build();
+        }
+        try {
+            List<Long> values = new ArrayList<>();
+            List<Object> expectedValues = new ArrayList<>();
+            List<Object> expectedPrimeOrds = new ArrayList<>();
+            for (int i = 0; i < 100; i++) {
+                long v = i % 10 == 0 ? randomFrom(primeOrds.keySet()) : randomLongBetween(0, 100);
+                values.add(v);
+                expectedValues.add(v);
+                expectedPrimeOrds.add(primeOrds.get(v));
+            }
+
+            var actualValues = new ArrayList<>();
+            var actualPrimeOrds = new ArrayList<>();
+            try (
+                var driver = new Driver(
+                    driverContext,
+                    new SequenceLongBlockSourceOperator(driverContext.blockFactory(), values, 100),
+                    List.of(new HashLookupOperator(driverContext.blockFactory(), new Block[] { primesBlock }, new int[] { 0 })),
+                    new PageConsumerOperator(page -> {
+                        try {
+                            BlockTestUtils.readInto(actualValues, page.getBlock(0));
+                            BlockTestUtils.readInto(actualPrimeOrds, page.getBlock(1));
+                        } finally {
+                            page.releaseBlocks();
+                        }
+                    }),
+                    () -> {}
+                )
+            ) {
+                OperatorTestCase.runDriver(driver);
+            }
+
+            assertThat(actualValues, equalTo(expectedValues));
+            assertThat(actualPrimeOrds, equalTo(expectedPrimeOrds));
+            assertDriverContext(driverContext);
+        } finally {
+            primesBlock.close();
+        }
+
+    }
+
     /**
      * Creates a {@link BigArrays} that tracks releases but doesn't throw circuit breaking exceptions.
      */
diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/AbstractPageMappingToIteratorOperatorStatusTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/AbstractPageMappingToIteratorOperatorStatusTests.java
new file mode 100644
index 0000000000000..41db82b9b4c8c
--- /dev/null
+++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/AbstractPageMappingToIteratorOperatorStatusTests.java
@@ -0,0 +1,60 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.compute.operator;
+
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
+import org.elasticsearch.test.ESTestCase;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class AbstractPageMappingToIteratorOperatorStatusTests extends AbstractWireSerializingTestCase<
+    AbstractPageMappingToIteratorOperator.Status> {
+    public static AbstractPageMappingToIteratorOperator.Status simple() {
+        return new AbstractPageMappingToIteratorOperator.Status(200012, 123, 204);
+    }
+
+    public static String simpleToJson() {
+        return """
+            {
+              "process_nanos" : 200012,
+              "process_time" : "200micros",
+              "pages_received" : 123,
+              "pages_emitted" : 204
+            }""";
+    }
+
+    public void testToXContent() {
+        assertThat(Strings.toString(simple(), true, true), equalTo(simpleToJson()));
+    }
+
+    @Override
+    protected Writeable.Reader<AbstractPageMappingToIteratorOperator.Status> instanceReader() {
+        return AbstractPageMappingToIteratorOperator.Status::new;
+    }
+
+    @Override
+    public AbstractPageMappingToIteratorOperator.Status createTestInstance() {
+        return new AbstractPageMappingToIteratorOperator.Status(randomNonNegativeLong(), randomNonNegativeInt(), randomNonNegativeInt());
+    }
+
+    @Override
+    protected AbstractPageMappingToIteratorOperator.Status mutateInstance(AbstractPageMappingToIteratorOperator.Status instance) {
+        long processNanos = instance.processNanos();
+        int pagesReceived = instance.pagesReceived();
+        int pagesEmitted = instance.pagesEmitted();
+        switch (between(0, 2)) {
+            case 0 -> processNanos = randomValueOtherThan(processNanos, ESTestCase::randomNonNegativeLong);
+            case 1 -> pagesReceived = randomValueOtherThan(pagesReceived, ESTestCase::randomNonNegativeInt);
+            case 2 -> pagesEmitted = randomValueOtherThan(pagesEmitted, ESTestCase::randomNonNegativeInt);
+            default -> throw new UnsupportedOperationException();
+        }
+        return new AbstractPageMappingToIteratorOperator.Status(processNanos, pagesReceived, pagesEmitted);
+    }
+}
diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/HashLookupOperatorTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/HashLookupOperatorTests.java
new file mode 100644
index 0000000000000..ec69297718237
--- /dev/null
+++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/HashLookupOperatorTests.java
@@ -0,0 +1,48 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.compute.operator;
+
+import org.elasticsearch.compute.data.Block;
+import org.elasticsearch.compute.data.BlockFactory;
+import org.elasticsearch.compute.data.Page;
+import org.elasticsearch.compute.data.TestBlockFactory;
+
+import java.util.List;
+import java.util.stream.LongStream;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class HashLookupOperatorTests extends OperatorTestCase {
+    @Override
+    protected SourceOperator simpleInput(BlockFactory blockFactory, int size) {
+        return new SequenceLongBlockSourceOperator(blockFactory, LongStream.range(0, size).map(l -> randomFrom(1, 7, 14, 20)));
+    }
+
+    @Override
+    protected void assertSimpleOutput(List<Page> input, List<Page> results) {
+        assertThat(results.stream().mapToInt(Page::getPositionCount).sum(), equalTo(input.stream().mapToInt(Page::getPositionCount).sum()));
+    }
+
+    @Override
+    protected Operator.OperatorFactory simple() {
+        return new HashLookupOperator.Factory(
+            new Block[] { TestBlockFactory.getNonBreakingInstance().newLongArrayVector(new long[] { 7, 14, 20 }, 3).asBlock() },
+            new int[] { 0 }
+        );
+    }
+
+    @Override
+    protected String expectedDescriptionOfSimple() {
+        return "HashLookup[keys=[{type=LONG, positions=3, size=96b}], mapping=[0]]";
+    }
+
+    @Override
+    protected String expectedToStringOfSimple() {
+        return "HashLookup[hash=PackedValuesBlockHash{groups=[0:LONG], entries=3, size=536b}, mapping=[0]]";
+    }
+}
diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/IteratorAppendPageTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/IteratorAppendPageTests.java
new file mode 100644
index 0000000000000..ca0ebc64f09a6
--- /dev/null
+++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/IteratorAppendPageTests.java
@@ -0,0 +1,116 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.compute.operator;
+
+import org.elasticsearch.compute.data.Block;
+import org.elasticsearch.compute.data.BlockFactory;
+import org.elasticsearch.compute.data.Page;
+import org.elasticsearch.compute.data.TestBlockFactory;
+import org.elasticsearch.core.ReleasableIterator;
+
+import java.util.List;
+import java.util.stream.LongStream;
+
+import static org.hamcrest.Matchers.equalTo;
+
+/**
+ * Tests {@link AbstractPageMappingToIteratorOperator} against a test
+ * subclass that appends {@code 1} and chunks the incoming {@link Page}
+ * at {@code 100} positions.
+ */
+public class IteratorAppendPageTests extends OperatorTestCase {
+    private static final int ADDED_VALUE = 1;
+    private static final int CHUNK = 100;
+
+    private static class IteratorAppendPage extends AbstractPageMappingToIteratorOperator {
+        private static class Factory implements Operator.OperatorFactory {
+            @Override
+            public Operator get(DriverContext driverContext) {
+                return new IteratorAppendPage(driverContext.blockFactory());
+            }
+
+            @Override
+            public String describe() {
+                return "IteratorAppendPage[]";
+            }
+        }
+
+        private final BlockFactory blockFactory;
+
+        private IteratorAppendPage(BlockFactory blockFactory) {
+            this.blockFactory = blockFactory;
+        }
+
+        @Override
+        protected ReleasableIterator<Page> receive(Page page) {
+            return appendBlocks(page, new ReleasableIterator<>() {
+                private int positionOffset;
+
+                @Override
+                public boolean hasNext() {
+                    return positionOffset < page.getPositionCount();
+                }
+
+                @Override
+                public Block next() {
+                    if (hasNext() == false) {
+                        throw new IllegalStateException();
+                    }
+                    int positions = Math.min(page.getPositionCount() - positionOffset, CHUNK);
+                    positionOffset += positions;
+                    return blockFactory.newConstantIntBlockWith(ADDED_VALUE, positions);
+                }
+
+                @Override
+                public void close() {
+                    // Nothing to do, appendBlocks iterator closes the page for us.
+                }
+            });
+        }
+
+        @Override
+        public String toString() {
+            return "IteratorAppendPage[]";
+        }
+    }
+
+    @Override
+    protected SourceOperator simpleInput(BlockFactory blockFactory, int size) {
+        return new SequenceLongBlockSourceOperator(blockFactory, LongStream.range(0, size).map(l -> randomLong()));
+    }
+
+    @Override
+    protected void assertSimpleOutput(List<Page> input, List<Page> results) {
+        int r = 0;
+        for (Page in : input) {
+            for (int offset = 0; offset < in.getPositionCount(); offset += CHUNK) {
+                Page resultPage = results.get(r++);
+                assertThat(resultPage.getPositionCount(), equalTo(Math.min(CHUNK, in.getPositionCount() - offset)));
+                assertThat(
+                    resultPage.getBlock(1),
+                    equalTo(TestBlockFactory.getNonBreakingInstance().newConstantIntBlockWith(ADDED_VALUE, resultPage.getPositionCount()))
+                );
+            }
+        }
+    }
+
+    @Override
+    protected Operator.OperatorFactory simple() {
+        return new IteratorAppendPage.Factory();
+    }
+
+    @Override
+    protected String expectedDescriptionOfSimple() {
+        return "IteratorAppendPage[]";
+    }
+
+    @Override
+    protected String expectedToStringOfSimple() {
+        return expectedDescriptionOfSimple();
+    }
+}
diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/IteratorRemovePageTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/IteratorRemovePageTests.java
new file mode 100644
index 0000000000000..34943de834f9c
--- /dev/null
+++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/IteratorRemovePageTests.java
@@ -0,0 +1,118 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.compute.operator;
+
+import org.elasticsearch.compute.data.BlockFactory;
+import org.elasticsearch.compute.data.Page;
+import org.elasticsearch.core.ReleasableIterator;
+
+import java.util.List;
+import java.util.stream.LongStream;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasSize;
+
+/**
+ * Tests {@link AbstractPageMappingToIteratorOperator} against a test
+ * subclass that removes every other page.
+ */
+public class IteratorRemovePageTests extends OperatorTestCase {
+    private static class IteratorRemovePage extends AbstractPageMappingToIteratorOperator {
+        private static class Factory implements OperatorFactory {
+            @Override
+            public Operator get(DriverContext driverContext) {
+                return new IteratorRemovePage();
+            }
+
+            @Override
+            public String describe() {
+                return "IteratorRemovePage[]";
+            }
+        }
+
+        private boolean keep = true;
+
+        @Override
+        protected ReleasableIterator<Page> receive(Page page) {
+            if (keep) {
+                keep = false;
+                return new ReleasableIterator<>() {
+                    Page p = page;
+
+                    @Override
+                    public boolean hasNext() {
+                        return p != null;
+                    }
+
+                    @Override
+                    public Page next() {
+                        Page ret = p;
+                        p = null;
+                        return ret;
+                    }
+
+                    @Override
+                    public void close() {
+                        if (p != null) {
+                            p.releaseBlocks();
+                        }
+                    }
+                };
+            }
+            keep = true;
+            page.releaseBlocks();
+            return new ReleasableIterator<>() {
+                @Override
+                public boolean hasNext() {
+                    return false;
+                }
+
+                @Override
+                public Page next() {
+                    throw new UnsupportedOperationException();
+                }
+
+                @Override
+                public void close() {}
+            };
+        }
+
+        @Override
+        public String toString() {
+            return "IteratorRemovePage[]";
+        }
+    }
+
+    @Override
+    protected SourceOperator simpleInput(BlockFactory blockFactory, int size) {
+        return new SequenceLongBlockSourceOperator(blockFactory, LongStream.range(0, size).map(l -> randomLong()));
+    }
+
+    @Override
+    protected void assertSimpleOutput(List<Page> input, List<Page> results) {
+        assertThat(results, hasSize((input.size() + 1) / 2));
+        for (int i = 0; i < input.size(); i += 2) {
+            assertThat(input.get(i), equalTo(results.get(i / 2)));
+        }
+    }
+
+    @Override
+    protected Operator.OperatorFactory simple() {
+        return new IteratorRemovePage.Factory();
+    }
+
+    @Override
+    protected String expectedDescriptionOfSimple() {
+        return "IteratorRemovePage[]";
+    }
+
+    @Override
+    protected String expectedToStringOfSimple() {
+        return expectedDescriptionOfSimple();
+    }
+}
diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/OperatorTestCase.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/OperatorTestCase.java
index f8b53a9bcd3c0..eebcbc091d3ea 100644
--- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/OperatorTestCase.java
+++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/OperatorTestCase.java
@@ -201,19 +201,13 @@ protected final void assertSimple(DriverContext context, int size) {
 
         // Clone the input so that the operator can close it, then, later, we can read it again to build the assertion.
         List<Page> origInput = BlockTestUtils.deepCopyOf(input, TestBlockFactory.getNonBreakingInstance());
-        BigArrays bigArrays = context.bigArrays().withCircuitBreaking();
 
         List<Page> results = drive(simple().get(context), input.iterator(), context);
         assertSimpleOutput(origInput, results);
-        assertThat(bigArrays.breakerService().getBreaker(CircuitBreaker.REQUEST).getUsed(), equalTo(0L));
+        assertThat(context.breaker().getUsed(), equalTo(0L));
 
-        List<Block> resultBlocks = new ArrayList<>();
         // Release all result blocks. After this, all input blocks should be released as well, otherwise we have a leak.
         for (Page p : results) {
-            for (int i = 0; i < p.getBlockCount(); i++) {
-                resultBlocks.add(p.getBlock(i));
-            }
-
             p.releaseBlocks();
         }
 
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java
index 1e2557c040b06..043d07777ac4d 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java
@@ -25,6 +25,7 @@
 import org.elasticsearch.compute.lucene.LuceneOperator;
 import org.elasticsearch.compute.lucene.ValuesSourceReaderOperator;
 import org.elasticsearch.compute.operator.AbstractPageMappingOperator;
+import org.elasticsearch.compute.operator.AbstractPageMappingToIteratorOperator;
 import org.elasticsearch.compute.operator.AggregationOperator;
 import org.elasticsearch.compute.operator.AsyncOperator;
 import org.elasticsearch.compute.operator.DriverStatus;
@@ -175,6 +176,7 @@ public List<NamedWriteableRegistry.Entry> getNamedWriteables() {
             List.of(
                 DriverStatus.ENTRY,
                 AbstractPageMappingOperator.Status.ENTRY,
+                AbstractPageMappingToIteratorOperator.Status.ENTRY,
                 AggregationOperator.Status.ENTRY,
                 ExchangeSinkOperator.Status.ENTRY,
                 ExchangeSourceOperator.Status.ENTRY,

From 1aec77ecfe09f1d1dba59f10315dacbcb96f87d5 Mon Sep 17 00:00:00 2001
From: Mary Gouseti <mary.gouseti@elastic.co>
Date: Fri, 26 Apr 2024 16:09:31 +0300
Subject: [PATCH 51/58] Effective retention: improve passing params and add
 getters (#107943)

In oder to be able to enrich the following responses with the
`effective_retention` we need to ensure the `XContent.Params`  are
passed correctly and we have all the getters to copy the response.
---
 .../get/GetComposableIndexTemplateAction.java    |  4 ++++
 .../post/SimulateIndexTemplateResponse.java      | 16 ++++++++++++++++
 .../ExplainDataStreamLifecycleAction.java        |  2 +-
 .../lifecycle/GetDataStreamLifecycleAction.java  |  6 +++++-
 4 files changed, 26 insertions(+), 2 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetComposableIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetComposableIndexTemplateAction.java
index 240fdd2ae8199..f2fcbeff73c37 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetComposableIndexTemplateAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetComposableIndexTemplateAction.java
@@ -157,6 +157,10 @@ public Map<String, ComposableIndexTemplate> indexTemplates() {
             return indexTemplates;
         }
 
+        public RolloverConfiguration getRolloverConfiguration() {
+            return rolloverConfiguration;
+        }
+
         public DataStreamGlobalRetention getGlobalRetention() {
             return globalRetention;
         }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateResponse.java
index 4ff38222ccc99..52d40626f97ed 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateResponse.java
@@ -69,6 +69,22 @@ public SimulateIndexTemplateResponse(
         this.globalRetention = globalRetention;
     }
 
+    public Template getResolvedTemplate() {
+        return resolvedTemplate;
+    }
+
+    public Map<String, List<String>> getOverlappingTemplates() {
+        return overlappingTemplates;
+    }
+
+    public RolloverConfiguration getRolloverConfiguration() {
+        return rolloverConfiguration;
+    }
+
+    public DataStreamGlobalRetention getGlobalRetention() {
+        return globalRetention;
+    }
+
     public SimulateIndexTemplateResponse(StreamInput in) throws IOException {
         super(in);
         resolvedTemplate = in.readOptionalWriteable(Template::new);
diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/ExplainDataStreamLifecycleAction.java b/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/ExplainDataStreamLifecycleAction.java
index ee4f7fbaa9c59..17d33ae9167fd 100644
--- a/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/ExplainDataStreamLifecycleAction.java
+++ b/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/ExplainDataStreamLifecycleAction.java
@@ -212,7 +212,7 @@ public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params outerP
                 return builder;
             }), Iterators.map(indices.iterator(), explainIndexDataLifecycle -> (builder, params) -> {
                 builder.field(explainIndexDataLifecycle.getIndex());
-                explainIndexDataLifecycle.toXContent(builder, params, rolloverConfiguration, globalRetention);
+                explainIndexDataLifecycle.toXContent(builder, outerParams, rolloverConfiguration, globalRetention);
                 return builder;
             }), Iterators.single((builder, params) -> {
                 builder.endObject();
diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/GetDataStreamLifecycleAction.java b/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/GetDataStreamLifecycleAction.java
index d0dd67b4b4db5..1c9dbb0575a1d 100644
--- a/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/GetDataStreamLifecycleAction.java
+++ b/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/GetDataStreamLifecycleAction.java
@@ -220,6 +220,10 @@ public RolloverConfiguration getRolloverConfiguration() {
             return rolloverConfiguration;
         }
 
+        public DataStreamGlobalRetention getGlobalRetention() {
+            return globalRetention;
+        }
+
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             out.writeCollection(dataStreamLifecycles);
@@ -240,7 +244,7 @@ public Iterator<ToXContent> toXContentChunked(ToXContent.Params outerParams) {
                     dataStreamLifecycles.iterator(),
                     dataStreamLifecycle -> (builder, params) -> dataStreamLifecycle.toXContent(
                         builder,
-                        params,
+                        outerParams,
                         rolloverConfiguration,
                         globalRetention
                     )

From 98ed236f2bd58c0c3460a77493b231fd37a05cb9 Mon Sep 17 00:00:00 2001
From: Jedr Blaszyk <jedrazb@gmail.com>
Date: Fri, 26 Apr 2024 15:16:29 +0200
Subject: [PATCH 52/58] [Connector API] Update docs filtering, configuration,
 delete, scheduling (#107842)

---
 .../apis/delete-connector-api.asciidoc        |  14 +-
 ...pdate-connector-configuration-api.asciidoc | 367 +++++++++++-------
 .../update-connector-filtering-api.asciidoc   | 229 ++++++-----
 .../update-connector-scheduling-api.asciidoc  |  30 +-
 4 files changed, 394 insertions(+), 246 deletions(-)

diff --git a/docs/reference/connector/apis/delete-connector-api.asciidoc b/docs/reference/connector/apis/delete-connector-api.asciidoc
index b338f1db2a256..2e7c7a3b60708 100644
--- a/docs/reference/connector/apis/delete-connector-api.asciidoc
+++ b/docs/reference/connector/apis/delete-connector-api.asciidoc
@@ -28,6 +28,9 @@ Note: this action doesn't delete any API key, ingest pipeline or data index asso
 `<connector_id>`::
 (Required, string)
 
+`delete_sync_jobs`::
+(Optional, boolean) A flag indicating if associated sync jobs should be also removed. Defaults to `false`.
+
 [[delete-connector-api-response-codes]]
 ==== {api-response-codes-title}
 
@@ -47,7 +50,12 @@ The following example deletes the connector with ID `my-connector`:
 --------------------------------------------------
 PUT _connector/my-connector
 {
-  "index_name": "search-google-drive",
+  "name": "My Connector",
+  "service_type": "google_drive"
+}
+
+PUT _connector/another-connector
+{
   "name": "My Connector",
   "service_type": "google_drive"
 }
@@ -57,7 +65,7 @@ PUT _connector/my-connector
 
 [source,console]
 ----
-DELETE _connector/my-connector
+DELETE _connector/another-connector?delete_sync_jobs=true
 ----
 
 [source,console-result]
@@ -66,3 +74,5 @@ DELETE _connector/my-connector
     "acknowledged": true
 }
 ----
+
+The following example deletes the connector with ID `another-connector` and its associated sync jobs.
diff --git a/docs/reference/connector/apis/update-connector-configuration-api.asciidoc b/docs/reference/connector/apis/update-connector-configuration-api.asciidoc
index fea22eb8043b8..256621afb8fc5 100644
--- a/docs/reference/connector/apis/update-connector-configuration-api.asciidoc
+++ b/docs/reference/connector/apis/update-connector-configuration-api.asciidoc
@@ -6,7 +6,7 @@
 
 preview::[]
 
-Updates a connector's `configuration`, allowing for complete schema modifications or individual value updates within a registered configuration schema.
+Updates a connector's `configuration`, allowing for config value updates within a registered configuration schema.
 
 
 [[update-connector-configuration-api-request]]
@@ -19,7 +19,8 @@ Updates a connector's `configuration`, allowing for complete schema modification
 
 * To sync data using self-managed connectors, you need to deploy the {enterprise-search-ref}/build-connector.html[Elastic connector service] on your own infrastructure. This service runs automatically on Elastic Cloud for native connectors.
 * The `connector_id` parameter should reference an existing connector.
-* The configuration fields definition must be compatible with the specific connector type being used.
+* To update configuration `values`, the connector `configuration` schema must be first registered by a running instance of Elastic connector service.
+* Make sure configuration fields are compatible with the configuration schema for the third-party data source. Refer to the individual {enterprise-search-ref}/connectors-references.html[connectors references] for details.
 
 [[update-connector-configuration-api-path-params]]
 ==== {api-path-parms-title}
@@ -35,57 +36,7 @@ Updates a connector's `configuration`, allowing for complete schema modification
 (Optional, object) Configuration values for the connector, represented as a mapping of configuration fields to their respective values within a registered schema.
 
 `configuration`::
-(Optional, object) The configuration for the connector. The configuration field is a map where each key represents a specific configuration field name, and the value is a `ConnectorConfiguration` object.
-
-Each `ConnectorConfiguration` object contains the following attributes:
-
-* `category` (Optional, string) The category of the configuration field. This helps in grouping related configurations together in the user interface.
-
-* `default_value` (Required, string | number | bool) The default value for the configuration. This value is used if the value field is empty, applicable only for non-required fields.
-
-* `depends_on` (Required, array of `ConfigurationDependency`) An array of dependencies on other configurations. A field will not be enabled unless these dependencies are met. Each dependency specifies a field key and the required value for the dependency to be considered fulfilled.
-
-* `display` (Required, string) The display type for the UI element that represents this configuration. This defines how the field should be rendered in the user interface. Supported types are: `text`, `textbox`, `textarea`, `numeric`, `toggle` and `dropdown`.
-
-* `label` (Required, string) The display label for the configuration field. This label is shown in the user interface, adjacent to the field.
-
-* `options` (Required, array of `ConfigurationSelectOption`) An array of options for list-type fields. These options are used for inputs in the user interface, each having a label for display and a value.
-
-* `order` (Required, number) The order in which this configuration appears in the user interface. This helps in organizing fields logically.
-
-* `placeholder` (Required, string) Placeholder text for the configuration field. This text is displayed inside the field before a value is entered.
-
-* `required` (Required, boolean) Indicates whether the configuration is mandatory. If true, a value must be provided for the field.
-
-* `sensitive` (Required, boolean) Indicates whether the configuration contains sensitive information. Sensitive fields may be obfuscated in the user interface.
-
-* `tooltip` (Optional, string) Tooltip text providing additional information about the configuration. This text appears when the user hovers over the info icon next to the configuration field.
-
-* `type` (Required, string) The type of the configuration field, such as `str`, `int`, `bool`, `list`. This defines the data type and format of the field's value.
-
-* `ui_restrictions` (Required, array of strings) A list of UI restrictions. These restrictions define where in the user interface this field should be available or restricted.
-
-* `validations` (Required, array of `ConfigurationValidation`) An array of rules for validating the field's value. Each validation specifies a type and a constraint that the field's value must meet.
-
-* `value` (Required, string | number | bool) The current value of the configuration. This is the actual value set for the field and is used by the connector during its operations.
-
-`ConfigurationDependency` represents a dependency that a configuration field has on another field's value. It contains the following attributes:
-
-* `field` (Required, string) The name of the field in the configuration that this dependency relates to.
-
-* `value` (Required, string | number | bool) The required value of the specified field for this dependency to be met.
-
-`ConfigurationSelectOption` defines an option within a selectable configuration field. It contains the following attributes:
-
-* `label` (Required, string) The display label for the option.
-
-* `value` (Required, string) The actual value associated with the option.
-
-`ConfigurationValidation` specifies validation rules for configuration fields. Each ConfigurationValidation instance enforces a specific type of validation based on its type and constraint. It contains the following attributes:
-
-* `constraint` (Required, string | number) The validation constraint. The nature of this constraint depends on the validation type. It could be a numeric value, a list, a regular expression pattern.
-
-* `type` (Required, ConfigurationValidationType) The type of validation to be performed. Possible values include: `less_than`, `greater_than`, `list_type`, `included_in`, `regex` and `unset`.
+(Optional, object) The configuration schema definition for the connector. The configuration field is a map where each key represents a specific configuration field name, and the value is a `ConnectorConfiguration` object. For connector management use `values` to pass config values. The `configuration` object is used by the Elastic connector service to register the connector configuration schema.
 
 
 [[update-connector-configuration-api-response-codes]]
@@ -103,7 +54,7 @@ No connector matching `connector_id` could be found.
 [[update-connector-configuration-api-example]]
 ==== {api-examples-title}
 
-The following example updates the `configuration` for the connector with ID `my-connector`:
+The following example configures a `sharepoint_online` connector. Find the supported configuration options in the {enterprise-search-ref}/connectors-sharepoint-online.html[Sharepoint Online connector documentation] or by inspecting the schema in the connector's `configuration` field using the <<get-connector-api>>.
 
 ////
 [source, console]
@@ -118,35 +69,227 @@ PUT _connector/my-spo-connector
 PUT _connector/my-spo-connector/_configuration
 {
     "configuration": {
+        "tenant_id": {
+          "default_value": null,
+          "depends_on": [],
+          "display": "textbox",
+          "label": "Tenant ID",
+          "options": [],
+          "order": 1,
+          "required": true,
+          "sensitive": false,
+          "tooltip": "",
+          "type": "str",
+          "ui_restrictions": [],
+          "validations": [],
+          "value": ""
+        },
+        "tenant_name": {
+          "default_value": null,
+          "depends_on": [],
+          "display": "textbox",
+          "label": "Tenant name",
+          "options": [],
+          "order": 2,
+          "required": true,
+          "sensitive": false,
+          "tooltip": "",
+          "type": "str",
+          "ui_restrictions": [],
+          "validations": [],
+          "value": ""
+        },
         "client_id": {
-            "default_value": null,
-            "depends_on": [],
-            "display": "text",
-            "label": "Client ID",
-            "options": [],
-            "order": 3,
-            "required": true,
-            "sensitive": false,
-            "tooltip": null,
-            "type": "str",
-            "ui_restrictions": [],
-            "validations": [],
-            "value": null
+          "default_value": null,
+          "depends_on": [],
+          "display": "textbox",
+          "label": "Client ID",
+          "options": [],
+          "order": 3,
+          "required": true,
+          "sensitive": false,
+          "tooltip": "",
+          "type": "str",
+          "ui_restrictions": [],
+          "validations": [],
+          "value": ""
         },
         "secret_value": {
-            "default_value": null,
-            "depends_on": [],
-            "display": "text",
-            "label": "Secret value",
-            "options": [],
-            "order": 4,
-            "required": true,
-            "sensitive": true,
-            "tooltip": null,
-            "type": "str",
-            "ui_restrictions": [],
-            "validations": [],
-            "value": null
+          "default_value": null,
+          "depends_on": [],
+          "display": "textbox",
+          "label": "Secret value",
+          "options": [],
+          "order": 4,
+          "required": true,
+          "sensitive": true,
+          "tooltip": "",
+          "type": "str",
+          "ui_restrictions": [],
+          "validations": [],
+          "value": ""
+        },
+        "site_collections": {
+          "default_value": null,
+          "depends_on": [],
+          "display": "textarea",
+          "label": "Comma-separated list of sites",
+          "options": [],
+          "order": 5,
+          "required": true,
+          "sensitive": false,
+          "tooltip": "A comma-separated list of sites to ingest data from. Use * to include all available sites.",
+          "type": "list",
+          "ui_restrictions": [],
+          "validations": [],
+          "value": ""
+        },
+        "use_text_extraction_service": {
+          "default_value": false,
+          "depends_on": [],
+          "display": "toggle",
+          "label": "Use text extraction service",
+          "options": [],
+          "order": 6,
+          "required": true,
+          "sensitive": false,
+          "tooltip": "Requires a separate deployment of the Elastic Data Extraction Service. Also requires that pipeline settings disable text extraction.",
+          "type": "bool",
+          "ui_restrictions": [
+            "advanced"
+          ],
+          "validations": [],
+          "value": false
+        },
+        "use_document_level_security": {
+          "default_value": false,
+          "depends_on": [],
+          "display": "toggle",
+          "label": "Enable document level security",
+          "options": [],
+          "order": 7,
+          "required": true,
+          "sensitive": false,
+          "tooltip": "Document level security ensures identities and permissions set in Sharepoint Online are maintained in Elasticsearch. This metadata is added to your Elasticsearch documents, so you can control user and group read-access. Access control syncs ensure this metadata is kept up to date.",
+          "type": "bool",
+          "ui_restrictions": [],
+          "validations": [],
+          "value": false
+        },
+        "fetch_drive_item_permissions": {
+          "default_value": true,
+          "depends_on": [
+            {
+              "field": "use_document_level_security",
+              "value": true
+            }
+          ],
+          "display": "toggle",
+          "label": "Fetch drive item permissions",
+          "options": [],
+          "order": 8,
+          "required": true,
+          "sensitive": false,
+          "tooltip": "Enable this option to fetch drive item specific permissions. This setting can increase sync time.",
+          "type": "bool",
+          "ui_restrictions": [],
+          "validations": [],
+          "value": true
+        },
+        "fetch_unique_page_permissions": {
+          "default_value": true,
+          "depends_on": [
+            {
+              "field": "use_document_level_security",
+              "value": true
+            }
+          ],
+          "display": "toggle",
+          "label": "Fetch unique page permissions",
+          "options": [],
+          "order": 9,
+          "required": true,
+          "sensitive": false,
+          "tooltip": "Enable this option to fetch unique page permissions. This setting can increase sync time. If this setting is disabled a page will inherit permissions from its parent site.",
+          "type": "bool",
+          "ui_restrictions": [],
+          "validations": [],
+          "value": true
+        },
+        "fetch_unique_list_permissions": {
+          "default_value": true,
+          "depends_on": [
+            {
+              "field": "use_document_level_security",
+              "value": true
+            }
+          ],
+          "display": "toggle",
+          "label": "Fetch unique list permissions",
+          "options": [],
+          "order": 10,
+          "required": true,
+          "sensitive": false,
+          "tooltip": "Enable this option to fetch unique list permissions. This setting can increase sync time. If this setting is disabled a list will inherit permissions from its parent site.",
+          "type": "bool",
+          "ui_restrictions": [],
+          "validations": [],
+          "value": true
+        },
+        "fetch_unique_list_item_permissions": {
+          "default_value": true,
+          "depends_on": [
+            {
+              "field": "use_document_level_security",
+              "value": true
+            }
+          ],
+          "display": "toggle",
+          "label": "Fetch unique list item permissions",
+          "options": [],
+          "order": 11,
+          "required": true,
+          "sensitive": false,
+          "tooltip": "Enable this option to fetch unique list item permissions. This setting can increase sync time. If this setting is disabled a list item will inherit permissions from its parent site.",
+          "type": "bool",
+          "ui_restrictions": [],
+          "validations": [],
+          "value": true
+        },
+        "enumerate_all_sites": {
+          "default_value": true,
+          "depends_on": [],
+          "display": "toggle",
+          "label": "Enumerate all sites?",
+          "options": [],
+          "order": 6,
+          "required": false,
+          "sensitive": false,
+          "tooltip": "If enabled, sites will be fetched in bulk, then filtered down to the configured list of sites. This is efficient when syncing many sites. If disabled, each configured site will be fetched with an individual request. This is efficient when syncing fewer sites.",
+          "type": "bool",
+          "ui_restrictions": [],
+          "validations": [],
+          "value": true
+        },
+        "fetch_subsites": {
+          "default_value": false,
+          "depends_on": [
+            {
+              "field": "enumerate_all_sites",
+              "value": false
+            }
+          ],
+          "display": "toggle",
+          "label": "Fetch sub-sites of configured sites?",
+          "options": [],
+          "order": 7,
+          "required": false,
+          "sensitive": false,
+          "tooltip": "Whether subsites of the configured site(s) should be automatically fetched.",
+          "type": "bool",
+          "ui_restrictions": [],
+          "validations": [],
+          "value": true
         }
     }
 }
@@ -160,63 +303,16 @@ DELETE _connector/my-spo-connector
 // TEARDOWN
 ////
 
-This example demonstrates how to register a `sharepoint_online` connector configuration schema. Note: The example does not cover all the necessary configuration fields for operating the Sharepoint Online connector.
-
-[source,console]
-----
-PUT _connector/my-spo-connector/_configuration
-{
-    "configuration": {
-        "client_id": {
-            "default_value": null,
-            "depends_on": [],
-            "display": "text",
-            "label": "Client ID",
-            "options": [],
-            "order": 3,
-            "required": true,
-            "sensitive": false,
-            "tooltip": null,
-            "type": "str",
-            "ui_restrictions": [],
-            "validations": [],
-            "value": null
-        },
-        "secret_value": {
-            "default_value": null,
-            "depends_on": [],
-            "display": "text",
-            "label": "Secret value",
-            "options": [],
-            "order": 4,
-            "required": true,
-            "sensitive": true,
-            "tooltip": null,
-            "type": "str",
-            "ui_restrictions": [],
-            "validations": [],
-            "value": null
-        }
-    }
-}
-----
-
-[source,console-result]
-----
-{
-    "result": "updated"
-}
-----
-
-An example to update configuration values for the `sharepoint_online` connector:
-
 [source,console]
 ----
 PUT _connector/my-spo-connector/_configuration
 {
     "values": {
-        "client_id": "my-client-id",
-        "secret_value": "super-secret-value"
+        "tenant_id": "my-tenant-id",
+        "tenant_name": "my-sharepoint-site",
+        "client_id": "foo",
+        "secret_value": "bar",
+        "site_collections": "*"
     }
 }
 ----
@@ -229,14 +325,17 @@ PUT _connector/my-spo-connector/_configuration
 ----
 
 
-An example to update single configuration field of the `sharepoint_online` connector. In this case other configuration values won't change:
+When you're first setting up your connector you'll need to provide all required configuration details to start running syncs.
+But you can also use this API to only update a subset of fields.
+Here's an example that only updates the `secret_value` field for a `sharepoint_online` connector.
+The other configuration values won't change.
 
 [source,console]
 ----
 PUT _connector/my-spo-connector/_configuration
 {
     "values": {
-        "secret_value": "new-super-secret-value"
+        "secret_value": "foo-bar"
     }
 }
 ----
diff --git a/docs/reference/connector/apis/update-connector-filtering-api.asciidoc b/docs/reference/connector/apis/update-connector-filtering-api.asciidoc
index 0f6bd442e78cb..c028eece2e168 100644
--- a/docs/reference/connector/apis/update-connector-filtering-api.asciidoc
+++ b/docs/reference/connector/apis/update-connector-filtering-api.asciidoc
@@ -6,19 +6,22 @@
 
 preview::[]
 
+Updates the draft `filtering` configuration of a connector and marks the draft validation state as `edited`. The filtering configuration can be activated once validated by the Elastic connector service.
 
-Updates the `filtering` configuration of a connector. Learn more about filtering in the {enterprise-search-ref}/sync-rules.html[sync rules] documentation.
+The filtering property is used to configure sync rules (both basic and advanced) for a connector. Learn more in the {enterprise-search-ref}/sync-rules.html[sync rules documentation].
 
 [[update-connector-filtering-api-request]]
 ==== {api-request-title}
 
 `PUT _connector/<connector_id>/_filtering`
+`PUT _connector/<connector_id>/_filtering/_activate`
 
 [[update-connector-filtering-api-prereq]]
 ==== {api-prereq-title}
 
 * To sync data using self-managed connectors, you need to deploy the {enterprise-search-ref}/build-connector.html[Elastic connector service] on your own infrastructure. This service runs automatically on Elastic Cloud for native connectors.
 * The `connector_id` parameter should reference an existing connector.
+* To activate filtering rules, the `draft.validation.state` must be `valid`.
 
 [[update-connector-filtering-api-path-params]]
 ==== {api-path-parms-title}
@@ -30,65 +33,42 @@ Updates the `filtering` configuration of a connector. Learn more about filtering
 [[update-connector-filtering-api-request-body]]
 ==== {api-request-body-title}
 
-`filtering`::
-(Required, array) The filtering configuration for the connector. This configuration determines the set of rules applied for filtering data during syncs.
-
-Each entry in the `filtering` array represents a set of filtering rules for a specific data domain and includes the following attributes:
-
-- `domain` (Required, string) +
-Specifies the data domain to which these filtering rules apply.
-
-- `active` (Required, object) +
-Contains the set of rules that are actively used for sync jobs. The `active` object includes:
-
-  * `rules` (Required, array of objects) +
-  An array of individual filtering rule objects, each with the following sub-attributes:
-    ** `id` (Required, string) +
-    A unique identifier for the rule.
-    ** `policy` (Required, string) +
-    Specifies the policy, such as "include" or "exclude".
-    ** `field` (Required, string) +
-    The field in the document to which this rule applies.
-    ** `rule` (Required, string) +
-    The type of rule, such as "regex", "starts_with", "ends_with", "contains", "equals", "<", ">", etc.
-    ** `value` (Required, string) +
-    The value to be used in conjunction with the rule for matching the contents of the document's field.
-    ** `order` (Required, number) +
-    The order in which the rules are applied. The first rule to match has its policy applied.
-    ** `created_at` (Required, datetime) +
-    The timestamp when the rule was added.
-    ** `updated_at` (Required, datetime) +
-    The timestamp when the rule was last edited.
-
-  * `advanced_snippet` (Required, object) +
-  Used for {enterprise-search-ref}/sync-rules.html#sync-rules-advanced[advanced filtering] at query time, with the following sub-attributes:
-    ** `value` (Required, object) +
-    A JSON object passed directly to the connector for advanced filtering.
-    ** `created_at` (Required, datetime) +
-    The timestamp when this JSON object was created.
-    ** `updated_at` (Required, datetime) +
-    The timestamp when this JSON object was last edited.
-
-  * `validation` (Required, object) +
-  Provides validation status for the rules, including:
-    ** `state` (Required, string) +
-    Indicates the validation state: "edited", "valid", or "invalid".
-    ** `errors` (Required, object) +
-    Contains details about any validation errors, with sub-attributes:
-      *** `ids` (Required, string) +
-      The ID(s) of any rules deemed invalid.
-      *** `messages` (Required, string) +
-      Messages explaining what is invalid about the rules.
-
-- `draft` (Required, object) +
-An object identical in structure to the `active` object, but used for drafting and editing filtering rules before they become active.
+`rules`::
+(Optional, array of objects)
+An array of {enterprise-search-ref}/sync-rules.html#sync-rules-basic[basic sync rules], each with the following sub-attributes:
+* `id` (Required, string) +
+A unique identifier for the rule.
+* `policy` (Required, string) +
+Specifies the policy, such as `include` or `exclude`.
+* `field` (Required, string) +
+The field in the document to which this rule applies.
+* `rule` (Required, string) +
+The type of rule, such as `regex`, `starts_with`, `ends_with`, `contains`, `equals`, `<`, `>`, etc.
+* `value` (Required, string) +
+The value to be used in conjunction with the rule for matching the contents of the document's field.
+* `order` (Required, number) +
+The order in which the rules are applied. The first rule to match has its policy applied.
+* `created_at` (Optional, datetime) +
+The timestamp when the rule was added. Defaults to `now` UTC timestamp.
+* `updated_at` (Optional, datetime) +
+The timestamp when the rule was last edited. Defaults to `now` UTC timestamp.
+
+`advanced_snippet`::
+(Optional, object)
+Used for {enterprise-search-ref}/sync-rules.html#sync-rules-advanced[advanced filtering] at query time, with the following sub-attributes:
+* `value` (Required, object or array) +
+A JSON object/array passed directly to the connector for advanced filtering.
+* `created_at` (Optional, datetime) +
+The timestamp when this JSON object was created. Defaults to `now` UTC timestamp.
+* `updated_at` (Optional, datetime) +
+The timestamp when this JSON object was last edited. Defaults to `now` UTC timestamp.
 
 
 [[update-connector-filtering-api-response-codes]]
 ==== {api-response-codes-title}
 
 `200`::
-Connector `filtering` field was successfully updated.
+Connector draft filtering was successfully updated.
 
 `400`::
 The `connector_id` was not provided or the request payload was malformed.
@@ -99,80 +79,56 @@ No connector matching `connector_id` could be found.
 [[update-connector-filtering-api-example]]
 ==== {api-examples-title}
 
-The following example updates the `filtering` property for the connector with ID `my-connector`:
+The following example updates the draft {enterprise-search-ref}/sync-rules.html#sync-rules-basic[basic sync rules] for a Google Drive connector with ID `my-g-drive-connector`. All Google Drive files with `.txt` extension will be skipped:
 
 ////
 [source, console]
 --------------------------------------------------
-PUT _connector/my-connector
+PUT _connector/my-g-drive-connector
 {
   "index_name": "search-google-drive",
   "name": "My Connector",
   "service_type": "google_drive"
 }
+
+PUT _connector/my-sql-connector
+{
+  "index_name": "search-sql",
+  "name": "My SQL Connector",
+  "service_type": "google_drive"
+}
+
 --------------------------------------------------
 // TESTSETUP
 
 [source,console]
 --------------------------------------------------
-DELETE _connector/my-connector
+DELETE _connector/my-g-drive-connector
+DELETE _connector/my-sql-connector
 --------------------------------------------------
 // TEARDOWN
 ////
 
 [source,console]
 ----
-PUT _connector/my-connector/_filtering
+PUT _connector/my-g-drive-connector/_filtering
 {
-    "filtering": [
+    "rules": [
+         {
+            "field": "file_extension",
+            "id": "exclude-txt-files",
+            "order": 0,
+            "policy": "exclude",
+            "rule": "equals",
+            "value": "txt"
+        },
         {
-            "active": {
-                "advanced_snippet": {
-                    "created_at": "2023-11-09T15:13:08.231Z",
-                    "updated_at": "2023-11-09T15:13:08.231Z",
-                    "value": {}
-                },
-                "rules": [
-                    {
-                        "created_at": "2023-11-09T15:13:08.231Z",
-                        "field": "_",
-                        "id": "DEFAULT",
-                        "order": 0,
-                        "policy": "include",
-                        "rule": "regex",
-                        "updated_at": "2023-11-09T15:13:08.231Z",
-                        "value": ".*"
-                    }
-                ],
-                "validation": {
-                    "errors": [],
-                    "state": "valid"
-                }
-            },
-            "domain": "DEFAULT",
-            "draft": {
-                "advanced_snippet": {
-                    "created_at": "2023-11-09T15:13:08.231Z",
-                    "updated_at": "2023-11-09T15:13:08.231Z",
-                    "value": {}
-                },
-                "rules": [
-                    {
-                        "created_at": "2023-11-09T15:13:08.231Z",
-                        "field": "_",
-                        "id": "DEFAULT",
-                        "order": 0,
-                        "policy": "include",
-                        "rule": "regex",
-                        "updated_at": "2023-11-09T15:13:08.231Z",
-                        "value": ".*"
-                    }
-                ],
-                "validation": {
-                    "errors": [],
-                    "state": "valid"
-                }
-            }
+            "field": "_",
+            "id": "DEFAULT",
+            "order": 1,
+            "policy": "include",
+            "rule": "regex",
+            "value": ".*"
         }
     ]
 }
@@ -184,3 +140,64 @@ PUT _connector/my-connector/_filtering
     "result": "updated"
 }
 ----
+
+The following example updates the draft advanced sync rules for a MySQL connector with id `my-sql-connector`. Advanced sync rules are specific to each connector type. Refer to the references for connectors that support {enterprise-search-ref}/sync-rules.html#sync-rules-advanced[advanced sync rules] for syntax and examples.
+
+[source,console]
+----
+PUT _connector/my-sql-connector/_filtering
+{
+    "advanced_snippet": {
+        "value": [{
+            "tables": [
+                "users",
+                "orders"
+            ],
+            "query": "SELECT users.id AS id, orders.order_id AS order_id FROM users JOIN orders ON users.id = orders.user_id"
+        }]
+    }
+}
+----
+
+[source,console-result]
+----
+{
+    "result": "updated"
+}
+----
+
+
+////
+[source, console]
+--------------------------------------------------
+PUT _connector/my-sql-connector/_filtering/_validation
+{
+  "validation": {
+    "state": "valid",
+    "errors": []
+  }
+}
+--------------------------------------------------
+// TEST[continued]
+////
+
+
+Note, you can also update draft `rules` and `advanced_snippet` in a single request.
+
+Once the draft is updated, its validation state is set to `edited`. The connector service will then validate the rules and report the validation state as either `invalid` or `valid`. If the state is `valid`, the draft filtering can be activated with:
+
+
+[source,console]
+----
+PUT _connector/my-sql-connector/_filtering/_activate
+----
+// TEST[continued]
+
+[source,console-result]
+----
+{
+    "result": "updated"
+}
+----
+
+Once filtering rules are activated, they will be applied to all subsequent full or incremental syncs.
diff --git a/docs/reference/connector/apis/update-connector-scheduling-api.asciidoc b/docs/reference/connector/apis/update-connector-scheduling-api.asciidoc
index 1b9a2854649e4..df7a18ec6ad66 100644
--- a/docs/reference/connector/apis/update-connector-scheduling-api.asciidoc
+++ b/docs/reference/connector/apis/update-connector-scheduling-api.asciidoc
@@ -32,13 +32,13 @@ Updates the `scheduling` configuration of a connector.
 `scheduling`::
 (Required, object) The scheduling configuration for the connector. This configuration determines frequency of synchronization operations for the connector.
 
-The scheduling configuration includes the following attributes, each represented as a `ScheduleConfig` object:
+The scheduling configuration includes the following attributes, each represented as a `ScheduleConfig` object. If the `scheduling` object does not include all schedule types, only those provided will be updated; the others will remain unchanged.
 
-- `access_control` (Required, `ScheduleConfig` object) Defines the schedule for synchronizing access control settings of the connector.
+- `access_control` (Optional, `ScheduleConfig` object) Defines the schedule for synchronizing access control settings of the connector.
 
-- `full` (Required, `ScheduleConfig` object) Defines the schedule for a full content syncs.
+- `full` (Optional, `ScheduleConfig` object) Defines the schedule for a full content syncs.
 
-- `incremental` (Required, `ScheduleConfig` object) Defines the schedule for incremental content syncs.
+- `incremental` (Optional, `ScheduleConfig` object) Defines the schedule for incremental content syncs.
 
 Each `ScheduleConfig` object includes the following sub-attributes:
 
@@ -110,3 +110,25 @@ PUT _connector/my-connector/_scheduling
     "result": "updated"
 }
 ----
+
+The following example updates `full` sync schedule only, other schedule types remain unchanged:
+
+[source,console]
+----
+PUT _connector/my-connector/_scheduling
+{
+    "scheduling": {
+        "full": {
+            "enabled": true,
+            "interval": "0 10 0 * * ?"
+        }
+    }
+}
+----
+
+[source,console-result]
+----
+{
+    "result": "updated"
+}
+----

From 0c41cb7e71ac31fde65bcad8971e67467f0a279a Mon Sep 17 00:00:00 2001
From: Kostas Krikellas <131142368+kkrik-es@users.noreply.github.com>
Date: Fri, 26 Apr 2024 16:24:20 +0300
Subject: [PATCH 53/58] [TEST] simplify synthetic source yaml test (#107949)

---
 .../indices.create/20_synthetic_source.yml    | 22 +++----------------
 1 file changed, 3 insertions(+), 19 deletions(-)

diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml
index 39787366c0cc9..874778f9bdb5c 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml
@@ -68,25 +68,18 @@ object with unmapped fields:
         body:
           - '{ "create": { } }'
           - '{ "name": "aaaa", "some_string": "AaAa", "some_int": 1000, "some_double": 123.456789, "some_bool": true, "a.very.deeply.nested.field": "AAAA" }'
-          - '{ "create": { } }'
-          - '{ "name": "bbbb", "some_string": "BbBb", "some_int": 2000, "some_double": 321.987654, "some_bool": false, "a.very.deeply.nested.field": "BBBB" }'
 
   - do:
       search:
         index: test
 
-  - match: { hits.total.value: 2 }
+  - match: { hits.total.value: 1 }
   - match: { hits.hits.0._source.name: aaaa }
   - match: { hits.hits.0._source.some_string: AaAa }
   - match: { hits.hits.0._source.some_int: 1000 }
   - match: { hits.hits.0._source.some_double: 123.456789 }
   - match: { hits.hits.0._source.a.very.deeply.nested.field: AAAA }
   - match: { hits.hits.0._source.some_bool: true }
-  - match: { hits.hits.1._source.name: bbbb }
-  - match: { hits.hits.1._source.some_string: BbBb }
-  - match: { hits.hits.1._source.some_int: 2000 }
-  - match: { hits.hits.1._source.some_double: 321.987654 }
-  - match: { hits.hits.1._source.a.very.deeply.nested.field: BBBB }
 
 
 ---
@@ -124,20 +117,15 @@ nested object with unmapped fields:
         body:
           - '{ "create": { } }'
           - '{ "path.to.name": "aaaa", "path.to.surname": "AaAa", "path.some.other.name": "AaAaAa"  }'
-          - '{ "create": { } }'
-          - '{ "path.to.name": "bbbb", "path.to.surname": "BbBb", "path.some.other.name": "BbBbBb"  }'
 
   - do:
       search:
         index: test
 
-  - match: { hits.total.value: 2 }
+  - match: { hits.total.value: 1 }
   - match: { hits.hits.0._source.path.to.name: aaaa }
   - match: { hits.hits.0._source.path.to.surname: AaAa }
   - match: { hits.hits.0._source.path.some.other.name: AaAaAa }
-  - match: { hits.hits.1._source.path.to.name: bbbb }
-  - match: { hits.hits.1._source.path.to.surname: BbBb }
-  - match: { hits.hits.1._source.path.some.other.name: BbBbBb }
 
 
 ---
@@ -175,15 +163,11 @@ empty object with unmapped fields:
         body:
           - '{ "create": { } }'
           - '{ "path.to.surname": "AaAa", "path.some.other.name": "AaAaAa"  }'
-          - '{ "create": { } }'
-          - '{ "path.to.surname": "BbBb", "path.some.other.name": "BbBbBb"  }'
 
   - do:
       search:
         index: test
 
-  - match: { hits.total.value: 2 }
+  - match: { hits.total.value: 1 }
   - match: { hits.hits.0._source.path.to.surname: AaAa }
   - match: { hits.hits.0._source.path.some.other.name: AaAaAa }
-  - match: { hits.hits.1._source.path.to.surname: BbBb }
-  - match: { hits.hits.1._source.path.some.other.name: BbBbBb }

From 3ed42f38c3363b5108d0ed07c49af186d438694e Mon Sep 17 00:00:00 2001
From: Parker Timmins <parker.timmins@elastic.co>
Date: Fri, 26 Apr 2024 08:24:13 -0600
Subject: [PATCH 54/58] Add data-stream auto-sharding APM metrics (#107593)

Add APM metrics to monitor data stream auto-sharding events. The new metrics are:
- es.auto_sharding.increase_shards.total
- es.auto_sharding.decrease_shards.total
- es.auto_sharding.cooldown_prevented_increase.total
- es.auto_sharding.cooldown_prevented_decrease.total

The first two track situations where the shards increase or decrease during a rollover. The latter two events track when the auto-sharding logic recommends an increase or decrease but the shard change did not take place because we are in a cooldown period due to a recent increase or decrease auto-sharding event.
---
 docs/changelog/107593.yaml                    |   5 +
 .../datastreams/DataStreamAutoshardingIT.java |  62 +++++++-
 .../DataStreamGetWriteIndexTests.java         |   5 +-
 ...etadataDataStreamRolloverServiceTests.java |  25 +++-
 .../rollover/MetadataRolloverService.java     |  33 ++++-
 .../elasticsearch/cluster/ClusterModule.java  |   5 +
 ...adataRolloverServiceAutoShardingTests.java | 140 +++++++++++++++++-
 .../MetadataRolloverServiceTests.java         |  26 +++-
 .../TransportRolloverActionTests.java         |   5 +-
 .../metadata/DataStreamTestHelper.java        |   7 +-
 10 files changed, 291 insertions(+), 22 deletions(-)
 create mode 100644 docs/changelog/107593.yaml

diff --git a/docs/changelog/107593.yaml b/docs/changelog/107593.yaml
new file mode 100644
index 0000000000000..2e3d2cbc80119
--- /dev/null
+++ b/docs/changelog/107593.yaml
@@ -0,0 +1,5 @@
+pr: 107593
+summary: Add auto-sharding APM metrics
+area: Infra/Metrics
+type: enhancement
+issues: []
diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamAutoshardingIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamAutoshardingIT.java
index f7743ebac9caf..a4c9a9d3e1c67 100644
--- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamAutoshardingIT.java
+++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamAutoshardingIT.java
@@ -11,6 +11,7 @@
 import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
 import org.elasticsearch.action.admin.indices.rollover.Condition;
 import org.elasticsearch.action.admin.indices.rollover.MaxDocsCondition;
+import org.elasticsearch.action.admin.indices.rollover.MetadataRolloverService;
 import org.elasticsearch.action.admin.indices.rollover.OptimalShardCountCondition;
 import org.elasticsearch.action.admin.indices.rollover.RolloverConditions;
 import org.elasticsearch.action.admin.indices.rollover.RolloverInfo;
@@ -25,6 +26,7 @@
 import org.elasticsearch.action.bulk.BulkRequest;
 import org.elasticsearch.action.bulk.BulkResponse;
 import org.elasticsearch.action.datastreams.CreateDataStreamAction;
+import org.elasticsearch.action.datastreams.autosharding.AutoShardingType;
 import org.elasticsearch.action.datastreams.autosharding.DataStreamAutoShardingService;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.cluster.ClusterState;
@@ -49,7 +51,11 @@
 import org.elasticsearch.index.shard.ShardPath;
 import org.elasticsearch.index.store.StoreStats;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.plugins.PluginsService;
 import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.telemetry.InstrumentType;
+import org.elasticsearch.telemetry.Measurement;
+import org.elasticsearch.telemetry.TestTelemetryPlugin;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.transport.MockTransportService;
 import org.elasticsearch.xcontent.XContentType;
@@ -60,6 +66,7 @@
 import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
@@ -67,7 +74,9 @@
 import static org.elasticsearch.action.datastreams.autosharding.DataStreamAutoShardingService.DATA_STREAMS_AUTO_SHARDING_ENABLED;
 import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.DEFAULT_TIMESTAMP_FIELD;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.nullValue;
@@ -77,7 +86,12 @@ public class DataStreamAutoshardingIT extends ESIntegTestCase {
 
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return List.of(DataStreamsPlugin.class, MockTransportService.TestPlugin.class, TestAutoshardingPlugin.class);
+        return List.of(
+            DataStreamsPlugin.class,
+            MockTransportService.TestPlugin.class,
+            TestAutoshardingPlugin.class,
+            TestTelemetryPlugin.class
+        );
     }
 
     @Before
@@ -109,6 +123,7 @@ public void testRolloverOnAutoShardCondition() throws Exception {
         indexDocs(dataStreamName, randomIntBetween(100, 200));
 
         {
+            resetTelemetry();
             ClusterState clusterStateBeforeRollover = internalCluster().getCurrentMasterNodeInstance(ClusterService.class).state();
             DataStream dataStreamBeforeRollover = clusterStateBeforeRollover.getMetadata().dataStreams().get(dataStreamName);
             String assignedShardNodeId = clusterStateBeforeRollover.routingTable()
@@ -152,11 +167,14 @@ public void testRolloverOnAutoShardCondition() throws Exception {
             assertThat(metConditions.get(0).value(), instanceOf(Integer.class));
             int autoShardingRolloverInfo = (int) metConditions.get(0).value();
             assertThat(autoShardingRolloverInfo, is(5));
+
+            assertTelemetry(MetadataRolloverService.AUTO_SHARDING_METRIC_NAMES.get(AutoShardingType.INCREASE_SHARDS));
         }
 
         // let's do another rollover now that will not increase the number of shards because the increase shards cooldown has not lapsed,
         // however the rollover will use the existing/previous auto shard configuration and the new generation index will have 5 shards
         {
+            resetTelemetry();
             ClusterState clusterStateBeforeRollover = internalCluster().getCurrentMasterNodeInstance(ClusterService.class).state();
             DataStream dataStreamBeforeRollover = clusterStateBeforeRollover.getMetadata().dataStreams().get(dataStreamName);
             String assignedShardNodeId = clusterStateBeforeRollover.routingTable()
@@ -193,6 +211,8 @@ public void testRolloverOnAutoShardCondition() throws Exception {
 
             // we remained on 5 shards due to the increase shards cooldown
             assertThat(thirdGenerationMeta.getNumberOfShards(), is(5));
+
+            assertTelemetry(MetadataRolloverService.AUTO_SHARDING_METRIC_NAMES.get(AutoShardingType.COOLDOWN_PREVENTED_INCREASE));
         }
 
         {
@@ -566,4 +586,44 @@ private static void mockStatsForIndex(
             }
         }
     }
+
+    private static void resetTelemetry() {
+        for (PluginsService pluginsService : internalCluster().getInstances(PluginsService.class)) {
+            final TestTelemetryPlugin telemetryPlugin = pluginsService.filterPlugins(TestTelemetryPlugin.class).findFirst().orElseThrow();
+            telemetryPlugin.resetMeter();
+        }
+    }
+
+    private static void assertTelemetry(String expectedEmittedMetric) {
+        Map<String, List<Measurement>> measurements = new HashMap<>();
+        for (PluginsService pluginsService : internalCluster().getInstances(PluginsService.class)) {
+            final TestTelemetryPlugin telemetryPlugin = pluginsService.filterPlugins(TestTelemetryPlugin.class).findFirst().orElseThrow();
+
+            telemetryPlugin.collect();
+
+            List<String> autoShardingMetrics = telemetryPlugin.getRegisteredMetrics(InstrumentType.LONG_COUNTER)
+                .stream()
+                .filter(metric -> metric.startsWith("es.auto_sharding."))
+                .sorted()
+                .toList();
+
+            assertEquals(autoShardingMetrics, MetadataRolloverService.AUTO_SHARDING_METRIC_NAMES.values().stream().sorted().toList());
+
+            for (String metricName : MetadataRolloverService.AUTO_SHARDING_METRIC_NAMES.values()) {
+                measurements.computeIfAbsent(metricName, n -> new ArrayList<>())
+                    .addAll(telemetryPlugin.getLongCounterMeasurement(metricName));
+            }
+        }
+
+        // assert other metrics not emitted
+        MetadataRolloverService.AUTO_SHARDING_METRIC_NAMES.values()
+            .stream()
+            .filter(metric -> metric.equals(expectedEmittedMetric) == false)
+            .forEach(metric -> assertThat(measurements.get(metric), empty()));
+
+        assertThat(measurements.get(expectedEmittedMetric), hasSize(1));
+        Measurement measurement = measurements.get(expectedEmittedMetric).get(0);
+        assertThat(measurement.getLong(), is(1L));
+        assertFalse(measurement.isDouble());
+    }
 }
diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamGetWriteIndexTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamGetWriteIndexTests.java
index 111a46bb7098b..ccb8abbb9efab 100644
--- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamGetWriteIndexTests.java
+++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamGetWriteIndexTests.java
@@ -49,6 +49,7 @@
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.indices.ShardLimitValidator;
 import org.elasticsearch.script.ScriptCompiler;
+import org.elasticsearch.telemetry.TestTelemetryPlugin;
 import org.elasticsearch.test.ClusterServiceUtils;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.TestThreadPool;
@@ -272,13 +273,15 @@ public void setup() throws Exception {
                 indicesService,
                 xContentRegistry()
             );
+            TestTelemetryPlugin telemetryPlugin = new TestTelemetryPlugin();
             rolloverService = new MetadataRolloverService(
                 testThreadPool,
                 createIndexService,
                 indexAliasesService,
                 EmptySystemIndices.INSTANCE,
                 WriteLoadForecaster.DEFAULT,
-                clusterService
+                clusterService,
+                telemetryPlugin.getTelemetryProvider(Settings.EMPTY)
             );
         }
 
diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataDataStreamRolloverServiceTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataDataStreamRolloverServiceTests.java
index 2185f8f50a93f..86f6dea220e84 100644
--- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataDataStreamRolloverServiceTests.java
+++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataDataStreamRolloverServiceTests.java
@@ -31,6 +31,7 @@
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.index.MapperTestUtils;
+import org.elasticsearch.telemetry.TestTelemetryPlugin;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -88,6 +89,7 @@ public void testRolloverClusterStateForDataStream() throws Exception {
         );
         builder.put(dataStream);
         final ClusterState clusterState = ClusterState.builder(new ClusterName("test")).metadata(builder).build();
+        final TestTelemetryPlugin telemetryPlugin = new TestTelemetryPlugin();
 
         ThreadPool testThreadPool = new TestThreadPool(getTestName());
         try {
@@ -95,7 +97,8 @@ public void testRolloverClusterStateForDataStream() throws Exception {
                 dataStream,
                 testThreadPool,
                 Set.of(createSettingsProvider(xContentRegistry())),
-                xContentRegistry()
+                xContentRegistry(),
+                telemetryPlugin.getTelemetryProvider(Settings.EMPTY)
             );
             MaxDocsCondition condition = new MaxDocsCondition(randomNonNegativeLong());
             List<Condition<?>> metConditions = Collections.singletonList(condition);
@@ -184,6 +187,7 @@ public void testRolloverAndMigrateDataStream() throws Exception {
         );
         builder.put(dataStream);
         final ClusterState clusterState = ClusterState.builder(new ClusterName("test")).metadata(builder).build();
+        final TestTelemetryPlugin telemetryPlugin = new TestTelemetryPlugin();
 
         ThreadPool testThreadPool = new TestThreadPool(getTestName());
         try {
@@ -191,7 +195,8 @@ public void testRolloverAndMigrateDataStream() throws Exception {
                 dataStream,
                 testThreadPool,
                 Set.of(createSettingsProvider(xContentRegistry())),
-                xContentRegistry()
+                xContentRegistry(),
+                telemetryPlugin.getTelemetryProvider(Settings.EMPTY)
             );
             MaxDocsCondition condition = new MaxDocsCondition(randomNonNegativeLong());
             List<Condition<?>> metConditions = Collections.singletonList(condition);
@@ -271,14 +276,15 @@ public void testChangingIndexModeFromTimeSeriesToSomethingElseNoEffectOnExisting
         );
         builder.put(dataStream);
         final ClusterState clusterState = ClusterState.builder(new ClusterName("test")).metadata(builder).build();
-
+        final TestTelemetryPlugin telemetryPlugin = new TestTelemetryPlugin();
         ThreadPool testThreadPool = new TestThreadPool(getTestName());
         try {
             MetadataRolloverService rolloverService = DataStreamTestHelper.getMetadataRolloverService(
                 dataStream,
                 testThreadPool,
                 Set.of(createSettingsProvider(xContentRegistry())),
-                xContentRegistry()
+                xContentRegistry(),
+                telemetryPlugin.getTelemetryProvider(Settings.EMPTY)
             );
             MaxDocsCondition condition = new MaxDocsCondition(randomNonNegativeLong());
             List<Condition<?>> metConditions = Collections.singletonList(condition);
@@ -336,14 +342,16 @@ public void testRolloverClusterStateWithBrokenOlderTsdbDataStream() throws Excep
         int numberOfBackingIndices = randomIntBetween(1, 3);
         ClusterState clusterState = createClusterState(dataStreamName, numberOfBackingIndices, now, true);
         DataStream dataStream = clusterState.metadata().dataStreams().get(dataStreamName);
-
         ThreadPool testThreadPool = new TestThreadPool(getTestName());
+        final TestTelemetryPlugin telemetryPlugin = new TestTelemetryPlugin();
+
         try {
             MetadataRolloverService rolloverService = DataStreamTestHelper.getMetadataRolloverService(
                 dataStream,
                 testThreadPool,
                 Set.of(createSettingsProvider(xContentRegistry())),
-                xContentRegistry()
+                xContentRegistry(),
+                telemetryPlugin.getTelemetryProvider(Settings.EMPTY)
             );
             MaxDocsCondition condition = new MaxDocsCondition(randomNonNegativeLong());
             List<Condition<?>> metConditions = Collections.singletonList(condition);
@@ -417,14 +425,15 @@ public void testRolloverClusterStateWithBrokenTsdbDataStream() throws Exception
         int numberOfBackingIndices = randomIntBetween(1, 3);
         ClusterState clusterState = createClusterState(dataStreamName, numberOfBackingIndices, now, false);
         DataStream dataStream = clusterState.metadata().dataStreams().get(dataStreamName);
-
+        final TestTelemetryPlugin telemetryPlugin = new TestTelemetryPlugin();
         ThreadPool testThreadPool = new TestThreadPool(getTestName());
         try {
             MetadataRolloverService rolloverService = DataStreamTestHelper.getMetadataRolloverService(
                 dataStream,
                 testThreadPool,
                 Set.of(createSettingsProvider(xContentRegistry())),
-                xContentRegistry()
+                xContentRegistry(),
+                telemetryPlugin.getTelemetryProvider(Settings.EMPTY)
             );
             MaxDocsCondition condition = new MaxDocsCondition(randomNonNegativeLong());
             List<Condition<?>> metConditions = Collections.singletonList(condition);
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java
index 45368c185fb77..4284d860d85c0 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.action.admin.indices.create.CreateIndexClusterStateUpdateRequest;
 import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
 import org.elasticsearch.action.datastreams.autosharding.AutoShardingResult;
+import org.elasticsearch.action.datastreams.autosharding.AutoShardingType;
 import org.elasticsearch.action.support.ActiveShardCount;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.AliasAction;
@@ -46,6 +47,8 @@
 import org.elasticsearch.indices.SystemIndices;
 import org.elasticsearch.snapshots.SnapshotInProgressException;
 import org.elasticsearch.snapshots.SnapshotsService;
+import org.elasticsearch.telemetry.TelemetryProvider;
+import org.elasticsearch.telemetry.metric.MeterRegistry;
 import org.elasticsearch.threadpool.ThreadPool;
 
 import java.time.Instant;
@@ -70,8 +73,17 @@ public class MetadataRolloverService {
     private static final Logger logger = LogManager.getLogger(MetadataRolloverService.class);
     private static final Pattern INDEX_NAME_PATTERN = Pattern.compile("^.*-\\d+$");
     private static final List<IndexAbstraction.Type> VALID_ROLLOVER_TARGETS = List.of(ALIAS, DATA_STREAM);
-
     public static final Settings HIDDEN_INDEX_SETTINGS = Settings.builder().put(IndexMetadata.SETTING_INDEX_HIDDEN, true).build();
+    public static final Map<AutoShardingType, String> AUTO_SHARDING_METRIC_NAMES = Map.of(
+        AutoShardingType.INCREASE_SHARDS,
+        "es.auto_sharding.increase_shards.total",
+        AutoShardingType.DECREASE_SHARDS,
+        "es.auto_sharding.decrease_shards.total",
+        AutoShardingType.COOLDOWN_PREVENTED_INCREASE,
+        "es.auto_sharding.cooldown_prevented_increase.total",
+        AutoShardingType.COOLDOWN_PREVENTED_DECREASE,
+        "es.auto_sharding.cooldown_prevented_decrease.total"
+    );
 
     private final ThreadPool threadPool;
     private final MetadataCreateIndexService createIndexService;
@@ -79,6 +91,7 @@ public class MetadataRolloverService {
     private final SystemIndices systemIndices;
     private final WriteLoadForecaster writeLoadForecaster;
     private final ClusterService clusterService;
+    private final MeterRegistry meterRegistry;
 
     @Inject
     public MetadataRolloverService(
@@ -87,7 +100,8 @@ public MetadataRolloverService(
         MetadataIndexAliasesService indexAliasesService,
         SystemIndices systemIndices,
         WriteLoadForecaster writeLoadForecaster,
-        ClusterService clusterService
+        ClusterService clusterService,
+        TelemetryProvider telemetryProvider
     ) {
         this.threadPool = threadPool;
         this.createIndexService = createIndexService;
@@ -95,6 +109,14 @@ public MetadataRolloverService(
         this.systemIndices = systemIndices;
         this.writeLoadForecaster = writeLoadForecaster;
         this.clusterService = clusterService;
+        this.meterRegistry = telemetryProvider.getMeterRegistry();
+
+        for (var entry : AUTO_SHARDING_METRIC_NAMES.entrySet()) {
+            final AutoShardingType type = entry.getKey();
+            final String metricName = entry.getValue();
+            final String description = String.format(Locale.ROOT, "auto-sharding %s counter", type.name().toLowerCase(Locale.ROOT));
+            meterRegistry.registerLongCounter(metricName, description, "unit");
+        }
     }
 
     public record RolloverResult(String rolloverIndexName, String sourceIndexName, ClusterState clusterState) {
@@ -330,6 +352,13 @@ private RolloverResult rolloverDataStream(
                 (builder, indexMetadata) -> builder.put(dataStream.rolloverFailureStore(indexMetadata.getIndex(), newGeneration))
             );
         } else {
+            if (autoShardingResult != null) {
+                final String metricName = AUTO_SHARDING_METRIC_NAMES.get(autoShardingResult.type());
+                if (metricName != null) {
+                    meterRegistry.getLongCounter(metricName).increment();
+                }
+            }
+
             DataStreamAutoShardingEvent dataStreamAutoShardingEvent = autoShardingResult == null
                 ? dataStream.getAutoShardingEvent()
                 : switch (autoShardingResult.type()) {
diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java b/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java
index 809e069b0028b..60140e2a08714 100644
--- a/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java
+++ b/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java
@@ -8,6 +8,7 @@
 
 package org.elasticsearch.cluster;
 
+import org.elasticsearch.action.admin.indices.rollover.MetadataRolloverService;
 import org.elasticsearch.cluster.action.index.MappingUpdatedAction;
 import org.elasticsearch.cluster.action.shard.ShardStateAction;
 import org.elasticsearch.cluster.metadata.ComponentTemplateMetadata;
@@ -120,6 +121,7 @@ public class ClusterModule extends AbstractModule {
     final ShardsAllocator shardsAllocator;
     private final ShardRoutingRoleStrategy shardRoutingRoleStrategy;
     private final AllocationStatsService allocationStatsService;
+    private final TelemetryProvider telemetryProvider;
 
     public ClusterModule(
         Settings settings,
@@ -157,6 +159,7 @@ public ClusterModule(
         );
         this.metadataDeleteIndexService = new MetadataDeleteIndexService(settings, clusterService, allocationService);
         this.allocationStatsService = new AllocationStatsService(clusterService, clusterInfoService, shardsAllocator, writeLoadForecaster);
+        this.telemetryProvider = telemetryProvider;
     }
 
     static ShardRoutingRoleStrategy getShardRoutingRoleStrategy(List<ClusterPlugin> clusterPlugins) {
@@ -444,6 +447,8 @@ protected void configure() {
         bind(ShardsAllocator.class).toInstance(shardsAllocator);
         bind(ShardRoutingRoleStrategy.class).toInstance(shardRoutingRoleStrategy);
         bind(AllocationStatsService.class).toInstance(allocationStatsService);
+        bind(TelemetryProvider.class).toInstance(telemetryProvider);
+        bind(MetadataRolloverService.class).asEagerSingleton();
     }
 
     public void setExistingShardsAllocators(GatewayAllocator gatewayAllocator) {
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceAutoShardingTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceAutoShardingTests.java
index 906b2434f7d39..41176276a42c0 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceAutoShardingTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceAutoShardingTests.java
@@ -25,6 +25,8 @@
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexVersion;
+import org.elasticsearch.telemetry.Measurement;
+import org.elasticsearch.telemetry.TestTelemetryPlugin;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -41,6 +43,7 @@
 import static org.elasticsearch.action.datastreams.autosharding.AutoShardingType.NOT_APPLICABLE;
 import static org.elasticsearch.action.datastreams.autosharding.AutoShardingType.NO_CHANGE_REQUIRED;
 import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_INDEX_UUID;
+import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.hasItem;
@@ -82,17 +85,20 @@ public void testRolloverDataStreamWithoutExistingAutosharding() throws Exception
         builder.put(dataStream);
         final ClusterState clusterState = ClusterState.builder(new ClusterName("test")).metadata(builder).build();
 
+        final TestTelemetryPlugin telemetryPlugin = new TestTelemetryPlugin();
         ThreadPool testThreadPool = new TestThreadPool(getTestName());
         try {
             MetadataRolloverService rolloverService = DataStreamTestHelper.getMetadataRolloverService(
                 dataStream,
                 testThreadPool,
                 Set.of(),
-                xContentRegistry()
+                xContentRegistry(),
+                telemetryPlugin.getTelemetryProvider(Settings.EMPTY)
             );
 
             // let's rollover the data stream using all the possible autosharding recommendations
             for (AutoShardingType type : AutoShardingType.values()) {
+                telemetryPlugin.resetMeter();
                 long before = testThreadPool.absoluteTimeInMillis();
                 switch (type) {
                     case INCREASE_SHARDS -> {
@@ -111,6 +117,15 @@ public void testRolloverDataStreamWithoutExistingAutosharding() throws Exception
                             false
                         );
                         assertRolloverResult(dataStream, rolloverResult, before, testThreadPool.absoluteTimeInMillis(), metConditions, 5);
+                        assertTelemetry(
+                            telemetryPlugin,
+                            "es.auto_sharding.increase_shards.total",
+                            List.of(
+                                "es.auto_sharding.decrease_shards.total",
+                                "es.auto_sharding.cooldown_prevented_increase.total",
+                                "es.auto_sharding.cooldown_prevented_decrease.total"
+                            )
+                        );
                     }
                     case DECREASE_SHARDS -> {
                         {
@@ -138,6 +153,15 @@ public void testRolloverDataStreamWithoutExistingAutosharding() throws Exception
                                 metConditions,
                                 1
                             );
+                            assertTelemetry(
+                                telemetryPlugin,
+                                "es.auto_sharding.decrease_shards.total",
+                                List.of(
+                                    "es.auto_sharding.increase_shards.total",
+                                    "es.auto_sharding.cooldown_prevented_increase.total",
+                                    "es.auto_sharding.cooldown_prevented_decrease.total"
+                                )
+                            );
                         }
 
                         {
@@ -190,6 +214,15 @@ public void testRolloverDataStreamWithoutExistingAutosharding() throws Exception
                         );
                         // the expected number of shards remains 3 for the data stream due to the remaining cooldown
                         assertRolloverResult(dataStream, rolloverResult, before, testThreadPool.absoluteTimeInMillis(), List.of(), 3);
+                        assertTelemetry(
+                            telemetryPlugin,
+                            "es.auto_sharding.cooldown_prevented_increase.total",
+                            List.of(
+                                "es.auto_sharding.increase_shards.total",
+                                "es.auto_sharding.decrease_shards.total",
+                                "es.auto_sharding.cooldown_prevented_decrease.total"
+                            )
+                        );
                     }
                     case COOLDOWN_PREVENTED_DECREASE -> {
                         MetadataRolloverService.RolloverResult rolloverResult = rolloverService.rolloverClusterState(
@@ -207,6 +240,15 @@ public void testRolloverDataStreamWithoutExistingAutosharding() throws Exception
                         );
                         // the expected number of shards remains 3 for the data stream due to the remaining cooldown
                         assertRolloverResult(dataStream, rolloverResult, before, testThreadPool.absoluteTimeInMillis(), List.of(), 3);
+                        assertTelemetry(
+                            telemetryPlugin,
+                            "es.auto_sharding.cooldown_prevented_decrease.total",
+                            List.of(
+                                "es.auto_sharding.increase_shards.total",
+                                "es.auto_sharding.decrease_shards.total",
+                                "es.auto_sharding.cooldown_prevented_increase.total"
+                            )
+                        );
                     }
                     case NO_CHANGE_REQUIRED -> {
                         List<Condition<?>> metConditions = List.of(new MaxDocsCondition(randomNonNegativeLong()));
@@ -224,6 +266,16 @@ public void testRolloverDataStreamWithoutExistingAutosharding() throws Exception
                             false
                         );
                         assertRolloverResult(dataStream, rolloverResult, before, testThreadPool.absoluteTimeInMillis(), metConditions, 3);
+                        assertTelemetry(
+                            telemetryPlugin,
+                            null,
+                            List.of(
+                                "es.auto_sharding.increase_shards.total",
+                                "es.auto_sharding.decrease_shards.total",
+                                "es.auto_sharding.cooldown_prevented_increase.total",
+                                "es.auto_sharding.cooldown_prevented_decrease.total"
+                            )
+                        );
                     }
                     case NOT_APPLICABLE -> {
                         List<Condition<?>> metConditions = List.of(new MaxDocsCondition(randomNonNegativeLong()));
@@ -241,6 +293,16 @@ public void testRolloverDataStreamWithoutExistingAutosharding() throws Exception
                             false
                         );
                         assertRolloverResult(dataStream, rolloverResult, before, testThreadPool.absoluteTimeInMillis(), metConditions, 3);
+                        assertTelemetry(
+                            telemetryPlugin,
+                            null,
+                            List.of(
+                                "es.auto_sharding.increase_shards.total",
+                                "es.auto_sharding.decrease_shards.total",
+                                "es.auto_sharding.cooldown_prevented_increase.total",
+                                "es.auto_sharding.cooldown_prevented_decrease.total"
+                            )
+                        );
                     }
                 }
             }
@@ -285,17 +347,20 @@ public void testRolloverDataStreamWithExistingAutoShardEvent() throws Exception
         builder.put(dataStream);
         final ClusterState clusterState = ClusterState.builder(new ClusterName("test")).metadata(builder).build();
 
+        final TestTelemetryPlugin telemetryPlugin = new TestTelemetryPlugin();
         ThreadPool testThreadPool = new TestThreadPool(getTestName());
         try {
             MetadataRolloverService rolloverService = DataStreamTestHelper.getMetadataRolloverService(
                 dataStream,
                 testThreadPool,
                 Set.of(),
-                xContentRegistry()
+                xContentRegistry(),
+                telemetryPlugin.getTelemetryProvider(Settings.EMPTY)
             );
 
             // let's rollover the data stream using all the possible autosharding recommendations
             for (AutoShardingType type : AutoShardingType.values()) {
+                telemetryPlugin.resetMeter();
                 long before = testThreadPool.absoluteTimeInMillis();
                 switch (type) {
                     case INCREASE_SHARDS -> {
@@ -314,6 +379,15 @@ public void testRolloverDataStreamWithExistingAutoShardEvent() throws Exception
                             false
                         );
                         assertRolloverResult(dataStream, rolloverResult, before, testThreadPool.absoluteTimeInMillis(), metConditions, 5);
+                        assertTelemetry(
+                            telemetryPlugin,
+                            "es.auto_sharding.increase_shards.total",
+                            List.of(
+                                "es.auto_sharding.decrease_shards.total",
+                                "es.auto_sharding.cooldown_prevented_increase.total",
+                                "es.auto_sharding.cooldown_prevented_decrease.total"
+                            )
+                        );
                     }
                     case DECREASE_SHARDS -> {
                         {
@@ -341,6 +415,15 @@ public void testRolloverDataStreamWithExistingAutoShardEvent() throws Exception
                                 metConditions,
                                 1
                             );
+                            assertTelemetry(
+                                telemetryPlugin,
+                                "es.auto_sharding.decrease_shards.total",
+                                List.of(
+                                    "es.auto_sharding.increase_shards.total",
+                                    "es.auto_sharding.cooldown_prevented_increase.total",
+                                    "es.auto_sharding.cooldown_prevented_decrease.total"
+                                )
+                            );
                         }
 
                         {
@@ -386,6 +469,15 @@ public void testRolloverDataStreamWithExistingAutoShardEvent() throws Exception
                         );
                         // the expected number of shards remains 3 for the data stream due to the remaining cooldown
                         assertRolloverResult(dataStream, rolloverResult, before, testThreadPool.absoluteTimeInMillis(), List.of(), 3);
+                        assertTelemetry(
+                            telemetryPlugin,
+                            "es.auto_sharding.cooldown_prevented_increase.total",
+                            List.of(
+                                "es.auto_sharding.decrease_shards.total",
+                                "es.auto_sharding.increase_shards.total",
+                                "es.auto_sharding.cooldown_prevented_decrease.total"
+                            )
+                        );
                     }
                     case COOLDOWN_PREVENTED_DECREASE -> {
                         MetadataRolloverService.RolloverResult rolloverResult = rolloverService.rolloverClusterState(
@@ -403,6 +495,15 @@ public void testRolloverDataStreamWithExistingAutoShardEvent() throws Exception
                         );
                         // the expected number of shards remains 3 for the data stream due to the remaining cooldown
                         assertRolloverResult(dataStream, rolloverResult, before, testThreadPool.absoluteTimeInMillis(), List.of(), 3);
+                        assertTelemetry(
+                            telemetryPlugin,
+                            "es.auto_sharding.cooldown_prevented_decrease.total",
+                            List.of(
+                                "es.auto_sharding.increase_shards.total",
+                                "es.auto_sharding.decrease_shards.total",
+                                "es.auto_sharding.cooldown_prevented_increase.total"
+                            )
+                        );
                     }
                     case NO_CHANGE_REQUIRED -> {
                         List<Condition<?>> metConditions = List.of(new MaxDocsCondition(randomNonNegativeLong()));
@@ -420,6 +521,16 @@ public void testRolloverDataStreamWithExistingAutoShardEvent() throws Exception
                             false
                         );
                         assertRolloverResult(dataStream, rolloverResult, before, testThreadPool.absoluteTimeInMillis(), metConditions, 3);
+                        assertTelemetry(
+                            telemetryPlugin,
+                            null,
+                            List.of(
+                                "es.auto_sharding.decrease_shards.total",
+                                "es.auto_sharding.increase_shards.total",
+                                "es.auto_sharding.cooldown_prevented_increase.total",
+                                "es.auto_sharding.cooldown_prevented_decrease.total"
+                            )
+                        );
                     }
                     case NOT_APPLICABLE -> {
                         List<Condition<?>> metConditions = List.of(new MaxDocsCondition(randomNonNegativeLong()));
@@ -438,6 +549,16 @@ public void testRolloverDataStreamWithExistingAutoShardEvent() throws Exception
                         );
                         // if the auto sharding is not applicable we just use whatever's in the index template (1 shard in this case)
                         assertRolloverResult(dataStream, rolloverResult, before, testThreadPool.absoluteTimeInMillis(), metConditions, 1);
+                        assertTelemetry(
+                            telemetryPlugin,
+                            null,
+                            List.of(
+                                "es.auto_sharding.decrease_shards.total",
+                                "es.auto_sharding.increase_shards.total",
+                                "es.auto_sharding.cooldown_prevented_increase.total",
+                                "es.auto_sharding.cooldown_prevented_decrease.total"
+                            )
+                        );
                     }
                 }
             }
@@ -500,4 +621,19 @@ private static IndexMetadata.Builder getIndexMetadataBuilderForIndex(Index index
             .numberOfShards(numberOfShards)
             .numberOfReplicas(1);
     }
+
+    private static void assertTelemetry(TestTelemetryPlugin telemetryPlugin, String presentMetric, List<String> missingMetrics) {
+        if (presentMetric != null) {
+            final List<Measurement> measurements = telemetryPlugin.getLongCounterMeasurement(presentMetric);
+            assertThat(measurements, hasSize(1));
+            Measurement measurement = measurements.get(0);
+            assertThat(measurement.getLong(), is(1L));
+            assertFalse(measurement.isDouble());
+        }
+
+        for (String metric : missingMetrics) {
+            final List<Measurement> measurements = telemetryPlugin.getLongCounterMeasurement(metric);
+            assertThat(measurements, empty());
+        }
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java
index 0bf92df006894..149752578e1ea 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java
@@ -37,6 +37,7 @@
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.indices.EmptySystemIndices;
+import org.elasticsearch.telemetry.TestTelemetryPlugin;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -52,6 +53,7 @@
 import java.util.Set;
 
 import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.hasItem;
@@ -533,6 +535,7 @@ public void testRolloverClusterState() throws Exception {
         final ClusterState clusterState = ClusterState.builder(new ClusterName("test"))
             .metadata(Metadata.builder().put(indexMetadata))
             .build();
+        final TestTelemetryPlugin telemetryPlugin = new TestTelemetryPlugin();
 
         ThreadPool testThreadPool = new TestThreadPool(getTestName());
         try {
@@ -540,7 +543,8 @@ public void testRolloverClusterState() throws Exception {
                 null,
                 testThreadPool,
                 Set.of(),
-                xContentRegistry()
+                xContentRegistry(),
+                telemetryPlugin.getTelemetryProvider(Settings.EMPTY)
             );
 
             MaxDocsCondition condition = new MaxDocsCondition(randomNonNegativeLong());
@@ -586,6 +590,10 @@ public void testRolloverClusterState() throws Exception {
             assertThat(info.getTime(), greaterThanOrEqualTo(before));
             assertThat(info.getMetConditions(), hasSize(1));
             assertThat(info.getMetConditions().get(0).value(), equalTo(condition.value()));
+
+            for (String metric : MetadataRolloverService.AUTO_SHARDING_METRIC_NAMES.values()) {
+                assertThat(telemetryPlugin.getLongCounterMeasurement(metric), empty());
+            }
         } finally {
             testThreadPool.shutdown();
         }
@@ -606,6 +614,7 @@ public void testRolloverClusterStateForDataStream() throws Exception {
         }
         builder.put(dataStream);
         final ClusterState clusterState = ClusterState.builder(new ClusterName("test")).metadata(builder).build();
+        final TestTelemetryPlugin telemetryPlugin = new TestTelemetryPlugin();
 
         ThreadPool testThreadPool = new TestThreadPool(getTestName());
         try {
@@ -613,7 +622,8 @@ public void testRolloverClusterStateForDataStream() throws Exception {
                 dataStream,
                 testThreadPool,
                 Set.of(),
-                xContentRegistry()
+                xContentRegistry(),
+                telemetryPlugin.getTelemetryProvider(Settings.EMPTY)
             );
 
             MaxDocsCondition condition = new MaxDocsCondition(randomNonNegativeLong());
@@ -675,6 +685,7 @@ public void testRolloverClusterStateForDataStreamFailureStore() throws Exception
         dataStream.getFailureIndices().forEach(index -> builder.put(DataStreamTestHelper.getIndexMetadataBuilderForIndex(index)));
         builder.put(dataStream);
         final ClusterState clusterState = ClusterState.builder(new ClusterName("test")).metadata(builder).build();
+        final TestTelemetryPlugin telemetryPlugin = new TestTelemetryPlugin();
 
         ThreadPool testThreadPool = new TestThreadPool(getTestName());
         try {
@@ -682,7 +693,8 @@ public void testRolloverClusterStateForDataStreamFailureStore() throws Exception
                 dataStream,
                 testThreadPool,
                 Set.of(),
-                xContentRegistry()
+                xContentRegistry(),
+                telemetryPlugin.getTelemetryProvider(Settings.EMPTY)
             );
 
             MaxDocsCondition condition = new MaxDocsCondition(randomNonNegativeLong());
@@ -782,13 +794,15 @@ public void testValidation() throws Exception {
         MetadataCreateIndexService createIndexService = mock(MetadataCreateIndexService.class);
         MetadataIndexAliasesService metadataIndexAliasesService = mock(MetadataIndexAliasesService.class);
         ClusterService clusterService = mock(ClusterService.class);
+        final TestTelemetryPlugin telemetryPlugin = new TestTelemetryPlugin();
         MetadataRolloverService rolloverService = new MetadataRolloverService(
             null,
             createIndexService,
             metadataIndexAliasesService,
             EmptySystemIndices.INSTANCE,
             WriteLoadForecaster.DEFAULT,
-            clusterService
+            clusterService,
+            telemetryPlugin.getTelemetryProvider(Settings.EMPTY)
         );
 
         String newIndexName = useDataStream == false && randomBoolean() ? "logs-index-9" : null;
@@ -821,13 +835,15 @@ public void testRolloverClusterStateForDataStreamNoTemplate() throws Exception {
         }
         builder.put(dataStream);
         final ClusterState clusterState = ClusterState.builder(new ClusterName("test")).metadata(builder).build();
+        final TestTelemetryPlugin telemetryPlugin = new TestTelemetryPlugin();
 
         ThreadPool testThreadPool = mock(ThreadPool.class);
         MetadataRolloverService rolloverService = DataStreamTestHelper.getMetadataRolloverService(
             dataStream,
             testThreadPool,
             Set.of(),
-            xContentRegistry()
+            xContentRegistry(),
+            telemetryPlugin.getTelemetryProvider(Settings.EMPTY)
         );
 
         MaxDocsCondition condition = new MaxDocsCondition(randomNonNegativeLong());
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java
index 427d2769b7399..42c4dec3e219b 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java
@@ -64,6 +64,7 @@
 import org.elasticsearch.indices.EmptySystemIndices;
 import org.elasticsearch.search.suggest.completion.CompletionStats;
 import org.elasticsearch.tasks.CancellableTask;
+import org.elasticsearch.telemetry.TestTelemetryPlugin;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
@@ -105,13 +106,15 @@ public class TransportRolloverActionTests extends ESTestCase {
     final MetadataDataStreamsService mockMetadataDataStreamService = mock(MetadataDataStreamsService.class);
     final Client mockClient = mock(Client.class);
     final AllocationService mockAllocationService = mock(AllocationService.class);
+    final TestTelemetryPlugin telemetryPlugin = new TestTelemetryPlugin();
     final MetadataRolloverService rolloverService = new MetadataRolloverService(
         mockThreadPool,
         mockCreateIndexService,
         mdIndexAliasesService,
         EmptySystemIndices.INSTANCE,
         WriteLoadForecaster.DEFAULT,
-        mockClusterService
+        mockClusterService,
+        telemetryPlugin.getTelemetryProvider(Settings.EMPTY)
     );
 
     final DataStreamAutoShardingService dataStreamAutoShardingService = new DataStreamAutoShardingService(
diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/metadata/DataStreamTestHelper.java b/test/framework/src/main/java/org/elasticsearch/cluster/metadata/DataStreamTestHelper.java
index 6c038470b158d..e6252e46a12a3 100644
--- a/test/framework/src/main/java/org/elasticsearch/cluster/metadata/DataStreamTestHelper.java
+++ b/test/framework/src/main/java/org/elasticsearch/cluster/metadata/DataStreamTestHelper.java
@@ -47,6 +47,7 @@
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.indices.ShardLimitValidator;
 import org.elasticsearch.script.ScriptCompiler;
+import org.elasticsearch.telemetry.TelemetryProvider;
 import org.elasticsearch.test.ClusterServiceUtils;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -623,7 +624,8 @@ public static MetadataRolloverService getMetadataRolloverService(
         DataStream dataStream,
         ThreadPool testThreadPool,
         Set<IndexSettingProvider> providers,
-        NamedXContentRegistry registry
+        NamedXContentRegistry registry,
+        TelemetryProvider telemetryProvider
     ) throws Exception {
         DateFieldMapper dateFieldMapper = new DateFieldMapper.Builder(
             "@timestamp",
@@ -684,7 +686,8 @@ public static MetadataRolloverService getMetadataRolloverService(
             indexAliasesService,
             EmptySystemIndices.INSTANCE,
             WriteLoadForecaster.DEFAULT,
-            clusterService
+            clusterService,
+            telemetryProvider
         );
     }
 

From 658c01401696c4685e8f7acf18865e2061fa534e Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Fri, 26 Apr 2024 08:03:30 -0700
Subject: [PATCH 55/58] Block readiness on bad initial file settings (#107775)

If file settings have an update that fails, existing applied file
settings continue to work. But if the initial file settings fail to
process, readiness should be blocked. This commit adjusts readiness to
look for this special initialization case.

relates #107738
---
 .../java/org/elasticsearch/readiness/ReadinessClusterIT.java  | 1 -
 .../elasticsearch/cluster/metadata/ReservedStateMetadata.java | 4 +++-
 .../java/org/elasticsearch/readiness/ReadinessService.java    | 2 +-
 .../cluster/metadata/ReservedStateMetadataTests.java          | 3 ++-
 .../cluster/metadata/ToAndFromJsonMetadataTests.java          | 4 ++--
 5 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/server/src/internalClusterTest/java/org/elasticsearch/readiness/ReadinessClusterIT.java b/server/src/internalClusterTest/java/org/elasticsearch/readiness/ReadinessClusterIT.java
index 2ecdd06f379d2..1f8d55516d508 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/readiness/ReadinessClusterIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/readiness/ReadinessClusterIT.java
@@ -251,7 +251,6 @@ private void writeFileSettings(String json) throws Exception {
         logger.info("--> New file settings: [{}]", Strings.format(json, version));
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/107744")
     public void testNotReadyOnBadFileSettings() throws Exception {
         internalCluster().setBootstrapMasterNodeIndex(0);
         logger.info("--> start data node / non master node");
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ReservedStateMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ReservedStateMetadata.java
index dd5ca03cf759a..ec8200bf2d701 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/ReservedStateMetadata.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ReservedStateMetadata.java
@@ -46,6 +46,8 @@ public record ReservedStateMetadata(
     ReservedStateErrorMetadata errorMetadata
 ) implements SimpleDiffable<ReservedStateMetadata>, ToXContentFragment {
 
+    public static final Long NO_VERSION = Long.MIN_VALUE; // use min long as sentinel for uninitialized version
+
     private static final ParseField VERSION = new ParseField("version");
     private static final ParseField HANDLERS = new ParseField("handlers");
     private static final ParseField ERRORS_METADATA = new ParseField("errors");
@@ -209,7 +211,7 @@ public static class Builder {
          */
         public Builder(String namespace) {
             this.namespace = namespace;
-            this.version = -1L;
+            this.version = NO_VERSION;
             this.handlers = new HashMap<>();
             this.errorMetadata = null;
         }
diff --git a/server/src/main/java/org/elasticsearch/readiness/ReadinessService.java b/server/src/main/java/org/elasticsearch/readiness/ReadinessService.java
index 1cac133106403..61425250c19b4 100644
--- a/server/src/main/java/org/elasticsearch/readiness/ReadinessService.java
+++ b/server/src/main/java/org/elasticsearch/readiness/ReadinessService.java
@@ -254,7 +254,7 @@ public void clusterChanged(ClusterChangedEvent event) {
     // protected to allow mock service to override
     protected boolean areFileSettingsApplied(ClusterState clusterState) {
         ReservedStateMetadata fileSettingsMetadata = clusterState.metadata().reservedStateMetadata().get(FileSettingsService.NAMESPACE);
-        return fileSettingsMetadata != null;
+        return fileSettingsMetadata != null && fileSettingsMetadata.version().equals(ReservedStateMetadata.NO_VERSION) == false;
     }
 
     private void setReady(boolean ready) {
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/ReservedStateMetadataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/ReservedStateMetadataTests.java
index 46be49ad7111f..5086813cc5c13 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/ReservedStateMetadataTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/ReservedStateMetadataTests.java
@@ -20,6 +20,7 @@
 import java.util.Collections;
 import java.util.List;
 
+import static org.elasticsearch.cluster.metadata.ReservedStateMetadata.NO_VERSION;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.not;
 
@@ -78,7 +79,7 @@ public void testXContent() throws IOException {
 
     public void testReservedStateVersionWithError() {
         final ReservedStateMetadata meta = createRandom(false, true);
-        assertEquals(-1L, meta.version().longValue());
+        assertEquals(NO_VERSION.longValue(), meta.version().longValue());
     }
 
     private static ReservedStateMetadata createRandom(boolean addHandlers, boolean addErrors) {
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetadataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetadataTests.java
index aa9d0b9368fa6..3a522f3f5c06c 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetadataTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetadataTests.java
@@ -776,7 +776,7 @@ public void testToXContentAPIReservedMetadata() throws IOException {
                 },
                 "reserved_state" : {
                   "namespace_one" : {
-                    "version" : -1,
+                    "version" : -9223372036854775808,
                     "handlers" : {
                       "one" : {
                         "keys" : [
@@ -801,7 +801,7 @@ public void testToXContentAPIReservedMetadata() throws IOException {
                     }
                   },
                   "namespace_two" : {
-                    "version" : -1,
+                    "version" : -9223372036854775808,
                     "handlers" : {
                       "three" : {
                         "keys" : [

From d7e524fcf9835f4b31369dd2cd0ef8da4994c9a3 Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Fri, 26 Apr 2024 08:08:24 -0700
Subject: [PATCH 56/58] Make auto heap configuration configurable in server cli
 subclasses (#107919)

This commit makes auto heap configuration extendible so that serverless
can tweak the configuration based on project settings.
---
 .../server/cli/JvmOptionsParser.java          |  20 +-
 .../server/cli/MachineDependentHeap.java      | 228 ++++++++----------
 .../elasticsearch/server/cli/ServerCli.java   |   2 +-
 .../server/cli/MachineDependentHeapTests.java |  97 +++-----
 .../server/cli/NodeRoleParserTests.java       | 103 --------
 .../windows/service/WindowsServiceDaemon.java |   3 +-
 6 files changed, 143 insertions(+), 310 deletions(-)
 delete mode 100644 distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/NodeRoleParserTests.java

diff --git a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/JvmOptionsParser.java b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/JvmOptionsParser.java
index 35f3f62122f0c..0bfa0f211807d 100644
--- a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/JvmOptionsParser.java
+++ b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/JvmOptionsParser.java
@@ -72,13 +72,18 @@ SortedMap<Integer, String> invalidLines() {
      * @param args            the start-up arguments
      * @param processInfo     information about the CLI process.
      * @param tmpDir          the directory that should be passed to {@code -Djava.io.tmpdir}
+     * @param machineDependentHeap the heap configurator to use
      * @return the list of options to put on the Java command line
      * @throws InterruptedException if the java subprocess is interrupted
      * @throws IOException          if there is a problem reading any of the files
      * @throws UserException        if there is a problem parsing the `jvm.options` file or `jvm.options.d` files
      */
-    public static List<String> determineJvmOptions(ServerArgs args, ProcessInfo processInfo, Path tmpDir) throws InterruptedException,
-        IOException, UserException {
+    public static List<String> determineJvmOptions(
+        ServerArgs args,
+        ProcessInfo processInfo,
+        Path tmpDir,
+        MachineDependentHeap machineDependentHeap
+    ) throws InterruptedException, IOException, UserException {
         final JvmOptionsParser parser = new JvmOptionsParser();
 
         final Map<String, String> substitutions = new HashMap<>();
@@ -89,7 +94,7 @@ public static List<String> determineJvmOptions(ServerArgs args, ProcessInfo proc
 
         try {
             return Collections.unmodifiableList(
-                parser.jvmOptions(args, args.configDir(), tmpDir, envOptions, substitutions, processInfo.sysprops())
+                parser.jvmOptions(args, args.configDir(), tmpDir, envOptions, substitutions, processInfo.sysprops(), machineDependentHeap)
             );
         } catch (final JvmOptionsFileParserException e) {
             final String errorMessage = String.format(
@@ -125,7 +130,8 @@ private List<String> jvmOptions(
         Path tmpDir,
         final String esJavaOpts,
         final Map<String, String> substitutions,
-        final Map<String, String> cliSysprops
+        final Map<String, String> cliSysprops,
+        final MachineDependentHeap machineDependentHeap
     ) throws InterruptedException, IOException, JvmOptionsFileParserException, UserException {
 
         final List<String> jvmOptions = readJvmOptionsFiles(config);
@@ -135,10 +141,8 @@ private List<String> jvmOptions(
         }
 
         final List<String> substitutedJvmOptions = substitutePlaceholders(jvmOptions, Collections.unmodifiableMap(substitutions));
-        final MachineDependentHeap machineDependentHeap = new MachineDependentHeap(
-            new OverridableSystemMemoryInfo(substitutedJvmOptions, new DefaultSystemMemoryInfo())
-        );
-        substitutedJvmOptions.addAll(machineDependentHeap.determineHeapSettings(config, substitutedJvmOptions));
+        final SystemMemoryInfo memoryInfo = new OverridableSystemMemoryInfo(substitutedJvmOptions, new DefaultSystemMemoryInfo());
+        substitutedJvmOptions.addAll(machineDependentHeap.determineHeapSettings(args.nodeSettings(), memoryInfo, substitutedJvmOptions));
         final List<String> ergonomicJvmOptions = JvmErgonomics.choose(substitutedJvmOptions, args.nodeSettings());
         final List<String> systemJvmOptions = SystemJvmOptions.systemJvmOptions(args.nodeSettings(), cliSysprops);
 
diff --git a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/MachineDependentHeap.java b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/MachineDependentHeap.java
index 87c4883ca3073..b7ef9e46a758d 100644
--- a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/MachineDependentHeap.java
+++ b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/MachineDependentHeap.java
@@ -8,24 +8,22 @@
 
 package org.elasticsearch.server.cli;
 
-import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.xcontent.XContentParserConfiguration;
-import org.elasticsearch.xcontent.yaml.YamlXContent;
+import org.elasticsearch.node.NodeRoleSettings;
 
 import java.io.IOException;
-import java.io.InputStream;
-import java.nio.file.Files;
-import java.nio.file.Path;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
-import java.util.function.Function;
 
 import static java.lang.Math.max;
 import static java.lang.Math.min;
+import static org.elasticsearch.cluster.node.DiscoveryNodeRole.MASTER_ROLE;
+import static org.elasticsearch.cluster.node.DiscoveryNodeRole.ML_ROLE;
+import static org.elasticsearch.cluster.node.DiscoveryNodeRole.REMOTE_CLUSTER_CLIENT_ROLE;
 import static org.elasticsearch.server.cli.JvmOption.isInitialHeapSpecified;
 import static org.elasticsearch.server.cli.JvmOption.isMaxHeapSpecified;
 import static org.elasticsearch.server.cli.JvmOption.isMinHeapSpecified;
@@ -33,28 +31,26 @@
 /**
  * Determines optimal default heap settings based on available system memory and assigned node roles.
  */
-public final class MachineDependentHeap {
+public class MachineDependentHeap {
     private static final long GB = 1024L * 1024L * 1024L; // 1GB
     private static final long MAX_HEAP_SIZE = GB * 31; // 31GB
     private static final long MIN_HEAP_SIZE = 1024 * 1024 * 128; // 128MB
-    private static final int DEFAULT_HEAP_SIZE_MB = 1024;
-    private static final String ELASTICSEARCH_YML = "elasticsearch.yml";
 
-    private final SystemMemoryInfo systemMemoryInfo;
-
-    public MachineDependentHeap(SystemMemoryInfo systemMemoryInfo) {
-        this.systemMemoryInfo = systemMemoryInfo;
-    }
+    public MachineDependentHeap() {}
 
     /**
      * Calculate heap options.
      *
-     * @param configDir path to config directory
+     * @param nodeSettings the settings for the node
      * @param userDefinedJvmOptions JVM arguments provided by the user
      * @return final heap options, or an empty collection if user provided heap options are to be used
      * @throws IOException if unable to load elasticsearch.yml
      */
-    public List<String> determineHeapSettings(Path configDir, List<String> userDefinedJvmOptions) throws IOException, InterruptedException {
+    public final List<String> determineHeapSettings(
+        Settings nodeSettings,
+        SystemMemoryInfo systemMemoryInfo,
+        List<String> userDefinedJvmOptions
+    ) throws IOException, InterruptedException {
         // TODO: this could be more efficient, to only parse final options once
         final Map<String, JvmOption> finalJvmOptions = JvmOption.findFinalOptions(userDefinedJvmOptions);
         if (isMaxHeapSpecified(finalJvmOptions) || isMinHeapSpecified(finalJvmOptions) || isInitialHeapSpecified(finalJvmOptions)) {
@@ -62,139 +58,103 @@ public List<String> determineHeapSettings(Path configDir, List<String> userDefin
             return Collections.emptyList();
         }
 
-        Path config = configDir.resolve(ELASTICSEARCH_YML);
-        try (InputStream in = Files.newInputStream(config)) {
-            return determineHeapSettings(in);
-        }
-    }
-
-    List<String> determineHeapSettings(InputStream config) {
-        MachineNodeRole nodeRole = NodeRoleParser.parse(config);
+        List<DiscoveryNodeRole> roles = NodeRoleSettings.NODE_ROLES_SETTING.get(nodeSettings);
         long availableSystemMemory = systemMemoryInfo.availableSystemMemory();
-        return options(nodeRole.heap(availableSystemMemory));
+        MachineNodeRole nodeRole = mapNodeRole(roles);
+        return options(getHeapSizeMb(nodeSettings, nodeRole, availableSystemMemory));
     }
 
-    private static List<String> options(int heapSize) {
-        return List.of("-Xms" + heapSize + "m", "-Xmx" + heapSize + "m");
-    }
-
-    /**
-     * Parses role information from elasticsearch.yml and determines machine node role.
-     */
-    static class NodeRoleParser {
-
-        @SuppressWarnings("unchecked")
-        public static MachineNodeRole parse(InputStream config) {
-            final Settings settings;
-            try (var parser = YamlXContent.yamlXContent.createParser(XContentParserConfiguration.EMPTY, config)) {
-                if (parser.currentToken() == null && parser.nextToken() == null) {
-                    settings = null;
+    protected int getHeapSizeMb(Settings nodeSettings, MachineNodeRole role, long availableMemory) {
+        return switch (role) {
+            /*
+             * Master-only node.
+             *
+             * <p>Heap is computed as 60% of total system memory up to a maximum of 31 gigabytes.
+             */
+            case MASTER_ONLY -> mb(min((long) (availableMemory * .6), MAX_HEAP_SIZE));
+            /*
+             * Machine learning only node.
+             *
+             * <p>Heap is computed as:
+             * <ul>
+             *     <li>40% of total system memory when total system memory 16 gigabytes or less.</li>
+             *     <li>40% of the first 16 gigabytes plus 10% of memory above that when total system memory is more than 16 gigabytes.</li>
+             *     <li>The absolute maximum heap size is 31 gigabytes.</li>
+             * </ul>
+             *
+             * In all cases the result is rounded down to the next whole multiple of 4 megabytes.
+             * The reason for doing this is that Java will round requested heap sizes to a multiple
+             * of 4 megabytes (certainly versions 11 to 18 do this), so by doing this ourselves we
+             * are more likely to actually get the amount we request. This is worthwhile for ML where
+             * the ML autoscaling code needs to be able to calculate the JVM size for different sizes
+             * of ML node, and if Java is also rounding then this causes a discrepancy. It's possible
+             * that a future version of Java could round to an even bigger number of megabytes, which
+             * would cause a discrepancy for people using that version of Java. But there's no harm
+             * in a bit of extra rounding here - it can only reduce discrepancies.
+             *
+             * If this formula is changed then corresponding changes must be made to the {@code NativeMemoryCalculator} and
+             * {@code MlAutoscalingDeciderServiceTests} classes in the ML plugin code. Failure to keep the logic synchronized
+             * could result in repeated autoscaling up and down.
+             */
+            case ML_ONLY -> {
+                if (availableMemory <= (GB * 16)) {
+                    yield mb((long) (availableMemory * .4), 4);
                 } else {
-                    settings = Settings.fromXContent(parser);
+                    yield mb((long) min((GB * 16) * .4 + (availableMemory - GB * 16) * .1, MAX_HEAP_SIZE), 4);
                 }
-            } catch (IOException | ParsingException ex) {
-                // Strangely formatted config, so just return defaults and let startup settings validation catch the problem
-                return MachineNodeRole.UNKNOWN;
             }
-
-            if (settings != null && settings.isEmpty() == false) {
-                List<String> roles = settings.getAsList("node.roles");
-
-                if (roles.isEmpty()) {
-                    // If roles are missing or empty (coordinating node) assume defaults and consider this a data node
-                    return MachineNodeRole.DATA;
-                } else if (containsOnly(roles, "master")) {
-                    return MachineNodeRole.MASTER_ONLY;
-                } else if (roles.contains("ml") && containsOnly(roles, "ml", "remote_cluster_client")) {
-                    return MachineNodeRole.ML_ONLY;
+            /*
+             * Data node. Essentially any node that isn't a master or ML only node.
+             *
+             * <p>Heap is computed as:
+             * <ul>
+             *     <li>40% of total system memory when less than 1 gigabyte with a minimum of 128 megabytes.</li>
+             *     <li>50% of total system memory when greater than 1 gigabyte up to a maximum of 31 gigabytes.</li>
+             * </ul>
+             */
+            case DATA -> {
+                if (availableMemory < GB) {
+                    yield mb(max((long) (availableMemory * .4), MIN_HEAP_SIZE));
                 } else {
-                    return MachineNodeRole.DATA;
+                    yield mb(min((long) (availableMemory * .5), MAX_HEAP_SIZE));
                 }
-            } else { // if the config is completely empty, then assume defaults and consider this a data node
-                return MachineNodeRole.DATA;
             }
-        }
-
-        @SuppressWarnings("unchecked")
-        private static <T> boolean containsOnly(Collection<T> collection, T... items) {
-            return Arrays.asList(items).containsAll(collection);
-        }
+        };
     }
 
-    enum MachineNodeRole {
-        /**
-         * Master-only node.
-         *
-         * <p>Heap is computed as 60% of total system memory up to a maximum of 31 gigabytes.
-         */
-        MASTER_ONLY(m -> mb(min((long) (m * .6), MAX_HEAP_SIZE))),
-
-        /**
-         * Machine learning only node.
-         *
-         * <p>Heap is computed as:
-         * <ul>
-         *     <li>40% of total system memory when total system memory 16 gigabytes or less.</li>
-         *     <li>40% of the first 16 gigabytes plus 10% of memory above that when total system memory is more than 16 gigabytes.</li>
-         *     <li>The absolute maximum heap size is 31 gigabytes.</li>
-         * </ul>
-         *
-         * In all cases the result is rounded down to the next whole multiple of 4 megabytes.
-         * The reason for doing this is that Java will round requested heap sizes to a multiple
-         * of 4 megabytes (certainly versions 11 to 18 do this), so by doing this ourselves we
-         * are more likely to actually get the amount we request. This is worthwhile for ML where
-         * the ML autoscaling code needs to be able to calculate the JVM size for different sizes
-         * of ML node, and if Java is also rounding then this causes a discrepancy. It's possible
-         * that a future version of Java could round to an even bigger number of megabytes, which
-         * would cause a discrepancy for people using that version of Java. But there's no harm
-         * in a bit of extra rounding here - it can only reduce discrepancies.
-         *
-         * If this formula is changed then corresponding changes must be made to the {@code NativeMemoryCalculator} and
-         * {@code MlAutoscalingDeciderServiceTests} classes in the ML plugin code. Failure to keep the logic synchronized
-         * could result in repeated autoscaling up and down.
-         */
-        ML_ONLY(m -> mb(m <= (GB * 16) ? (long) (m * .4) : (long) min((GB * 16) * .4 + (m - GB * 16) * .1, MAX_HEAP_SIZE), 4)),
-
-        /**
-         * Data node. Essentially any node that isn't a master or ML only node.
-         *
-         * <p>Heap is computed as:
-         * <ul>
-         *     <li>40% of total system memory when less than 1 gigabyte with a minimum of 128 megabytes.</li>
-         *     <li>50% of total system memory when greater than 1 gigabyte up to a maximum of 31 gigabytes.</li>
-         * </ul>
-         */
-        DATA(m -> mb(m < GB ? max((long) (m * .4), MIN_HEAP_SIZE) : min((long) (m * .5), MAX_HEAP_SIZE))),
-
-        /**
-         * Unknown role node.
-         *
-         * <p>Hard-code heap to a default of 1 gigabyte.
-         */
-        UNKNOWN(m -> DEFAULT_HEAP_SIZE_MB);
+    protected static int mb(long bytes) {
+        return (int) (bytes / (1024 * 1024));
+    }
 
-        private final Function<Long, Integer> formula;
+    protected static int mb(long bytes, int toLowerMultipleOfMb) {
+        return toLowerMultipleOfMb * (int) (bytes / (1024 * 1024 * toLowerMultipleOfMb));
+    }
 
-        MachineNodeRole(Function<Long, Integer> formula) {
-            this.formula = formula;
+    private static MachineNodeRole mapNodeRole(List<DiscoveryNodeRole> roles) {
+        if (roles.isEmpty()) {
+            // If roles are missing or empty (coordinating node) assume defaults and consider this a data node
+            return MachineNodeRole.DATA;
+        } else if (containsOnly(roles, MASTER_ROLE)) {
+            return MachineNodeRole.MASTER_ONLY;
+        } else if (roles.contains(ML_ROLE) && containsOnly(roles, ML_ROLE, REMOTE_CLUSTER_CLIENT_ROLE)) {
+            return MachineNodeRole.ML_ONLY;
+        } else {
+            return MachineNodeRole.DATA;
         }
+    }
 
-        /**
-         * Determine the appropriate heap size for the given role and available system memory.
-         *
-         * @param systemMemory total available system memory in bytes
-         * @return recommended heap size in megabytes
-         */
-        public int heap(long systemMemory) {
-            return formula.apply(systemMemory);
-        }
+    @SuppressWarnings("unchecked")
+    private static <T> boolean containsOnly(Collection<T> collection, T... items) {
+        return Arrays.asList(items).containsAll(collection);
+    }
 
-        private static int mb(long bytes) {
-            return (int) (bytes / (1024 * 1024));
-        }
+    private static List<String> options(int heapSize) {
+        return List.of("-Xms" + heapSize + "m", "-Xmx" + heapSize + "m");
+    }
 
-        private static int mb(long bytes, int toLowerMultipleOfMb) {
-            return toLowerMultipleOfMb * (int) (bytes / (1024 * 1024 * toLowerMultipleOfMb));
-        }
+    protected enum MachineNodeRole {
+        MASTER_ONLY,
+        ML_ONLY,
+        DATA;
     }
 }
diff --git a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerCli.java b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerCli.java
index 6dbff2fbfff9c..0505ab86127cf 100644
--- a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerCli.java
+++ b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerCli.java
@@ -250,7 +250,7 @@ protected Command loadTool(String toolname, String libs) {
     // protected to allow tests to override
     protected ServerProcess startServer(Terminal terminal, ProcessInfo processInfo, ServerArgs args) throws Exception {
         var tempDir = ServerProcessUtils.setupTempDir(processInfo);
-        var jvmOptions = JvmOptionsParser.determineJvmOptions(args, processInfo, tempDir);
+        var jvmOptions = JvmOptionsParser.determineJvmOptions(args, processInfo, tempDir, new MachineDependentHeap());
         var serverProcessBuilder = new ServerProcessBuilder().withTerminal(terminal)
             .withProcessInfo(processInfo)
             .withServerArgs(args)
diff --git a/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/MachineDependentHeapTests.java b/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/MachineDependentHeapTests.java
index 5b30c2246c624..0774773cbfa0b 100644
--- a/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/MachineDependentHeapTests.java
+++ b/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/MachineDependentHeapTests.java
@@ -8,16 +8,13 @@
 
 package org.elasticsearch.server.cli;
 
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.ESTestCase.WithoutSecurityManager;
+import org.hamcrest.Matcher;
 
-import java.io.ByteArrayInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.UncheckedIOException;
 import java.net.URISyntaxException;
 import java.net.URL;
-import java.nio.charset.StandardCharsets;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.Collections;
@@ -25,95 +22,69 @@
 
 import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.empty;
-import static org.junit.Assert.assertThat;
 
 // TODO: rework these tests to mock jvm option finder so they can run with security manager, no forking needed
 @WithoutSecurityManager
 public class MachineDependentHeapTests extends ESTestCase {
 
     public void testDefaultHeapSize() throws Exception {
-        MachineDependentHeap heap = new MachineDependentHeap(systemMemoryInGigabytes(8));
-        List<String> options = heap.determineHeapSettings(configPath(), Collections.emptyList());
+        MachineDependentHeap heap = new MachineDependentHeap();
+        List<String> options = heap.determineHeapSettings(Settings.EMPTY, systemMemoryInGigabytes(8), Collections.emptyList());
         assertThat(options, containsInAnyOrder("-Xmx4096m", "-Xms4096m"));
     }
 
     public void testUserPassedHeapArgs() throws Exception {
-        MachineDependentHeap heap = new MachineDependentHeap(systemMemoryInGigabytes(8));
-        List<String> options = heap.determineHeapSettings(configPath(), List.of("-Xmx4g"));
+        var systemMemoryInfo = systemMemoryInGigabytes(8);
+        MachineDependentHeap heap = new MachineDependentHeap();
+        List<String> options = heap.determineHeapSettings(Settings.EMPTY, systemMemoryInfo, List.of("-Xmx4g"));
         assertThat(options, empty());
 
-        options = heap.determineHeapSettings(configPath(), List.of("-Xms4g"));
+        options = heap.determineHeapSettings(Settings.EMPTY, systemMemoryInfo, List.of("-Xms4g"));
         assertThat(options, empty());
     }
 
     // Explicitly test odd heap sizes
     // See: https://github.com/elastic/elasticsearch/issues/86431
     public void testOddUserPassedHeapArgs() throws Exception {
-        MachineDependentHeap heap = new MachineDependentHeap(systemMemoryInGigabytes(8));
-        List<String> options = heap.determineHeapSettings(configPath(), List.of("-Xmx409m"));
+        var systemMemoryInfo = systemMemoryInGigabytes(8);
+        MachineDependentHeap heap = new MachineDependentHeap();
+        List<String> options = heap.determineHeapSettings(Settings.EMPTY, systemMemoryInfo, List.of("-Xmx409m"));
         assertThat(options, empty());
 
-        options = heap.determineHeapSettings(configPath(), List.of("-Xms409m"));
+        options = heap.determineHeapSettings(Settings.EMPTY, systemMemoryInfo, List.of("-Xms409m"));
         assertThat(options, empty());
     }
 
-    public void testMasterOnlyOptions() {
-        List<String> options = calculateHeap(16, "master");
-        assertThat(options, containsInAnyOrder("-Xmx9830m", "-Xms9830m"));
-
-        options = calculateHeap(64, "master");
-        assertThat(options, containsInAnyOrder("-Xmx31744m", "-Xms31744m"));
+    public void testMasterOnlyOptions() throws Exception {
+        assertHeapOptions(16, containsInAnyOrder("-Xmx9830m", "-Xms9830m"), "master");
+        assertHeapOptions(64, containsInAnyOrder("-Xmx31744m", "-Xms31744m"), "master");
     }
 
-    public void testMlOnlyOptions() {
-        List<String> options = calculateHeap(1, "ml");
-        assertThat(options, containsInAnyOrder("-Xmx408m", "-Xms408m"));
-
-        options = calculateHeap(4, "ml");
-        assertThat(options, containsInAnyOrder("-Xmx1636m", "-Xms1636m"));
-
-        options = calculateHeap(32, "ml");
-        assertThat(options, containsInAnyOrder("-Xmx8192m", "-Xms8192m"));
-
-        options = calculateHeap(64, "ml");
-        assertThat(options, containsInAnyOrder("-Xmx11468m", "-Xms11468m"));
-
+    public void testMlOnlyOptions() throws Exception {
+        assertHeapOptions(1, containsInAnyOrder("-Xmx408m", "-Xms408m"), "ml");
+        assertHeapOptions(4, containsInAnyOrder("-Xmx1636m", "-Xms1636m"), "ml");
+        assertHeapOptions(32, containsInAnyOrder("-Xmx8192m", "-Xms8192m"), "ml");
+        assertHeapOptions(64, containsInAnyOrder("-Xmx11468m", "-Xms11468m"), "ml");
         // We'd never see a node this big in Cloud, but this assertion proves that the 31GB absolute maximum
         // eventually kicks in (because 0.4 * 16 + 0.1 * (263 - 16) > 31)
-        options = calculateHeap(263, "ml");
-        assertThat(options, containsInAnyOrder("-Xmx31744m", "-Xms31744m"));
-
-    }
-
-    public void testDataNodeOptions() {
-        List<String> options = calculateHeap(1, "data");
-        assertThat(options, containsInAnyOrder("-Xmx512m", "-Xms512m"));
-
-        options = calculateHeap(8, "data");
-        assertThat(options, containsInAnyOrder("-Xmx4096m", "-Xms4096m"));
-
-        options = calculateHeap(64, "data");
-        assertThat(options, containsInAnyOrder("-Xmx31744m", "-Xms31744m"));
-
-        options = calculateHeap(0.5, "data");
-        assertThat(options, containsInAnyOrder("-Xmx204m", "-Xms204m"));
-
-        options = calculateHeap(0.2, "data");
-        assertThat(options, containsInAnyOrder("-Xmx128m", "-Xms128m"));
+        assertHeapOptions(263, containsInAnyOrder("-Xmx31744m", "-Xms31744m"), "ml");
     }
 
-    private static List<String> calculateHeap(double memoryInGigabytes, String... roles) {
-        MachineDependentHeap machineDependentHeap = new MachineDependentHeap(systemMemoryInGigabytes(memoryInGigabytes));
-        String configYaml = "node.roles: [" + String.join(",", roles) + "]";
-        return calculateHeap(machineDependentHeap, configYaml);
+    public void testDataNodeOptions() throws Exception {
+        assertHeapOptions(1, containsInAnyOrder("-Xmx512m", "-Xms512m"), "data");
+        assertHeapOptions(8, containsInAnyOrder("-Xmx4096m", "-Xms4096m"), "data");
+        assertHeapOptions(64, containsInAnyOrder("-Xmx31744m", "-Xms31744m"), "data");
+        assertHeapOptions(0.5, containsInAnyOrder("-Xmx204m", "-Xms204m"), "data");
+        assertHeapOptions(0.2, containsInAnyOrder("-Xmx128m", "-Xms128m"), "data");
     }
 
-    private static List<String> calculateHeap(MachineDependentHeap machineDependentHeap, String configYaml) {
-        try (InputStream in = new ByteArrayInputStream(configYaml.getBytes(StandardCharsets.UTF_8))) {
-            return machineDependentHeap.determineHeapSettings(in);
-        } catch (IOException e) {
-            throw new UncheckedIOException(e);
-        }
+    private void assertHeapOptions(double memoryInGigabytes, Matcher<Iterable<? extends String>> optionsMatcher, String... roles)
+        throws Exception {
+        SystemMemoryInfo systemMemoryInfo = systemMemoryInGigabytes(memoryInGigabytes);
+        MachineDependentHeap machineDependentHeap = new MachineDependentHeap();
+        Settings nodeSettings = Settings.builder().putList("node.roles", roles).build();
+        List<String> heapOptions = machineDependentHeap.determineHeapSettings(nodeSettings, systemMemoryInfo, Collections.emptyList());
+        assertThat(heapOptions, optionsMatcher);
     }
 
     private static SystemMemoryInfo systemMemoryInGigabytes(double gigabytes) {
diff --git a/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/NodeRoleParserTests.java b/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/NodeRoleParserTests.java
deleted file mode 100644
index 4d501c1116732..0000000000000
--- a/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/NodeRoleParserTests.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.server.cli;
-
-import org.elasticsearch.test.ESTestCase;
-
-import java.io.ByteArrayInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.nio.charset.StandardCharsets;
-import java.util.function.Consumer;
-
-import static org.elasticsearch.server.cli.MachineDependentHeap.MachineNodeRole.DATA;
-import static org.elasticsearch.server.cli.MachineDependentHeap.MachineNodeRole.MASTER_ONLY;
-import static org.elasticsearch.server.cli.MachineDependentHeap.MachineNodeRole.ML_ONLY;
-import static org.elasticsearch.server.cli.MachineDependentHeap.MachineNodeRole.UNKNOWN;
-import static org.hamcrest.CoreMatchers.not;
-import static org.hamcrest.Matchers.equalTo;
-import static org.junit.Assert.assertThat;
-
-public class NodeRoleParserTests extends ESTestCase {
-
-    public void testMasterOnlyNode() throws IOException {
-        MachineDependentHeap.MachineNodeRole nodeRole = parseConfig(sb -> sb.append("node.roles: [master]"));
-        assertThat(nodeRole, equalTo(MASTER_ONLY));
-
-        nodeRole = parseConfig(sb -> sb.append("node.roles: [master, some_other_role]"));
-        assertThat(nodeRole, not(equalTo(MASTER_ONLY)));
-    }
-
-    public void testMlOnlyNode() throws IOException {
-        MachineDependentHeap.MachineNodeRole nodeRole = parseConfig(sb -> sb.append("node.roles: [ml]"));
-        assertThat(nodeRole, equalTo(ML_ONLY));
-
-        nodeRole = parseConfig(sb -> sb.append("node.roles: [ml, remote_cluster_client]"));
-        assertThat(nodeRole, equalTo(ML_ONLY));
-
-        nodeRole = parseConfig(sb -> sb.append("node.roles: [remote_cluster_client, ml]"));
-        assertThat(nodeRole, equalTo(ML_ONLY));
-
-        nodeRole = parseConfig(sb -> sb.append("node.roles: [remote_cluster_client]"));
-        assertThat(nodeRole, not(equalTo(ML_ONLY)));
-
-        nodeRole = parseConfig(sb -> sb.append("node.roles: [ml, some_other_role]"));
-        assertThat(nodeRole, not(equalTo(ML_ONLY)));
-    }
-
-    public void testDataNode() throws IOException {
-        MachineDependentHeap.MachineNodeRole nodeRole = parseConfig(sb -> {});
-        assertThat(nodeRole, equalTo(DATA));
-
-        nodeRole = parseConfig(sb -> sb.append("node.roles: []"));
-        assertThat(nodeRole, equalTo(DATA));
-
-        nodeRole = parseConfig(sb -> sb.append("node.roles: [some_unknown_role]"));
-        assertThat(nodeRole, equalTo(DATA));
-
-        nodeRole = parseConfig(sb -> sb.append("node.roles: [master, ingest]"));
-        assertThat(nodeRole, equalTo(DATA));
-
-        nodeRole = parseConfig(sb -> sb.append("node.roles: [ml, master]"));
-        assertThat(nodeRole, equalTo(DATA));
-    }
-
-    public void testYamlSyntax() throws IOException {
-        MachineDependentHeap.MachineNodeRole nodeRole = parseConfig(sb -> sb.append("""
-            node:
-              roles:
-                - master"""));
-        assertThat(nodeRole, equalTo(MASTER_ONLY));
-
-        nodeRole = parseConfig(sb -> sb.append("""
-            node:
-              roles: [ml]"""));
-        assertThat(nodeRole, equalTo(ML_ONLY));
-    }
-
-    public void testInvalidYaml() throws IOException {
-        MachineDependentHeap.MachineNodeRole nodeRole = parseConfig(sb -> sb.append("notyaml"));
-        assertThat(nodeRole, equalTo(UNKNOWN));
-    }
-
-    public void testInvalidRoleSyntax() throws IOException {
-        MachineDependentHeap.MachineNodeRole nodeRole = parseConfig(sb -> sb.append("node.roles: foo"));
-        // roles we don't know about are considered data, but will fail validation when ES starts up
-        assertThat(nodeRole, equalTo(DATA));
-    }
-
-    private static MachineDependentHeap.MachineNodeRole parseConfig(Consumer<StringBuilder> action) throws IOException {
-        StringBuilder sb = new StringBuilder();
-        action.accept(sb);
-
-        try (InputStream config = new ByteArrayInputStream(sb.toString().getBytes(StandardCharsets.UTF_8))) {
-            return MachineDependentHeap.NodeRoleParser.parse(config);
-        }
-    }
-}
diff --git a/distribution/tools/windows-service-cli/src/main/java/org/elasticsearch/windows/service/WindowsServiceDaemon.java b/distribution/tools/windows-service-cli/src/main/java/org/elasticsearch/windows/service/WindowsServiceDaemon.java
index 2c42dcf5cb2f5..22474e63ab0df 100644
--- a/distribution/tools/windows-service-cli/src/main/java/org/elasticsearch/windows/service/WindowsServiceDaemon.java
+++ b/distribution/tools/windows-service-cli/src/main/java/org/elasticsearch/windows/service/WindowsServiceDaemon.java
@@ -18,6 +18,7 @@
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.server.cli.JvmOptionsParser;
+import org.elasticsearch.server.cli.MachineDependentHeap;
 import org.elasticsearch.server.cli.ServerProcess;
 import org.elasticsearch.server.cli.ServerProcessBuilder;
 import org.elasticsearch.server.cli.ServerProcessUtils;
@@ -42,7 +43,7 @@ public void execute(Terminal terminal, OptionSet options, Environment env, Proce
         try (var loadedSecrets = KeyStoreWrapper.bootstrap(env.configFile(), () -> new SecureString(new char[0]))) {
             var args = new ServerArgs(false, true, null, loadedSecrets, env.settings(), env.configFile(), env.logsFile());
             var tempDir = ServerProcessUtils.setupTempDir(processInfo);
-            var jvmOptions = JvmOptionsParser.determineJvmOptions(args, processInfo, tempDir);
+            var jvmOptions = JvmOptionsParser.determineJvmOptions(args, processInfo, tempDir, new MachineDependentHeap());
             var serverProcessBuilder = new ServerProcessBuilder().withTerminal(terminal)
                 .withProcessInfo(processInfo)
                 .withServerArgs(args)

From 01cc967ce9306b5f21939efadc3771a6d0f323db Mon Sep 17 00:00:00 2001
From: Nhat Nguyen <nhat.nguyen@elastic.co>
Date: Fri, 26 Apr 2024 09:34:11 -0700
Subject: [PATCH 57/58] Mute synthetic source YAML tests (#107958)

Relates #107567
---
 .../indices.create/20_synthetic_source.yml     | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml
index 874778f9bdb5c..9376f3598d6f1 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml
@@ -39,9 +39,9 @@ nested is disabled:
 
 ---
 object with unmapped fields:
-  - requires:
-      cluster_features: ["mapper.track_ignored_source"]
-      reason: requires tracking ignored values
+  - skip:
+      version: " - "
+      reason: "mapper.track_ignored_source"
 
   - do:
       indices.create:
@@ -84,9 +84,9 @@ object with unmapped fields:
 
 ---
 nested object with unmapped fields:
-  - requires:
-      cluster_features: ["mapper.track_ignored_source"]
-      reason: requires tracking ignored values
+  - skip:
+      version: " - "
+      reason: "mapper.track_ignored_source"
 
   - do:
       indices.create:
@@ -130,9 +130,9 @@ nested object with unmapped fields:
 
 ---
 empty object with unmapped fields:
-  - requires:
-      cluster_features: ["mapper.track_ignored_source"]
-      reason: requires tracking ignored values
+  - skip:
+      version: " - "
+      reason: "mapper.track_ignored_source"
 
   - do:
       indices.create:

From ca513b17527adb7e9f1a97872e1ab12474cca391 Mon Sep 17 00:00:00 2001
From: Oleksandr Kolomiiets <oleksandr.kolomiiets@elastic.co>
Date: Fri, 26 Apr 2024 10:12:17 -0700
Subject: [PATCH 58/58] Annotated text fields are stored by default with
 synthetic source (#107922)

This change follows existing implementation for text field.

Closes #107734.
---
 docs/changelog/107922.yaml                    |  6 ++
 .../AnnotatedTextFieldMapper.java             | 30 ++++++---
 .../AnnotatedTextFieldMapperTests.java        | 61 +++++++++++++++++++
 .../AnnotatedTextFieldTypeTests.java          |  9 ++-
 4 files changed, 96 insertions(+), 10 deletions(-)
 create mode 100644 docs/changelog/107922.yaml

diff --git a/docs/changelog/107922.yaml b/docs/changelog/107922.yaml
new file mode 100644
index 0000000000000..e28d0f6262af4
--- /dev/null
+++ b/docs/changelog/107922.yaml
@@ -0,0 +1,6 @@
+pr: 107922
+summary: Feature/annotated text store defaults
+area: Mapping
+type: enhancement
+issues:
+ - 107734
diff --git a/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java b/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java
index 6d2b83185d5b7..e5e396888e168 100644
--- a/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java
+++ b/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java
@@ -83,11 +83,7 @@ private static NamedAnalyzer wrapAnalyzer(NamedAnalyzer in) {
 
     public static class Builder extends FieldMapper.Builder {
 
-        private final Parameter<Boolean> store = Parameter.storeParam(m -> builder(m).store.getValue(), false);
-
-        final TextParams.Analyzers analyzers;
         final Parameter<SimilarityProvider> similarity = TextParams.similarity(m -> builder(m).similarity.getValue());
-
         final Parameter<String> indexOptions = TextParams.textIndexOptions(m -> builder(m).indexOptions.getValue());
         final Parameter<Boolean> norms = TextParams.norms(true, m -> builder(m).norms.getValue());
         final Parameter<String> termVectors = TextParams.termVectors(m -> builder(m).termVectors.getValue());
@@ -95,8 +91,16 @@ public static class Builder extends FieldMapper.Builder {
         private final Parameter<Map<String, String>> meta = Parameter.metaParam();
 
         private final IndexVersion indexCreatedVersion;
+        private final TextParams.Analyzers analyzers;
+        private final boolean isSyntheticSourceEnabledViaIndexMode;
+        private final Parameter<Boolean> store;
 
-        public Builder(String name, IndexVersion indexCreatedVersion, IndexAnalyzers indexAnalyzers) {
+        public Builder(
+            String name,
+            IndexVersion indexCreatedVersion,
+            IndexAnalyzers indexAnalyzers,
+            boolean isSyntheticSourceEnabledViaIndexMode
+        ) {
             super(name);
             this.indexCreatedVersion = indexCreatedVersion;
             this.analyzers = new TextParams.Analyzers(
@@ -105,6 +109,11 @@ public Builder(String name, IndexVersion indexCreatedVersion, IndexAnalyzers ind
                 m -> builder(m).analyzers.positionIncrementGap.getValue(),
                 indexCreatedVersion
             );
+            this.isSyntheticSourceEnabledViaIndexMode = isSyntheticSourceEnabledViaIndexMode;
+            this.store = Parameter.storeParam(
+                m -> builder(m).store.getValue(),
+                () -> isSyntheticSourceEnabledViaIndexMode && multiFieldsBuilder.hasSyntheticSourceCompatibleKeywordField() == false
+            );
         }
 
         @Override
@@ -164,7 +173,9 @@ public AnnotatedTextFieldMapper build(MapperBuilderContext context) {
         }
     }
 
-    public static TypeParser PARSER = new TypeParser((n, c) -> new Builder(n, c.indexVersionCreated(), c.getIndexAnalyzers()));
+    public static TypeParser PARSER = new TypeParser(
+        (n, c) -> new Builder(n, c.indexVersionCreated(), c.getIndexAnalyzers(), c.getIndexSettings().getMode().isSyntheticSourceEnabled())
+    );
 
     /**
      * Parses markdown-like syntax into plain text and AnnotationTokens with offsets for
@@ -552,7 +563,12 @@ protected String contentType() {
 
     @Override
     public FieldMapper.Builder getMergeBuilder() {
-        return new Builder(simpleName(), builder.indexCreatedVersion, builder.analyzers.indexAnalyzers).init(this);
+        return new Builder(
+            simpleName(),
+            builder.indexCreatedVersion,
+            builder.analyzers.indexAnalyzers,
+            builder.isSyntheticSourceEnabledViaIndexMode
+        ).init(this);
     }
 
     @Override
diff --git a/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java b/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java
index 3b27cdb132851..4e3a53d64a841 100644
--- a/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java
+++ b/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java
@@ -24,7 +24,9 @@
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.index.TermsEnum;
 import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.analysis.AnalyzerScope;
 import org.elasticsearch.index.analysis.CharFilterFactory;
@@ -42,6 +44,7 @@
 import org.elasticsearch.index.mapper.ParsedDocument;
 import org.elasticsearch.index.mapper.TextFieldFamilySyntheticSourceTestSetup;
 import org.elasticsearch.index.mapper.TextFieldMapper;
+import org.elasticsearch.index.mapper.TimeSeriesRoutingHashFieldMapper;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContentBuilder;
@@ -288,6 +291,64 @@ public void testEnableStore() throws IOException {
         assertTrue(fields.get(0).fieldType().stored());
     }
 
+    public void testStoreParameterDefaults() throws IOException {
+        var timeSeriesIndexMode = randomBoolean();
+        var isStored = randomBoolean();
+        var hasKeywordFieldForSyntheticSource = randomBoolean();
+
+        var indexSettingsBuilder = getIndexSettingsBuilder();
+        if (timeSeriesIndexMode) {
+            indexSettingsBuilder.put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES)
+                .putList(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "dimension")
+                .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), "2000-01-08T23:40:53.384Z")
+                .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), "2106-01-08T23:40:53.384Z");
+        }
+        var indexSettings = indexSettingsBuilder.build();
+
+        var mapping = mapping(b -> {
+            b.startObject("field");
+            b.field("type", "annotated_text");
+            if (isStored) {
+                b.field("store", isStored);
+            }
+            if (hasKeywordFieldForSyntheticSource) {
+                b.startObject("fields");
+                b.startObject("keyword");
+                b.field("type", "keyword");
+                b.endObject();
+                b.endObject();
+            }
+            b.endObject();
+
+            if (timeSeriesIndexMode) {
+                b.startObject("@timestamp");
+                b.field("type", "date");
+                b.endObject();
+                b.startObject("dimension");
+                b.field("type", "keyword");
+                b.field("time_series_dimension", "true");
+                b.endObject();
+            }
+        });
+        DocumentMapper mapper = createMapperService(getVersion(), indexSettings, () -> true, mapping).documentMapper();
+
+        var source = source(TimeSeriesRoutingHashFieldMapper.DUMMY_ENCODED_VALUE, b -> {
+            b.field("field", "1234");
+            if (timeSeriesIndexMode) {
+                b.field("@timestamp", randomMillisUpToYear9999());
+                b.field("dimension", "dimension1");
+            }
+        }, null);
+        ParsedDocument doc = mapper.parse(source);
+        List<IndexableField> fields = doc.rootDoc().getFields("field");
+        IndexableFieldType fieldType = fields.get(0).fieldType();
+        if (isStored || (timeSeriesIndexMode && hasKeywordFieldForSyntheticSource == false)) {
+            assertTrue(fieldType.stored());
+        } else {
+            assertFalse(fieldType.stored());
+        }
+    }
+
     public void testDisableNorms() throws IOException {
 
         DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> {
diff --git a/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldTypeTests.java b/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldTypeTests.java
index 1b9f3b9447378..2a78699c8a4a9 100644
--- a/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldTypeTests.java
+++ b/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldTypeTests.java
@@ -29,9 +29,12 @@ public void testIntervals() throws IOException {
     }
 
     public void testFetchSourceValue() throws IOException {
-        MappedFieldType fieldType = new AnnotatedTextFieldMapper.Builder("field", IndexVersion.current(), createDefaultIndexAnalyzers())
-            .build(MapperBuilderContext.root(false, false))
-            .fieldType();
+        MappedFieldType fieldType = new AnnotatedTextFieldMapper.Builder(
+            "field",
+            IndexVersion.current(),
+            createDefaultIndexAnalyzers(),
+            false
+        ).build(MapperBuilderContext.root(false, false)).fieldType();
 
         assertEquals(List.of("value"), fetchSourceValue(fieldType, "value"));
         assertEquals(List.of("42"), fetchSourceValue(fieldType, 42L));