Add index_mode to data streams (elastic#82621)

This adds a `index_mode` field to data stream snippet
in a composable index template. Specifying `index_mode`
is optional.

If `index_mode` on a composable index template then
this also sets the `index_mode` property on the data stream
and the `index.mode` index setting on the backing
indices created by this data stream.

An `index_mode` of value `TIME_SERIES` on a data stream
also triggers time series validation. In particular
the temporal ranges of backing indices are checked on
whether these ranges do not overlap.

Relates to elastic#74660

server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java

@@ -13,6 +13,7 @@
 import org.elasticsearch.action.support.ActiveShardCount;
 import org.elasticsearch.cluster.ack.ClusterStateUpdateRequest;
 import org.elasticsearch.cluster.block.ClusterBlock;
+import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.Index;
@@ -48,6 +49,8 @@ public class CreateIndexClusterStateUpdateRequest extends ClusterStateUpdateRequ
 
     private boolean performReroute = true;
 
+    private ComposableIndexTemplate matchingTemplate;
+
     public CreateIndexClusterStateUpdateRequest(String cause, String index, String providedName) {
         this.cause = cause;
         this.index = index;
@@ -186,6 +189,21 @@ public CreateIndexClusterStateUpdateRequest performReroute(boolean performRerout
         return this;
     }
 
+    /**
+     * @return The composable index template that matches with the index that will be cretaed by this request.
+     */
+    public ComposableIndexTemplate matchingTemplate() {
+        return matchingTemplate;
+    }
+
+    /**
+     * Sets the composable index template that matches with index that will be created by this request.
+     */
+    public CreateIndexClusterStateUpdateRequest setMatchingTemplate(ComposableIndexTemplate matchingTemplate) {
+        this.matchingTemplate = matchingTemplate;
+        return this;
+    }
+
     @Override
     public String toString() {
         return "CreateIndexClusterStateUpdateRequest{"
@@ -217,6 +235,8 @@ public String toString() {
             + waitForActiveShards
             + ", systemDataStreamDescriptor="
             + systemDataStreamDescriptor
+            + ", matchingTemplate="
+            + matchingTemplate
             + '}';
     }
 }

server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java

@@ -14,6 +14,7 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.AliasAction;
 import org.elasticsearch.cluster.metadata.AliasMetadata;
+import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
 import org.elasticsearch.cluster.metadata.DataStream;
 import org.elasticsearch.cluster.metadata.IndexAbstraction;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
@@ -242,15 +243,17 @@ private RolloverResult rolloverDataStream(
             );
         }
 
+        final ComposableIndexTemplate templateV2;
         final SystemDataStreamDescriptor systemDataStreamDescriptor;
         if (dataStream.isSystem() == false) {
             systemDataStreamDescriptor = null;
-            lookupTemplateForDataStream(dataStreamName, currentState.metadata());
+            templateV2 = lookupTemplateForDataStream(dataStreamName, currentState.metadata());
         } else {
             systemDataStreamDescriptor = systemIndices.findMatchingDataStreamDescriptor(dataStreamName);
             if (systemDataStreamDescriptor == null) {
                 throw new IllegalArgumentException("no system data stream descriptor found for data stream [" + dataStreamName + "]");
             }
+            templateV2 = systemDataStreamDescriptor.getComposableIndexTemplate();
         }
 
         final DataStream ds = dataStream.getDataStream();
@@ -270,6 +273,7 @@ private RolloverResult rolloverDataStream(
             systemDataStreamDescriptor,
             now
         );
+        createIndexClusterStateRequest.setMatchingTemplate(templateV2);
         ClusterState newState = createIndexService.applyCreateIndexRequest(
             currentState,
             createIndexClusterStateRequest,

server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java

@@ -213,6 +213,7 @@ public static Template resolveTemplate(
             Settings result = provider.getAdditionalIndexSettings(
                 indexName,
                 template.getDataStreamTemplate() != null ? indexName : null,
+                template.getDataStreamTemplate() != null ? template.getDataStreamTemplate().getIndexMode() : null,
                 simulatedState.getMetadata(),
                 System.currentTimeMillis(),
                 templateSettings

server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java

@@ -17,6 +17,7 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.core.Nullable;
+import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.index.mapper.DataStreamTimestampFieldMapper;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.xcontent.ConstructingObjectParser;
@@ -305,28 +306,35 @@ public static class DataStreamTemplate implements Writeable, ToXContentObject {
 
         private static final ParseField HIDDEN = new ParseField("hidden");
         private static final ParseField ALLOW_CUSTOM_ROUTING = new ParseField("allow_custom_routing");
+        private static final ParseField INDEX_MODE = new ParseField("index_mode");
 
         public static final ConstructingObjectParser<DataStreamTemplate, Void> PARSER = new ConstructingObjectParser<>(
             "data_stream_template",
             false,
-            a -> new DataStreamTemplate(a[0] != null && (boolean) a[0], a[1] != null && (boolean) a[1])
+            args -> {
+                IndexMode indexMode = args[2] != null ? IndexMode.fromString((String) args[2]) : null;
+                return new DataStreamTemplate(args[0] != null && (boolean) args[0], args[1] != null && (boolean) args[1], indexMode);
+            }
         );
 
         static {
             PARSER.declareBoolean(ConstructingObjectParser.optionalConstructorArg(), HIDDEN);
             PARSER.declareBoolean(ConstructingObjectParser.optionalConstructorArg(), ALLOW_CUSTOM_ROUTING);
+            PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), INDEX_MODE);
         }
 
         private final boolean hidden;
         private final boolean allowCustomRouting;
+        private final IndexMode indexMode;
 
         public DataStreamTemplate() {
-            this(false, false);
+            this(false, false, null);
         }
 
-        public DataStreamTemplate(boolean hidden, boolean allowCustomRouting) {
+        public DataStreamTemplate(boolean hidden, boolean allowCustomRouting, IndexMode indexMode) {
             this.hidden = hidden;
             this.allowCustomRouting = allowCustomRouting;
+            this.indexMode = indexMode;
         }
 
         DataStreamTemplate(StreamInput in) throws IOException {
@@ -336,6 +344,11 @@ public DataStreamTemplate(boolean hidden, boolean allowCustomRouting) {
             } else {
                 allowCustomRouting = false;
             }
+            if (in.getVersion().onOrAfter(Version.V_8_1_0)) {
+                indexMode = in.readOptionalEnum(IndexMode.class);
+            } else {
+                indexMode = null;
+            }
         }
 
         public String getTimestampField() {
@@ -368,19 +381,30 @@ public boolean isAllowCustomRouting() {
             return allowCustomRouting;
         }
 
+        @Nullable
+        public IndexMode getIndexMode() {
+            return indexMode;
+        }
+
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             out.writeBoolean(hidden);
             if (out.getVersion().onOrAfter(Version.V_8_0_0)) {
                 out.writeBoolean(allowCustomRouting);
             }
+            if (out.getVersion().onOrAfter(Version.V_8_1_0)) {
+                out.writeOptionalEnum(indexMode);
+            }
         }
 
         @Override
         public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
             builder.startObject();
             builder.field("hidden", hidden);
             builder.field(ALLOW_CUSTOM_ROUTING.getPreferredName(), allowCustomRouting);
+            if (indexMode != null) {
+                builder.field(INDEX_MODE.getPreferredName(), indexMode);
+            }
             builder.endObject();
             return builder;
         }
@@ -390,12 +414,12 @@ public boolean equals(Object o) {
             if (this == o) return true;
             if (o == null || getClass() != o.getClass()) return false;
             DataStreamTemplate that = (DataStreamTemplate) o;
-            return hidden == that.hidden && allowCustomRouting == that.allowCustomRouting;
+            return hidden == that.hidden && allowCustomRouting == that.allowCustomRouting && indexMode == that.indexMode;
         }
 
         @Override
         public int hashCode() {
-            return Objects.hash(hidden, allowCustomRouting);
+            return Objects.hash(hidden, allowCustomRouting, indexMode);
         }
     }