add datastream support (#735)

### Description Add DataStream support to IndexTemplate. --------- Signed-off-by: Christian Groschupp <[email protected]> Co-authored-by: Christian Groschupp <[email protected]>
opensearch-project · Apr 5, 2024 · 9fff407 · 9fff407
1 parent 4b0fe24
commit 9fff407
Show file tree

Hide file tree

Showing 8 changed files with 129 additions and 22 deletions.
diff --git a/charts/opensearch-operator/files/opensearch.opster.io_opensearchindextemplates.yaml b/charts/opensearch-operator/files/opensearch.opster.io_opensearchindextemplates.yaml
@@ -3,8 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.10.0
-  creationTimestamp: null
+    controller-gen.kubebuilder.io/version: v0.14.0
   name: opensearchindextemplates.opensearch.opster.io
 spec:
   group: opensearch.opster.io
@@ -24,14 +23,19 @@ spec:
           templates API
         properties:
           apiVersion:
-            description: 'APIVersion defines the versioned schema of this representation
-              of an object. Servers should convert recognized schemas to the latest
-              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            description: |-
+              APIVersion defines the versioned schema of this representation of an object.
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
             type: string
           kind:
-            description: 'Kind is a string value representing the REST resource this
-              object represents. Servers may infer this from the endpoint the client
-              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            description: |-
+              Kind is a string value representing the REST resource this object represents.
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
             type: string
           metadata:
             type: object
@@ -41,12 +45,25 @@ spec:
                 description: Optional user metadata about the index template
                 x-kubernetes-preserve-unknown-fields: true
               composedOf:
-                description: An ordered list of component template names. Component
-                  templates are merged in the order specified, meaning that the last
-                  component template specified has the highest precedence
+                description: |-
+                  An ordered list of component template names. Component templates are merged in the order specified,
+                  meaning that the last component template specified has the highest precedence
                 items:
                   type: string
                 type: array
+              dataStream:
+                description: The dataStream config that should be applied
+                properties:
+                  timestamp_field:
+                    description: TimestampField for dataStream
+                    properties:
+                      name:
+                        description: Name of the field that are used for the DataStream
+                        type: string
+                    required:
+                    - name
+                    type: object
+                type: object
               indexPatterns:
                 description: Array of wildcard expressions used to match the names
                   of indices during creation
@@ -57,19 +74,22 @@ spec:
                 description: The name of the index template. Defaults to metadata.name
                 type: string
               opensearchCluster:
-                description: LocalObjectReference contains enough information to let
-                  you locate the referenced object inside the same namespace.
+                description: |-
+                  LocalObjectReference contains enough information to let you locate the
+                  referenced object inside the same namespace.
                 properties:
                   name:
-                    description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
-                      TODO: Add other useful fields. apiVersion, kind, uid?'
+                    description: |-
+                      Name of the referent.
+                      More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+                      TODO: Add other useful fields. apiVersion, kind, uid?
                     type: string
                 type: object
                 x-kubernetes-map-type: atomic
               priority:
-                description: Priority to determine index template precedence when
-                  a new data stream or index is created. The index template with the
-                  highest priority is chosen
+                description: |-
+                  Priority to determine index template precedence when a new data stream or index is created.
+                  The index template with the highest priority is chosen
                 type: integer
               template:
                 description: The template that should be applied
@@ -123,10 +143,10 @@ spec:
                 description: Name of the currently managed index template
                 type: string
               managedCluster:
-                description: UID is a type that holds unique ID values, including
-                  UUIDs.  Because we don't ONLY use UUIDs, this is an alias to string.  Being
-                  a type captures intent and helps make sure that UIDs and names do
-                  not get conflated.
+                description: |-
+                  UID is a type that holds unique ID values, including UUIDs.  Because we
+                  don't ONLY use UUIDs, this is an alias to string.  Being a type captures
+                  intent and helps make sure that UIDs and names do not get conflated.
                 type: string
               reason:
                 type: string

diff --git a/opensearch-operator/api/v1/opensearch_index_types.go b/opensearch-operator/api/v1/opensearch_index_types.go
@@ -4,6 +4,16 @@ import (
 	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 )
 
+type OpensearchDatastreamTimestampFieldSpec struct {
+	// Name of the field that are used for the DataStream
+	Name string `json:"name"`
+}
+
+type OpensearchDatastreamSpec struct {
+	// TimestampField for dataStream
+	TimestampField OpensearchDatastreamTimestampFieldSpec `json:"timestampField,omitempty"`
+}
+
 // Describes the specs of an index
 type OpensearchIndexSpec struct {
 	// Configuration options for the index

diff --git a/opensearch-operator/api/v1/opensearch_indextemplate_types.go b/opensearch-operator/api/v1/opensearch_indextemplate_types.go
@@ -48,6 +48,9 @@ type OpensearchIndexTemplateSpec struct {
 	// Array of wildcard expressions used to match the names of indices during creation
 	IndexPatterns []string `json:"indexPatterns"`
 
+	// The dataStream config that should be applied
+	DataStream *OpensearchDatastreamSpec `json:"dataStream,omitempty"`
+
 	// The template that should be applied
 	Template OpensearchIndexSpec `json:"template,omitempty"`
 

diff --git a/opensearch-operator/api/v1/zz_generated.deepcopy.go b/opensearch-operator/api/v1/zz_generated.deepcopy.go
diff --git a/opensearch-operator/config/crd/bases/opensearch.opster.io_opensearchindextemplates.yaml b/opensearch-operator/config/crd/bases/opensearch.opster.io_opensearchindextemplates.yaml
@@ -51,6 +51,19 @@ spec:
                 items:
                   type: string
                 type: array
+              dataStream:
+                description: The dataStream config that should be applied
+                properties:
+                  timestamp_field:
+                    description: TimestampField for dataStream
+                    properties:
+                      name:
+                        description: Name of the field that are used for the DataStream
+                        type: string
+                    required:
+                    - name
+                    type: object
+                type: object
               indexPatterns:
                 description: Array of wildcard expressions used to match the names
                   of indices during creation

diff --git a/opensearch-operator/opensearch-gateway/requests/Templates.go b/opensearch-operator/opensearch-gateway/requests/Templates.go
@@ -4,6 +4,7 @@ import apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1
 
 type IndexTemplate struct {
 	IndexPatterns []string              `json:"index_patterns"`
+	DataStream    *Datastream           `json:"data_stream,omitempty"`
 	Template      Index                 `json:"template,omitempty"`
 	ComposedOf    []string              `json:"composed_of,omitempty"`
 	Priority      int                   `json:"priority,omitempty"`
@@ -23,6 +24,14 @@ type Index struct {
 	Aliases  map[string]IndexAlias `json:"aliases,omitempty"`
 }
 
+type DatastreamTimestampFieldSpec struct {
+	Name string `json:"name"`
+}
+
+type Datastream struct {
+	TimestampField *DatastreamTimestampFieldSpec `json:"timestamp_field,omitempty"`
+}
+
 type IndexAlias struct {
 	Index        string                `json:"index,omitempty"`
 	Alias        string                `json:"alias,omitempty"`

diff --git a/opensearch-operator/pkg/helpers/translate.go b/opensearch-operator/pkg/helpers/translate.go
@@ -9,6 +9,7 @@ import (
 func TranslateIndexTemplateToRequest(spec v1.OpensearchIndexTemplateSpec) requests.IndexTemplate {
 	request := requests.IndexTemplate{
 		IndexPatterns: spec.IndexPatterns,
+		DataStream:    TranslateDatastreamToRequest(spec.DataStream),
 		Template:      TranslateIndexToRequest(spec.Template),
 		Priority:      spec.Priority,
 		Version:       spec.Version,
@@ -36,6 +37,19 @@ func TranslateComponentTemplateToRequest(spec v1.OpensearchComponentTemplateSpec
 	return request
 }
 
+// TranslateDatastreamToRequest rewrites the CRD format to the gateway format
+func TranslateDatastreamToRequest(spec *v1.OpensearchDatastreamSpec) *requests.Datastream {
+	if spec == nil {
+		return nil
+	}
+	request := requests.Datastream{}
+	if spec.TimestampField.Name != "" {
+		request.TimestampField = &requests.DatastreamTimestampFieldSpec{Name: spec.TimestampField.Name}
+	}
+
+	return &request
+}
+
 // TranslateIndexToRequest rewrites the CRD format to the gateway format
 func TranslateIndexToRequest(spec v1.OpensearchIndexSpec) requests.Index {
 	aliases := make(map[string]requests.IndexAlias)

diff --git a/opensearch-operator/pkg/reconcilers/indextemplate_test.go b/opensearch-operator/pkg/reconcilers/indextemplate_test.go
@@ -62,6 +62,7 @@ var _ = Describe("indextemplate reconciler", func() {
 				Priority:   0,
 				Version:    0,
 				Meta:       &apiextensionsv1.JSON{},
+				DataStream: &opsterv1.OpensearchDatastreamSpec{TimestampField: opsterv1.OpensearchDatastreamTimestampFieldSpec{Name: "@mytimestamp"}},
 			},
 		}
 
@@ -285,6 +286,7 @@ var _ = Describe("indextemplate reconciler", func() {
 							Priority:   0,
 							Version:    0,
 							Meta:       &apiextensionsv1.JSON{},
+							DataStream: &requests.Datastream{TimestampField: &requests.DatastreamTimestampFieldSpec{Name: "@mytimestamp"}},
 						},
 					}