From 97bd40fd2906038fa63d7042aa09a7b6c54c1575 Mon Sep 17 00:00:00 2001 From: Christian Groschupp Date: Thu, 22 Feb 2024 11:55:05 +0100 Subject: [PATCH 1/4] add datastream support Signed-off-by: Christian Groschupp --- .../api/v1/opensearch_index_types.go | 10 ++++++ .../api/v1/opensearch_indextemplate_types.go | 3 ++ .../api/v1/zz_generated.deepcopy.go | 36 +++++++++++++++++++ ...ch.opster.io_opensearchindextemplates.yaml | 13 +++++++ .../opensearch-gateway/requests/Templates.go | 9 +++++ opensearch-operator/pkg/helpers/translate.go | 14 ++++++++ 6 files changed, 85 insertions(+) diff --git a/opensearch-operator/api/v1/opensearch_index_types.go b/opensearch-operator/api/v1/opensearch_index_types.go index 99f3a93e..c2f2838d 100644 --- a/opensearch-operator/api/v1/opensearch_index_types.go +++ b/opensearch-operator/api/v1/opensearch_index_types.go @@ -4,6 +4,16 @@ import ( apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" ) +type OpensearchDatastreamTimestampFieldSpec struct { + // Name of the field that are used for the DataStream + Name string `json:"name"` +} + +type OpensearchDatastreamSpec struct { + // TimestampField for dataStream + TimestampField OpensearchDatastreamTimestampFieldSpec `json:"timestamp_field,omitempty"` +} + // Describes the specs of an index type OpensearchIndexSpec struct { // Configuration options for the index diff --git a/opensearch-operator/api/v1/opensearch_indextemplate_types.go b/opensearch-operator/api/v1/opensearch_indextemplate_types.go index 53e1fd9d..cb3d30ec 100644 --- a/opensearch-operator/api/v1/opensearch_indextemplate_types.go +++ b/opensearch-operator/api/v1/opensearch_indextemplate_types.go @@ -48,6 +48,9 @@ type OpensearchIndexTemplateSpec struct { // Array of wildcard expressions used to match the names of indices during creation IndexPatterns []string `json:"indexPatterns"` + // The dataStream config that should be applied + DataStream *OpensearchDatastreamSpec `json:"dataStream,omitempty"` + // The template that should be applied Template OpensearchIndexSpec `json:"template,omitempty"` diff --git a/opensearch-operator/api/v1/zz_generated.deepcopy.go b/opensearch-operator/api/v1/zz_generated.deepcopy.go index c1b5cf33..96a73efd 100644 --- a/opensearch-operator/api/v1/zz_generated.deepcopy.go +++ b/opensearch-operator/api/v1/zz_generated.deepcopy.go @@ -1467,6 +1467,37 @@ func (in *OpensearchComponentTemplateStatus) DeepCopy() *OpensearchComponentTemp return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *OpensearchDatastreamSpec) DeepCopyInto(out *OpensearchDatastreamSpec) { + *out = *in + out.TimestampField = in.TimestampField +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OpensearchDatastreamSpec. +func (in *OpensearchDatastreamSpec) DeepCopy() *OpensearchDatastreamSpec { + if in == nil { + return nil + } + out := new(OpensearchDatastreamSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *OpensearchDatastreamTimestampFieldSpec) DeepCopyInto(out *OpensearchDatastreamTimestampFieldSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OpensearchDatastreamTimestampFieldSpec. +func (in *OpensearchDatastreamTimestampFieldSpec) DeepCopy() *OpensearchDatastreamTimestampFieldSpec { + if in == nil { + return nil + } + out := new(OpensearchDatastreamTimestampFieldSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *OpensearchISMPolicyStatus) DeepCopyInto(out *OpensearchISMPolicyStatus) { *out = *in @@ -1612,6 +1643,11 @@ func (in *OpensearchIndexTemplateSpec) DeepCopyInto(out *OpensearchIndexTemplate *out = make([]string, len(*in)) copy(*out, *in) } + if in.DataStream != nil { + in, out := &in.DataStream, &out.DataStream + *out = new(OpensearchDatastreamSpec) + **out = **in + } in.Template.DeepCopyInto(&out.Template) if in.ComposedOf != nil { in, out := &in.ComposedOf, &out.ComposedOf diff --git a/opensearch-operator/config/crd/bases/opensearch.opster.io_opensearchindextemplates.yaml b/opensearch-operator/config/crd/bases/opensearch.opster.io_opensearchindextemplates.yaml index d6c534ad..db057514 100644 --- a/opensearch-operator/config/crd/bases/opensearch.opster.io_opensearchindextemplates.yaml +++ b/opensearch-operator/config/crd/bases/opensearch.opster.io_opensearchindextemplates.yaml @@ -51,6 +51,19 @@ spec: items: type: string type: array + dataStream: + description: The dataStream config that should be applied + properties: + timestamp_field: + description: TimestampField for dataStream + properties: + name: + description: Name of the field that are used for the DataStream + type: string + required: + - name + type: object + type: object indexPatterns: description: Array of wildcard expressions used to match the names of indices during creation diff --git a/opensearch-operator/opensearch-gateway/requests/Templates.go b/opensearch-operator/opensearch-gateway/requests/Templates.go index 93cfaeb8..f851922a 100644 --- a/opensearch-operator/opensearch-gateway/requests/Templates.go +++ b/opensearch-operator/opensearch-gateway/requests/Templates.go @@ -4,6 +4,7 @@ import apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1 type IndexTemplate struct { IndexPatterns []string `json:"index_patterns"` + DataStream *Datastream `json:"data_stream,omitempty"` Template Index `json:"template,omitempty"` ComposedOf []string `json:"composed_of,omitempty"` Priority int `json:"priority,omitempty"` @@ -23,6 +24,14 @@ type Index struct { Aliases map[string]IndexAlias `json:"aliases,omitempty"` } +type DatastreamTimestampFieldSpec struct { + Name string `json:"name"` +} + +type Datastream struct { + TimestampField *DatastreamTimestampFieldSpec `json:"timestamp_field,omitempty"` +} + type IndexAlias struct { Index string `json:"index,omitempty"` Alias string `json:"alias,omitempty"` diff --git a/opensearch-operator/pkg/helpers/translate.go b/opensearch-operator/pkg/helpers/translate.go index a279a172..ed48c86a 100644 --- a/opensearch-operator/pkg/helpers/translate.go +++ b/opensearch-operator/pkg/helpers/translate.go @@ -9,6 +9,7 @@ import ( func TranslateIndexTemplateToRequest(spec v1.OpensearchIndexTemplateSpec) requests.IndexTemplate { request := requests.IndexTemplate{ IndexPatterns: spec.IndexPatterns, + DataStream: TranslateDatastreamToRequest(spec.DataStream), Template: TranslateIndexToRequest(spec.Template), Priority: spec.Priority, Version: spec.Version, @@ -36,6 +37,19 @@ func TranslateComponentTemplateToRequest(spec v1.OpensearchComponentTemplateSpec return request } +// TranslateDatastreamToRequest rewrites the CRD format to the gateway format +func TranslateDatastreamToRequest(spec *v1.OpensearchDatastreamSpec) *requests.Datastream { + if spec == nil { + return nil + } + request := requests.Datastream{} + if spec.TimestampField.Name != "" { + request.TimestampField = &requests.DatastreamTimestampFieldSpec{Name: spec.TimestampField.Name} + } + + return &request +} + // TranslateIndexToRequest rewrites the CRD format to the gateway format func TranslateIndexToRequest(spec v1.OpensearchIndexSpec) requests.Index { aliases := make(map[string]requests.IndexAlias) From a011e677065e1a90edb680dff4506659b2c0a938 Mon Sep 17 00:00:00 2001 From: Christian Groschupp Date: Fri, 5 Apr 2024 08:59:52 +0200 Subject: [PATCH 2/4] add datastream test Signed-off-by: Christian Groschupp --- opensearch-operator/pkg/reconcilers/indextemplate_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/opensearch-operator/pkg/reconcilers/indextemplate_test.go b/opensearch-operator/pkg/reconcilers/indextemplate_test.go index 43849524..2d2a4606 100644 --- a/opensearch-operator/pkg/reconcilers/indextemplate_test.go +++ b/opensearch-operator/pkg/reconcilers/indextemplate_test.go @@ -62,6 +62,7 @@ var _ = Describe("indextemplate reconciler", func() { Priority: 0, Version: 0, Meta: &apiextensionsv1.JSON{}, + DataStream: &opsterv1.OpensearchDatastreamSpec{TimestampField: opsterv1.OpensearchDatastreamTimestampFieldSpec{Name: "@mytimestamp"}}, }, } @@ -285,6 +286,7 @@ var _ = Describe("indextemplate reconciler", func() { Priority: 0, Version: 0, Meta: &apiextensionsv1.JSON{}, + DataStream: &requests.Datastream{TimestampField: &requests.DatastreamTimestampFieldSpec{Name: "@mytimestamp"}}, }, } From 2973c9f49f30ace41afe612cf13778081ad7aeea Mon Sep 17 00:00:00 2001 From: Christian Groschupp Date: Fri, 5 Apr 2024 10:35:58 +0200 Subject: [PATCH 3/4] copy opensearchindextemplates crd to helm chart Signed-off-by: Christian Groschupp --- ...ch.opster.io_opensearchindextemplates.yaml | 64 ++++++++++++------- 1 file changed, 42 insertions(+), 22 deletions(-) diff --git a/charts/opensearch-operator/files/opensearch.opster.io_opensearchindextemplates.yaml b/charts/opensearch-operator/files/opensearch.opster.io_opensearchindextemplates.yaml index 4cfc668a..db057514 100644 --- a/charts/opensearch-operator/files/opensearch.opster.io_opensearchindextemplates.yaml +++ b/charts/opensearch-operator/files/opensearch.opster.io_opensearchindextemplates.yaml @@ -3,8 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.10.0 - creationTimestamp: null + controller-gen.kubebuilder.io/version: v0.14.0 name: opensearchindextemplates.opensearch.opster.io spec: group: opensearch.opster.io @@ -24,14 +23,19 @@ spec: templates API properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object @@ -41,12 +45,25 @@ spec: description: Optional user metadata about the index template x-kubernetes-preserve-unknown-fields: true composedOf: - description: An ordered list of component template names. Component - templates are merged in the order specified, meaning that the last - component template specified has the highest precedence + description: |- + An ordered list of component template names. Component templates are merged in the order specified, + meaning that the last component template specified has the highest precedence items: type: string type: array + dataStream: + description: The dataStream config that should be applied + properties: + timestamp_field: + description: TimestampField for dataStream + properties: + name: + description: Name of the field that are used for the DataStream + type: string + required: + - name + type: object + type: object indexPatterns: description: Array of wildcard expressions used to match the names of indices during creation @@ -57,19 +74,22 @@ spec: description: The name of the index template. Defaults to metadata.name type: string opensearchCluster: - description: LocalObjectReference contains enough information to let - you locate the referenced object inside the same namespace. + description: |- + LocalObjectReference contains enough information to let you locate the + referenced object inside the same namespace. properties: name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? type: string type: object x-kubernetes-map-type: atomic priority: - description: Priority to determine index template precedence when - a new data stream or index is created. The index template with the - highest priority is chosen + description: |- + Priority to determine index template precedence when a new data stream or index is created. + The index template with the highest priority is chosen type: integer template: description: The template that should be applied @@ -123,10 +143,10 @@ spec: description: Name of the currently managed index template type: string managedCluster: - description: UID is a type that holds unique ID values, including - UUIDs. Because we don't ONLY use UUIDs, this is an alias to string. Being - a type captures intent and helps make sure that UIDs and names do - not get conflated. + description: |- + UID is a type that holds unique ID values, including UUIDs. Because we + don't ONLY use UUIDs, this is an alias to string. Being a type captures + intent and helps make sure that UIDs and names do not get conflated. type: string reason: type: string From a058ed7fa246fa381105b748f2c25bc2a53a8e53 Mon Sep 17 00:00:00 2001 From: Christian Groschupp Date: Fri, 5 Apr 2024 13:59:23 +0200 Subject: [PATCH 4/4] change TimestampField to camelCase --- opensearch-operator/api/v1/opensearch_index_types.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opensearch-operator/api/v1/opensearch_index_types.go b/opensearch-operator/api/v1/opensearch_index_types.go index c2f2838d..37c9b94b 100644 --- a/opensearch-operator/api/v1/opensearch_index_types.go +++ b/opensearch-operator/api/v1/opensearch_index_types.go @@ -11,7 +11,7 @@ type OpensearchDatastreamTimestampFieldSpec struct { type OpensearchDatastreamSpec struct { // TimestampField for dataStream - TimestampField OpensearchDatastreamTimestampFieldSpec `json:"timestamp_field,omitempty"` + TimestampField OpensearchDatastreamTimestampFieldSpec `json:"timestampField,omitempty"` } // Describes the specs of an index