From 3450a26cda13bc225439477a456a1a85f397330e Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Tue, 9 Jul 2024 23:54:52 +0000 Subject: [PATCH 1/3] chore: stage DataformRepository apis Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- apis/dataform/v1alpha1/doc.go | 17 ++ apis/dataform/v1alpha1/groupversion_info.go | 40 ++++ apis/dataform/v1alpha1/repository_types.go | 114 +++++++++++ .../v1alpha1/zz_generated.deepcopy.go | 180 ++++++++++++++++++ 4 files changed, 351 insertions(+) create mode 100644 apis/dataform/v1alpha1/doc.go create mode 100644 apis/dataform/v1alpha1/groupversion_info.go create mode 100644 apis/dataform/v1alpha1/repository_types.go create mode 100644 apis/dataform/v1alpha1/zz_generated.deepcopy.go diff --git a/apis/dataform/v1alpha1/doc.go b/apis/dataform/v1alpha1/doc.go new file mode 100644 index 0000000000..70d8196be8 --- /dev/null +++ b/apis/dataform/v1alpha1/doc.go @@ -0,0 +1,17 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +kcc:proto=google.cloud.dataform.v1beta1 + +package v1alpha1 \ No newline at end of file diff --git a/apis/dataform/v1alpha1/groupversion_info.go b/apis/dataform/v1alpha1/groupversion_info.go new file mode 100644 index 0000000000..b9a9e042ef --- /dev/null +++ b/apis/dataform/v1alpha1/groupversion_info.go @@ -0,0 +1,40 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package v1alpha1 contains API Schema definitions for the dataform v1alpha1 API group +// +kubebuilder:object:generate=true +// +groupName=dataform.cnrm.cloud.google.com +package v1alpha1 + +import ( + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/scheme" +) + +var ( + // GroupVersion is group version used to register these objects + GroupVersion = schema.GroupVersion{Group: "dataform.cnrm.cloud.google.com", Version: "v1alpha1"} + + // SchemeBuilder is used to add go types to the GroupVersionKind scheme + SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} + + // AddToScheme adds the types in this group-version to the given scheme. + AddToScheme = SchemeBuilder.AddToScheme + + GroupVersionKind = schema.GroupVersionKind{ + Group: GroupVersion.Group, + Version: GroupVersion.Version, + Kind: "DataformRepository", + } +) diff --git a/apis/dataform/v1alpha1/repository_types.go b/apis/dataform/v1alpha1/repository_types.go new file mode 100644 index 0000000000..b34a92d3dc --- /dev/null +++ b/apis/dataform/v1alpha1/repository_types.go @@ -0,0 +1,114 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1alpha1 + +import ( + refv1beta1 "github.com/GoogleCloudPlatform/k8s-config-connector/apis/refs/v1beta1" + "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/clients/generated/apis/k8s/v1alpha1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + + +type RepositoryGitRemoteSettings struct { + /* The name of the Secret Manager secret version to use as an authentication token for Git operations. Must be in the format projects/* /secrets/* /versions/*. */ + AuthenticationTokenSecretVersion string `json:"authenticationTokenSecretVersion"` + + /* The Git remote's default branch name. */ + DefaultBranch string `json:"defaultBranch"` + + /* Indicates the status of the Git access token. https://cloud.google.com/dataform/reference/rest/v1beta1/projects.locations.repositories#TokenStatus. */ + // +optional + TokenStatus *string `json:"tokenStatus,omitempty"` + + /* The Git remote's URL. */ + Url string `json:"url"` +} + +type RepositoryWorkspaceCompilationOverrides struct { + /* Optional. The default database (Google Cloud project ID). */ + // +optional + DefaultDatabase *string `json:"defaultDatabase,omitempty"` + + /* Optional. The suffix that should be appended to all schema (BigQuery dataset ID) names. */ + // +optional + SchemaSuffix *string `json:"schemaSuffix,omitempty"` + + /* Optional. The prefix that should be prepended to all table names. */ + // +optional + TablePrefix *string `json:"tablePrefix,omitempty"` +} + +type DataformRepositorySpec struct { + /* Optional. If set, configures this repository to be linked to a Git remote. */ + // +optional + GitRemoteSettings *RepositoryGitRemoteSettings `json:"gitRemoteSettings,omitempty"` + + /* The project that this resource belongs to. */ + // +required + ProjectRef *refv1beta1.ProjectRef `json:"projectRef"` + + /* Immutable. A reference to the region. */ + Region string `json:"region"` + + /* Immutable. Optional. The name of the resource. Used for creation and acquisition. When unset, the value of `metadata.name` is used as the default. */ + // +optional + ResourceID *string `json:"resourceID,omitempty"` + + /* Optional. If set, fields of workspaceCompilationOverrides override the default compilation settings that are specified in dataform.json when creating workspace-scoped compilation results. */ + // +optional + WorkspaceCompilationOverrides *RepositoryWorkspaceCompilationOverrides `json:"workspaceCompilationOverrides,omitempty"` +} + +type DataformRepositoryStatus struct { + /* Conditions represent the latest available observations of the + DataformRepository's current state. */ + Conditions []v1alpha1.Condition `json:"conditions,omitempty"` + /* ObservedGeneration is the generation of the resource that was most recently observed by the Config Connector controller. If this is equal to metadata.generation, then that means that the current reported status reflects the most recent desired state of the resource. */ + // +optional + ObservedGeneration *int64 `json:"observedGeneration,omitempty"` +} + +// +genclient +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +kubebuilder:resource:categories=gcp,shortName=gcpdataformrepository;gcpdataformrepositories +// +kubebuilder:subresource:status +// +kubebuilder:metadata:labels="cnrm.cloud.google.com/managed-by-kcc=true";"cnrm.cloud.google.com/stability-level=alpha";"cnrm.cloud.google.com/system=true";"cnrm.cloud.google.com/tf2crd=true" +// +kubebuilder:printcolumn:name="Age",JSONPath=".metadata.creationTimestamp",type="date" +// +kubebuilder:printcolumn:name="Ready",JSONPath=".status.conditions[?(@.type=='Ready')].status",type="string",description="When 'True', the most recent reconcile of the resource succeeded" +// +kubebuilder:printcolumn:name="Status",JSONPath=".status.conditions[?(@.type=='Ready')].reason",type="string",description="The reason for the value in 'Ready'" +// +kubebuilder:printcolumn:name="Status Age",JSONPath=".status.conditions[?(@.type=='Ready')].lastTransitionTime",type="date",description="The last transition time for the value in 'Status'" + +// DataformRepository is the Schema for the dataform API +// +k8s:openapi-gen=true +type DataformRepository struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec DataformRepositorySpec `json:"spec,omitempty"` + Status DataformRepositoryStatus `json:"status,omitempty"` +} + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// DataformRepositoryList contains a list of DataformRepository +type DataformRepositoryList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []DataformRepository `json:"items"` +} + +func init() { + SchemeBuilder.Register(&DataformRepository{}, &DataformRepositoryList{}) +} diff --git a/apis/dataform/v1alpha1/zz_generated.deepcopy.go b/apis/dataform/v1alpha1/zz_generated.deepcopy.go new file mode 100644 index 0000000000..ea885b0b01 --- /dev/null +++ b/apis/dataform/v1alpha1/zz_generated.deepcopy.go @@ -0,0 +1,180 @@ +//go:build !ignore_autogenerated + +// Code generated by controller-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + "github.com/GoogleCloudPlatform/k8s-config-connector/apis/refs/v1beta1" + k8sv1alpha1 "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/clients/generated/apis/k8s/v1alpha1" + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DataformRepository) DeepCopyInto(out *DataformRepository) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DataformRepository. +func (in *DataformRepository) DeepCopy() *DataformRepository { + if in == nil { + return nil + } + out := new(DataformRepository) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *DataformRepository) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DataformRepositoryList) DeepCopyInto(out *DataformRepositoryList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]DataformRepository, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DataformRepositoryList. +func (in *DataformRepositoryList) DeepCopy() *DataformRepositoryList { + if in == nil { + return nil + } + out := new(DataformRepositoryList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *DataformRepositoryList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DataformRepositorySpec) DeepCopyInto(out *DataformRepositorySpec) { + *out = *in + if in.GitRemoteSettings != nil { + in, out := &in.GitRemoteSettings, &out.GitRemoteSettings + *out = new(RepositoryGitRemoteSettings) + (*in).DeepCopyInto(*out) + } + if in.ProjectRef != nil { + in, out := &in.ProjectRef, &out.ProjectRef + *out = new(v1beta1.ProjectRef) + **out = **in + } + if in.ResourceID != nil { + in, out := &in.ResourceID, &out.ResourceID + *out = new(string) + **out = **in + } + if in.WorkspaceCompilationOverrides != nil { + in, out := &in.WorkspaceCompilationOverrides, &out.WorkspaceCompilationOverrides + *out = new(RepositoryWorkspaceCompilationOverrides) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DataformRepositorySpec. +func (in *DataformRepositorySpec) DeepCopy() *DataformRepositorySpec { + if in == nil { + return nil + } + out := new(DataformRepositorySpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DataformRepositoryStatus) DeepCopyInto(out *DataformRepositoryStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]k8sv1alpha1.Condition, len(*in)) + copy(*out, *in) + } + if in.ObservedGeneration != nil { + in, out := &in.ObservedGeneration, &out.ObservedGeneration + *out = new(int64) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DataformRepositoryStatus. +func (in *DataformRepositoryStatus) DeepCopy() *DataformRepositoryStatus { + if in == nil { + return nil + } + out := new(DataformRepositoryStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RepositoryGitRemoteSettings) DeepCopyInto(out *RepositoryGitRemoteSettings) { + *out = *in + if in.TokenStatus != nil { + in, out := &in.TokenStatus, &out.TokenStatus + *out = new(string) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RepositoryGitRemoteSettings. +func (in *RepositoryGitRemoteSettings) DeepCopy() *RepositoryGitRemoteSettings { + if in == nil { + return nil + } + out := new(RepositoryGitRemoteSettings) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RepositoryWorkspaceCompilationOverrides) DeepCopyInto(out *RepositoryWorkspaceCompilationOverrides) { + *out = *in + if in.DefaultDatabase != nil { + in, out := &in.DefaultDatabase, &out.DefaultDatabase + *out = new(string) + **out = **in + } + if in.SchemaSuffix != nil { + in, out := &in.SchemaSuffix, &out.SchemaSuffix + *out = new(string) + **out = **in + } + if in.TablePrefix != nil { + in, out := &in.TablePrefix, &out.TablePrefix + *out = new(string) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RepositoryWorkspaceCompilationOverrides. +func (in *RepositoryWorkspaceCompilationOverrides) DeepCopy() *RepositoryWorkspaceCompilationOverrides { + if in == nil { + return nil + } + out := new(RepositoryWorkspaceCompilationOverrides) + in.DeepCopyInto(out) + return out +} From b243e1b11b145fcc18bb3c04d6d9afd1f64f5144 Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Tue, 9 Jul 2024 23:55:24 +0000 Subject: [PATCH 2/3] chore: add dataform for proto tool Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- dev/tools/proto-to-mapper/Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/dev/tools/proto-to-mapper/Makefile b/dev/tools/proto-to-mapper/Makefile index aa2829341b..f0fa6625a1 100644 --- a/dev/tools/proto-to-mapper/Makefile +++ b/dev/tools/proto-to-mapper/Makefile @@ -12,6 +12,7 @@ generate-pb: install-protoc-linux ./third_party/googleapis/google/logging/v2/*.proto \ ./third_party/googleapis/google/monitoring/dashboard/v1/*.proto \ ./third_party/googleapis/google/devtools/cloudbuild/*/*.proto \ + ./third_party/googleapis/google/cloud/dataform/*/*.proto \ -o build/googleapis.pb go run . -apis ../../../apis/ --api-packages github.com/GoogleCloudPlatform/apis From c00c82294ad88ced4c25fd40ae86b9dbb0bcacb0 Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Wed, 10 Jul 2024 19:48:44 +0000 Subject: [PATCH 3/3] chore: add exclusion for crd tool Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- dev/tasks/generate-crds | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/dev/tasks/generate-crds b/dev/tasks/generate-crds index 15c16d4499..c5b5b8b8fd 100755 --- a/dev/tasks/generate-crds +++ b/dev/tasks/generate-crds @@ -52,9 +52,28 @@ go run ./scripts/crd-tools reflow-descriptions --dir apis/config/crd/ # Copy generated CRDs to apis/config/crd/ # The crds directory is generated only as part of the release (currently) + +# excluded_resources are resources that are under development for a direct conversion +# we don't modify the CRD just yet for those but will in the future +excluded_resources=("dataformrepositorie") + cd ${REPO_ROOT} cd apis/config/crd/ for file in *.yaml; do + excluded=false + + for substring in "${excluded_resources[@]}"; do + if [[ "$file" == *"$substring"* ]]; then + excluded=true + break + fi + done + + if [ "$excluded" = true ]; then + echo "Skipping $file as its CRD conversion is excluded for now" + continue + fi + base_name=$(echo "$file" | sed 's/.yaml$//g') domain=$(echo "$base_name" | cut -d'_' -f1) resource=$(echo "$base_name" | cut -d'_' -f2)