From 1901cfb7a96c3d58cc84acd15f06b8962b22ff4b Mon Sep 17 00:00:00 2001 From: justinsb Date: Wed, 21 Feb 2024 09:57:36 -0500 Subject: [PATCH 1/5] powertool: change-state-into-spec can remove fields set by state-into-spec Using the field manager, we can detect and remove any fields that were set by state-into-spec. --- pkg/cli/cmd/root.go | 2 +- pkg/cli/powertools/changestateintospec/cmd.go | 204 ++++++++++++++++++ pkg/cli/powertools/cmd.go | 2 + pkg/cli/powertools/forcesetfield/cmd.go | 42 ++-- pkg/cli/powertools/kubecli/client.go | 23 +- pkg/cli/powertools/kubecli/options.go | 15 +- pkg/stateintospec/remove.go | 161 ++++++++++++++ 7 files changed, 418 insertions(+), 31 deletions(-) create mode 100644 pkg/cli/powertools/changestateintospec/cmd.go create mode 100644 pkg/stateintospec/remove.go diff --git a/pkg/cli/cmd/root.go b/pkg/cli/cmd/root.go index 303d99b506..d415056cd0 100644 --- a/pkg/cli/cmd/root.go +++ b/pkg/cli/cmd/root.go @@ -118,7 +118,7 @@ func isLegacyArgs() bool { return false } cmd, _, err := rootCmd.Find(args) - if err == nil && cmd.Args != nil { + if err == nil && cmd != nil { return false } for i := 0; i < len(args); { diff --git a/pkg/cli/powertools/changestateintospec/cmd.go b/pkg/cli/powertools/changestateintospec/cmd.go new file mode 100644 index 0000000000..f91a734865 --- /dev/null +++ b/pkg/cli/powertools/changestateintospec/cmd.go @@ -0,0 +1,204 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package changestateintospec + +import ( + "context" + "errors" + "fmt" + "io" + "strings" + + "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/cli/powertools/diffs" + "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/cli/powertools/kubecli" + "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/stateintospec" + "github.com/spf13/cobra" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/structured-merge-diff/v4/fieldpath" +) + +// Options configures the behaviour of the ChangeStateIntoSpec operation. +type Options struct { + kubecli.ClusterOptions + kubecli.ObjectOptions + + // NewStateIntoSpecAnnotation is the new value for the state-into-spec annotation + NewStateIntoSpecAnnotation string + + // FieldOwner is the field-manager owner value to use when making changes + FieldOwner string + + // DryRun is true if we should not actually make changes, just print the changes we would make + DryRun bool + + // KeepFields is a list of additional spec fields we should preserve + KeepFields []string +} + +func (o *Options) PopulateDefaults() { + o.ClusterOptions.PopulateDefaults() + o.ObjectOptions.PopulateDefaults() + + o.FieldOwner = "change-state-into-spec" + o.NewStateIntoSpecAnnotation = "absent" + o.DryRun = false +} + +func (o *Options) Validate() error { + var errs []error + + for _, keepField := range o.KeepFields { + if !strings.HasPrefix(keepField, ".spec.") { + errs = append(errs, fmt.Errorf("unexpected keep-field flag %q, should start with .spec. (e.g. `.spec.location`)", keepField)) + } + } + + return errors.Join(errs...) +} + +func AddCommand(parent *cobra.Command) { + var options Options + options.PopulateDefaults() + + cmd := &cobra.Command{ + Use: "change-state-into-spec", + Short: "Change the state-into-spec annotation on existing objects (experimental)", + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + + return Run(ctx, cmd.OutOrStdout(), cmd.ErrOrStderr(), options) + }, + Long: ` +change-state-into-spec updates the state-into-spec annotation on existing +objects in a cluster, primarily to convert from state-into-spec=merge +to state-into-spec=absent. + +Examples + +Change the StorageBucket "my-bucket" in the namespace "my-namespace" to +state-into-spec=absent. Fields that were set by state-into-spec=merge +will be removed. Run in dry-run: don't actually make changes, just print +the changes that would be made: + + config-connector powertools change-state-into-spec \ + --namespace=my-namespace --name=my-bucket \ + --kind=StorageBucket \ + --dry-run=true + +As before, but apply the changes. Additionally, use --keep-field +to ensure that .spec.location and .spec.resourceID are not removed. + + config-connector powertools change-state-into-spec \ + --namespace=my-namespace --name=my-bucket \ + --kind=StorageBucket \ + --keep-field=.spec.location \ + --keep-field=.spec.resourceID + +`, + Args: cobra.ArbitraryArgs, + } + + options.ObjectOptions.AddFlags(cmd) + options.ClusterOptions.AddFlags(cmd) + + cmd.Flags().StringVar(&options.NewStateIntoSpecAnnotation, "set", options.NewStateIntoSpecAnnotation, "New value for the state-into-spec annotation") + cmd.Flags().BoolVar(&options.DryRun, "dry-run", options.DryRun, "dry-run mode will not make changes, but only print the changes it would make") + cmd.Flags().StringSliceVar(&options.KeepFields, "keep-field", options.KeepFields, "Additional fields to preserve in the spec, even if they were set by state-into-spec=merge (example .spec.location)") + + parent.AddCommand(cmd) +} + +func Run(ctx context.Context, stdout io.Writer, stderr io.Writer, options Options) error { + // log := klog.FromContext(ctx) + + if err := options.Validate(); err != nil { + return err + } + + if options.ImpersonateUser == "" { + // Impersonate the KCC service account, which is allowed to make changes + options.ClusterOptions.Impersonate = &rest.ImpersonationConfig{ + UserName: "system:serviceaccount:cnrm-system:cnrm-controller-manager-" + options.Namespace, + Groups: []string{"system:serviceaccounts", "system:serviceaccounts:cnrm-system"}, + } + } else { + options.ClusterOptions.Impersonate = &rest.ImpersonationConfig{ + UserName: options.ImpersonateUser, + Groups: options.ImpersonateGroups, + } + } + kubeClient, err := kubecli.NewClient(ctx, options.ClusterOptions) + if err != nil { + return fmt.Errorf("creating client: %w", err) + } + + u, err := kubeClient.GetObject(ctx, options.ObjectOptions) + if err != nil { + return fmt.Errorf("getting object: %w", err) + } + + originalObject := u.DeepCopy() + + keepFields := make(map[string]bool) + // The exception to state-into-spec; we always want to preserve the resourceID field. + keepFields[".spec.resourceID"] = true + for _, keepField := range options.KeepFields { + keepFields[keepField] = true + } + + keepFieldFunc := func(fieldPath fieldpath.Path) bool { + fieldName := fieldPath.String() + return keepFields[fieldName] + } + + warnings, err := stateintospec.RemoveStateIntoSpecFields(ctx, u, keepFieldFunc) + if err != nil { + return err + } + + for _, warning := range warnings.Warnings { + fmt.Fprintf(stderr, "%v\n", warning.Message) + } + + annotations := u.GetAnnotations() + if annotations == nil { + annotations = make(map[string]string) + } + annotations["cnrm.cloud.google.com/state-into-spec"] = options.NewStateIntoSpecAnnotation + u.SetAnnotations(annotations) + + diff, err := diffs.BuildObjectDiff(originalObject, u) + if err != nil { + return fmt.Errorf("building object diff: %w", err) + } + + fmt.Fprintf(stdout, "\n\n") + printOpts := diffs.PrettyPrintOptions{PrintObjectInfo: true, Indent: " "} + diff.PrettyPrintTo(printOpts, stdout) + fmt.Fprintf(stdout, "\n\n") + + if options.DryRun { + fmt.Fprintf(stdout, "dry-run mode, not making changes\n") + return nil + } + + fmt.Fprintf(stdout, "applying changes\n") + if err := kubeClient.Update(ctx, u, client.FieldOwner(options.FieldOwner)); err != nil { + return fmt.Errorf("updating object: %w", err) + } + + return nil +} diff --git a/pkg/cli/powertools/cmd.go b/pkg/cli/powertools/cmd.go index fdaa3a23c9..fdcc7c5848 100644 --- a/pkg/cli/powertools/cmd.go +++ b/pkg/cli/powertools/cmd.go @@ -15,6 +15,7 @@ package powertools import ( + "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/cli/powertools/changestateintospec" "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/cli/powertools/forcesetfield" "github.com/spf13/cobra" ) @@ -28,4 +29,5 @@ func AddCommands(parent *cobra.Command) { parent.AddCommand(powertoolsCmd) forcesetfield.AddCommand(powertoolsCmd) + changestateintospec.AddCommand(powertoolsCmd) } diff --git a/pkg/cli/powertools/forcesetfield/cmd.go b/pkg/cli/powertools/forcesetfield/cmd.go index 3b63866072..664832bf93 100644 --- a/pkg/cli/powertools/forcesetfield/cmd.go +++ b/pkg/cli/powertools/forcesetfield/cmd.go @@ -53,29 +53,20 @@ func AddCommand(parent *cobra.Command) { options.PopulateDefaults() cmd := &cobra.Command{ - Use: "force-set-field KIND NAME FIELD.PATH=VALUE", + Use: "force-set-field FIELD.PATH=VALUE", Short: "Sets a field on a KCC object, even immutable fields (experimental)", RunE: func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() - if len(args) >= 1 { - options.Kind = args[0] - } - if len(args) >= 2 { - options.Name = args[1] - } - setFields := map[string]string{} - if len(args) >= 3 { - for i := 2; i < len(args); i++ { - tokens := strings.SplitN(args[i], "=", 2) - if len(tokens) < 2 { - return fmt.Errorf("expected spec.path=value, got %q", args[i]) - } - k := tokens[0] - v := tokens[1] - setFields[k] = v + for i := 0; i < len(args); i++ { + tokens := strings.SplitN(args[i], "=", 2) + if len(tokens) < 2 { + return fmt.Errorf("expected spec.path=value, got %q", args[i]) } + k := tokens[0] + v := tokens[1] + setFields[k] = v } return Run(ctx, cmd.OutOrStdout(), options, setFields) @@ -94,13 +85,18 @@ func AddCommand(parent *cobra.Command) { func Run(ctx context.Context, out io.Writer, options Options, setFields map[string]string) error { // log := klog.FromContext(ctx) - // Impersonate the KCC service account, which is allowed to make changes - // TODO: Make this configurable - maybe it only works in namespaced mode? - options.ClusterOptions.Impersonate = &rest.ImpersonationConfig{ - UserName: "system:serviceaccount:cnrm-system:cnrm-controller-manager-" + options.Namespace, - Groups: []string{"system:serviceaccounts", "system:serviceaccounts:cnrm-system"}, + if options.ImpersonateUser == "" { + // Impersonate the KCC service account, which is allowed to make changes + options.ClusterOptions.Impersonate = &rest.ImpersonationConfig{ + UserName: "system:serviceaccount:cnrm-system:cnrm-controller-manager-" + options.Namespace, + Groups: []string{"system:serviceaccounts", "system:serviceaccounts:cnrm-system"}, + } + } else { + options.ClusterOptions.Impersonate = &rest.ImpersonationConfig{ + UserName: options.ImpersonateUser, + Groups: options.ImpersonateGroups, + } } - kubeClient, err := kubecli.NewClient(ctx, options.ClusterOptions) if err != nil { return fmt.Errorf("creating client: %w", err) diff --git a/pkg/cli/powertools/kubecli/client.go b/pkg/cli/powertools/kubecli/client.go index da012663a0..2ed26c392d 100644 --- a/pkg/cli/powertools/kubecli/client.go +++ b/pkg/cli/powertools/kubecli/client.go @@ -30,6 +30,7 @@ import ( "k8s.io/client-go/discovery" diskcached "k8s.io/client-go/discovery/cached/disk" "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" "k8s.io/client-go/util/homedir" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/config" @@ -41,9 +42,19 @@ type Client struct { } func NewClient(ctx context.Context, options ClusterOptions) (*Client, error) { - restConfig, err := config.GetConfig() - if err != nil { - return nil, fmt.Errorf("getting kubernetes configuration: %w", err) + var restConfig *rest.Config + if options.Kubeconfig != "" { + rc, err := clientcmd.BuildConfigFromFlags("", options.Kubeconfig) + if err != nil { + return nil, fmt.Errorf("loading kubernetes configuration from %q: %w", options.Kubeconfig, err) + } + restConfig = rc + } else { + rc, err := config.GetConfig() + if err != nil { + return nil, fmt.Errorf("getting kubernetes configuration: %w", err) + } + restConfig = rc } if options.Impersonate != nil { @@ -123,15 +134,15 @@ func getDefaultCacheDir() string { func (c *Client) GetObject(ctx context.Context, options ObjectOptions) (*unstructured.Unstructured, error) { if options.Kind == "" { - return nil, fmt.Errorf("must specify object kind to target") + return nil, fmt.Errorf("must specify object kind to target (use --kind flag)") } if options.Name == "" { - return nil, fmt.Errorf("must specify object name to target") + return nil, fmt.Errorf("must specify object name to target (use --name flag)") } if options.Namespace == "" { - return nil, fmt.Errorf("must specify object namespace to target") + return nil, fmt.Errorf("must specify object namespace to target (use --namespace flag)") } resources, err := c.DiscoveryClient.ServerPreferredResources() diff --git a/pkg/cli/powertools/kubecli/options.go b/pkg/cli/powertools/kubecli/options.go index 3fbb535f27..30689febb5 100644 --- a/pkg/cli/powertools/kubecli/options.go +++ b/pkg/cli/powertools/kubecli/options.go @@ -20,8 +20,17 @@ import ( ) type ClusterOptions struct { + // Path to the kubeconfig file to use for CLI requests. + Kubeconfig string + // Impersonate is the configuration that RESTClient will use for impersonation. Impersonate *rest.ImpersonationConfig + + // ImpersonateUser is the user name to impersonate + ImpersonateUser string + + // Group to impersonate for the operation, this flag can be repeated to specify multiple groups. + ImpersonateGroups []string } func (o *ClusterOptions) PopulateDefaults() { @@ -29,6 +38,9 @@ func (o *ClusterOptions) PopulateDefaults() { } func (o *ClusterOptions) AddFlags(cmd *cobra.Command) { + cmd.Flags().StringVar(&o.Kubeconfig, "kubeconfig", o.Kubeconfig, "Path to the kubeconfig file to use for CLI requests.") + cmd.Flags().StringVar(&o.ImpersonateUser, "as", o.ImpersonateUser, "Username to impersonate for the operation. User could be a regular user or a service account in a namespace.") + cmd.Flags().StringSliceVar(&o.ImpersonateGroups, "as-group", o.ImpersonateGroups, "Group to impersonate for the operation, this flag can be repeated to specify multiple groups.") } type ObjectOptions struct { @@ -45,6 +57,7 @@ func (o *ObjectOptions) PopulateDefaults() { } func (o *ObjectOptions) AddFlags(cmd *cobra.Command) { - // cmd.Flags().StringVar(&o.Name, "name", o.Name, "Name of the object to change") + cmd.Flags().StringVar(&o.Kind, "kind", o.Kind, "Kind of the object to change") + cmd.Flags().StringVar(&o.Name, "name", o.Name, "Name of the object to change") cmd.Flags().StringVarP(&o.Namespace, "namespace", "n", o.Namespace, "Namespace of the object") } diff --git a/pkg/stateintospec/remove.go b/pkg/stateintospec/remove.go new file mode 100644 index 0000000000..2763913d7d --- /dev/null +++ b/pkg/stateintospec/remove.go @@ -0,0 +1,161 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package stateintospec + +import ( + "bytes" + "context" + "errors" + "fmt" + + "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/k8s" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/klog/v2" + "sigs.k8s.io/structured-merge-diff/v4/fieldpath" +) + +type Warning struct { + Message string +} + +type Warnings struct { + Warnings []Warning +} + +func (w *Warnings) AddWarningf(msg string, args ...any) { + w.Warnings = append(w.Warnings, Warning{ + Message: fmt.Sprintf(msg, args...), + }) +} + +func RemoveStateIntoSpecFields(ctx context.Context, u *unstructured.Unstructured, preserveFieldFunc func(fieldPath fieldpath.Path) bool) (*Warnings, error) { + log := klog.FromContext(ctx) + + warnings := &Warnings{} + + managedFields, err := parseManagedFields(u.GetManagedFields()) + if err != nil { + return warnings, fmt.Errorf("parsing managed fields: %w", err) + } + + var errs []error + + for manager, fields := range managedFields { + if manager != k8s.ControllerManagedFieldManager { + continue + } + + fields.Iterate(func(fieldPath fieldpath.Path) { + switch fieldPath[0].String() { + case ".spec": + pathName := fieldPath.String() + if preserveFieldFunc(fieldPath) { + log.Info("preserving field as requested", "field", pathName) + return + } + log.Info("removing field", "field", pathName) + errs = append(errs, removeFieldIfLeaf(ctx, u, fieldPath, warnings)) + case ".status", ".metadata": + // Never part of state-into-spec, ignore + default: + errs = append(errs, fmt.Errorf("found unknown field %q in managed fields", fieldPath.String())) + } + }) + } + + return warnings, errors.Join(errs...) +} + +func removeFieldIfLeaf(ctx context.Context, u *unstructured.Unstructured, fieldPath fieldpath.Path, warnings *Warnings) error { + // log := klog.FromContext(ctx) + + pos := u.Object + n := len(fieldPath) + for i := 0; i < n-1; i++ { + element := fieldPath[i] + + if element.FieldName != nil { + v, found := pos[*element.FieldName] + if !found { + return nil + } + m, ok := v.(map[string]any) + if ok { + pos = m + continue + } + return fmt.Errorf("unexpected type for %q: got %T, expected map", fieldPath, v) + } + return fmt.Errorf("removal of fieldPath %v not implemented", fieldPath) + } + + last := fieldPath[n-1] + if last.FieldName != nil { + v, found := pos[*last.FieldName] + if !found { + // Already removed + return nil + } + switch v := v.(type) { + case map[string]any: + warnings.AddWarningf("skipping field removal of map field %q (may be an indication of undetermined ownership)", fieldPath) + return nil + case []any: + warnings.AddWarningf("skipping field removal of array field %q (may be an indication of undetermined ownership)", fieldPath) + return nil + case string, int, int32, int64, float32, float64, bool: + delete(pos, *last.FieldName) + return nil + default: + return fmt.Errorf("unhandled type for field %q: got %T", fieldPath, v) + } + + } + + return fmt.Errorf("removal of fieldPath %v not implemented", fieldPath) +} + +// parseManagedFields takes the given managed field entries and constructs a +// set of all the k8s-managed fields from the spec, grouping by manager name. +func parseManagedFields(managedFields []metav1.ManagedFieldsEntry) (map[string]*fieldpath.Set, error) { + res := make(map[string]*fieldpath.Set) + for _, managedFieldEntry := range managedFields { + if managedFieldEntry.FieldsType != k8s.ManagedFieldsTypeFieldsV1 { + return nil, fmt.Errorf( + "expected managed field entry for manager '%v' and operation '%v' of type '%v', got type '%v'", + managedFieldEntry.Manager, managedFieldEntry.Operation, k8s.ManagedFieldsTypeFieldsV1, + managedFieldEntry.FieldsType) + } + fieldsV1 := managedFieldEntry.FieldsV1 + if fieldsV1 == nil { + return nil, fmt.Errorf("managed field entry for manager '%v' and operation '%v' has empty fieldsV1", + managedFieldEntry.Manager, managedFieldEntry.Operation) + } + entrySet := fieldpath.NewSet() + if err := entrySet.FromJSON(bytes.NewReader(fieldsV1.Raw)); err != nil { + return nil, fmt.Errorf("error marshaling managed fields for manager '%v' and operation '%v' from JSON: %w", + managedFieldEntry.Manager, managedFieldEntry.Operation, err) + } + + fields := res[managedFieldEntry.Manager] + if fields == nil { + fields = fieldpath.NewSet() + } + fields = fields.Union(entrySet) + res[managedFieldEntry.Manager] = fields + } + return res, nil +} From 040f92219893f13d91a04160acb5b59b1e15505b Mon Sep 17 00:00:00 2001 From: justinsb Date: Wed, 24 Apr 2024 21:03:54 -0400 Subject: [PATCH 2/5] tests: start to refactor normalization logic out of specific tests This way our various tests can share one normalization function --- tests/e2e/normalize.go | 57 +++++++++++++++++++++++++++++++++++++++ tests/e2e/script_test.go | 1 + tests/e2e/unified_test.go | 42 +---------------------------- 3 files changed, 59 insertions(+), 41 deletions(-) diff --git a/tests/e2e/normalize.go b/tests/e2e/normalize.go index 1d83e32031..a55ac560b7 100644 --- a/tests/e2e/normalize.go +++ b/tests/e2e/normalize.go @@ -17,10 +17,14 @@ package e2e import ( "encoding/json" "fmt" + "net/http" "regexp" "sort" "strings" + "testing" + "time" + "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/test" testgcp "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/test/gcp" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/util/sets" @@ -75,6 +79,13 @@ func normalizeObject(u *unstructured.Unstructured, project testgcp.GCPProject, u return s }) + // Specific to GCS + visitor.replacePaths[".softDeletePolicy.effectiveTime"] = "2024-04-01T12:34:56.123456Z" + visitor.replacePaths[".timeCreated"] = "2024-04-01T12:34:56.123456Z" + visitor.replacePaths[".updated"] = "2024-04-01T12:34:56.123456Z" + visitor.replacePaths[".acl[].etag"] = "abcdef0123A" + visitor.replacePaths[".defaultObjectAcl[].etag"] = "abcdef0123A=" + visitor.sortSlices = sets.New[string]() // TODO: This should not be needed, we want to avoid churning the kube objects visitor.sortSlices.Insert(".spec.access") @@ -251,3 +262,49 @@ func (o *objectWalker) VisitUnstructued(v *unstructured.Unstructured) error { } return nil } + +func NormalizeHTTPLog(t *testing.T, events test.LogEntries, project testgcp.GCPProject, uniqueID string) { + // Remove headers that just aren't very relevant to testing + events.RemoveHTTPResponseHeader("Date") + events.RemoveHTTPResponseHeader("Alt-Svc") + events.RemoveHTTPResponseHeader("Server-Timing") + events.RemoveHTTPResponseHeader("X-Guploader-Uploadid") + events.RemoveHTTPResponseHeader("Etag") + events.RemoveHTTPResponseHeader("Content-Length") // an artifact of encoding + + // Replace any expires headers with (rounded) relative offsets + for _, event := range events { + expires := event.Response.Header.Get("Expires") + if expires == "" { + continue + } + + if expires == "Mon, 01 Jan 1990 00:00:00 GMT" { + // Magic value meaning no-cache; don't change + continue + } + + expiresTime, err := time.Parse(http.TimeFormat, expires) + if err != nil { + t.Fatalf("parsing Expires header %q: %v", expires, err) + } + now := time.Now() + delta := expiresTime.Sub(now) + if delta > (55 * time.Minute) { + delta = delta.Round(time.Hour) + event.Response.Header.Set("Expires", fmt.Sprintf("{now+%vh}", delta.Hours())) + } else { + delta = delta.Round(time.Minute) + event.Response.Header.Set("Expires", fmt.Sprintf("{now+%vm}", delta.Minutes())) + } + } + + events.PrettifyJSON(func(obj map[string]any) { + u := &unstructured.Unstructured{} + u.Object = obj + if err := normalizeObject(u, project, uniqueID); err != nil { + t.Fatalf("error from normalizeObject: %v", err) + } + }) + +} diff --git a/tests/e2e/script_test.go b/tests/e2e/script_test.go index 37062eac34..c19a865d0a 100644 --- a/tests/e2e/script_test.go +++ b/tests/e2e/script_test.go @@ -254,6 +254,7 @@ func TestE2EScript(t *testing.T) { for i, stepEvents := range eventsByStep { expectedPath := filepath.Join(script.SourceDir, fmt.Sprintf("_http%02d.log", i)) + NormalizeHTTPLog(t, stepEvents, project, uniqueID) got := x.Render(stepEvents) h.CompareGoldenFile(expectedPath, got, IgnoreComments) } diff --git a/tests/e2e/unified_test.go b/tests/e2e/unified_test.go index 76d7f031b2..0542de30d1 100644 --- a/tests/e2e/unified_test.go +++ b/tests/e2e/unified_test.go @@ -535,13 +535,6 @@ func runScenario(ctx context.Context, t *testing.T, testPause bool, fixture reso } } - // Specific to GCS - addReplacement("timeCreated", "2024-04-01T12:34:56.123456Z") - addReplacement("updated", "2024-04-01T12:34:56.123456Z") - addReplacement("softDeletePolicy.effectiveTime", "2024-04-01T12:34:56.123456Z") - addSetStringReplacement(".acl[].etag", "abcdef0123A=") - addSetStringReplacement(".defaultObjectAcl[].etag", "abcdef0123A=") - // Specific to AlloyDB addReplacement("uid", "111111111111111111111") addReplacement("response.uid", "111111111111111111111") @@ -594,40 +587,7 @@ func runScenario(ctx context.Context, t *testing.T, testPause bool, fixture reso events.PrettifyJSON(jsonMutators...) - // Remove headers that just aren't very relevant to testing - events.RemoveHTTPResponseHeader("Date") - events.RemoveHTTPResponseHeader("Alt-Svc") - events.RemoveHTTPResponseHeader("Server-Timing") - events.RemoveHTTPResponseHeader("X-Guploader-Uploadid") - events.RemoveHTTPResponseHeader("Etag") - events.RemoveHTTPResponseHeader("Content-Length") // an artifact of encoding - - // Replace any expires headers with (rounded) relative offsets - for _, event := range events { - expires := event.Response.Header.Get("Expires") - if expires == "" { - continue - } - - if expires == "Mon, 01 Jan 1990 00:00:00 GMT" { - // Magic value meaning no-cache; don't change - continue - } - - expiresTime, err := time.Parse(http.TimeFormat, expires) - if err != nil { - t.Fatalf("parsing Expires header %q: %v", expires, err) - } - now := time.Now() - delta := expiresTime.Sub(now) - if delta > (55 * time.Minute) { - delta = delta.Round(time.Hour) - event.Response.Header.Set("Expires", fmt.Sprintf("{now+%vh}", delta.Hours())) - } else { - delta = delta.Round(time.Minute) - event.Response.Header.Set("Expires", fmt.Sprintf("{now+%vm}", delta.Minutes())) - } - } + NormalizeHTTPLog(t, events, project, uniqueID) // Remove repeated GET requests (after normalization) { From 9de6ebd827e64677b0c5b9458876a4a19c415114 Mon Sep 17 00:00:00 2001 From: justinsb Date: Wed, 24 Apr 2024 22:44:22 -0400 Subject: [PATCH 3/5] tests: Add ability to run CLI to script_test This enables us to test our powertools in our existing framework --- config/tests/samples/create/harness.go | 9 +++ pkg/cli/cmd/root.go | 35 ++++++-- tests/e2e/export.go | 2 - tests/e2e/script_test.go | 108 ++++++++++++++++++++++++- tests/e2e/testdata/scenarios/README.md | 5 +- 5 files changed, 149 insertions(+), 10 deletions(-) diff --git a/config/tests/samples/create/harness.go b/config/tests/samples/create/harness.go index cdafff7c2c..3cf6df6584 100644 --- a/config/tests/samples/create/harness.go +++ b/config/tests/samples/create/harness.go @@ -558,11 +558,20 @@ func (h *Harness) GetClient() client.Client { return h.client } +func (h *Harness) GetRESTConfig() *rest.Config { + return h.restConfig +} + func MaybeSkip(t *testing.T, name string, resources []*unstructured.Unstructured) { if os.Getenv("E2E_GCP_TARGET") == "mock" { for _, resource := range resources { gvk := resource.GroupVersionKind() + // Special fake types for testing + if gvk.Group == "" && gvk.Kind == "RunCLI" { + continue + } + switch gvk.Group { case "core.cnrm.cloud.google.com": continue diff --git a/pkg/cli/cmd/root.go b/pkg/cli/cmd/root.go index d415056cd0..2eb30c925b 100644 --- a/pkg/cli/cmd/root.go +++ b/pkg/cli/cmd/root.go @@ -15,6 +15,7 @@ package cmd import ( + "bytes" "fmt" "io/ioutil" golog "log" @@ -89,23 +90,45 @@ func recoverExecute() (err error) { } func execute() error { - defaultToBulkExport() + defaultToBulkExport(os.Args) return rootCmd.Execute() } -func defaultToBulkExport() { +type TestInvocationOptions struct { + Stdout bytes.Buffer + Stderr bytes.Buffer + Stdin bytes.Buffer + Args []string +} + +// ExecuteFromTest allows for invocation of the CLI from a test +func ExecuteFromTest(options *TestInvocationOptions) error { + rootCmd.SetIn(&options.Stdin) + rootCmd.SetOut(&options.Stdout) + rootCmd.SetErr(&options.Stderr) + rootCmd.SetArgs(options.Args[1:]) + + defaultToBulkExport(options.Args) + err := rootCmd.Execute() + if err != nil { + fmt.Fprintf(&options.Stderr, "%v\n", err) + } + return err +} + +func defaultToBulkExport(args []string) { // previously this command had no sub-commands and effectively defaulted to the bulk-export command for backwards // compatibility, if there is no sub-command and the flags appear to be the legacy flags format, default to the // bulk-export sub-command - if isLegacyArgs() { - newArgs := sanitizeArgsForBackwardsCompatibility(os.Args[1:]) + if isLegacyArgs(args) { + newArgs := sanitizeArgsForBackwardsCompatibility(args[1:]) newArgs = append([]string{bulkExportCommandName}, newArgs...) rootCmd.SetArgs(newArgs) } } -func isLegacyArgs() bool { - args := os.Args[1:] +func isLegacyArgs(args []string) bool { + args = args[1:] if len(args) == 0 { // a valid legacy workload was the piping of a list of assets to stdin piped, err := parameters.IsInputPiped(os.Stdin) diff --git a/tests/e2e/export.go b/tests/e2e/export.go index 8f43e63320..8fd7f00c45 100644 --- a/tests/e2e/export.go +++ b/tests/e2e/export.go @@ -22,7 +22,6 @@ import ( "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/cli/cmd/export" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/klog/v2" "sigs.k8s.io/yaml" ) @@ -89,7 +88,6 @@ func exportResourceAsUnstructured(h *create.Harness, obj *unstructured.Unstructu return nil } // TODO: Why are we outputing this prefix? - klog.Infof("exportResourceAsUnstructured %q", s) s = strings.TrimPrefix(s, "----") u := &unstructured.Unstructured{} if err := yaml.Unmarshal([]byte(s), &u); err != nil { diff --git a/tests/e2e/script_test.go b/tests/e2e/script_test.go index c19a865d0a..c17966e206 100644 --- a/tests/e2e/script_test.go +++ b/tests/e2e/script_test.go @@ -25,6 +25,7 @@ import ( "time" "github.com/GoogleCloudPlatform/k8s-config-connector/config/tests/samples/create" + "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/cli/cmd" "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/test" testcontroller "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/test/controller" testgcp "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/test/gcp" @@ -36,6 +37,9 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + clientcmdapi "k8s.io/client-go/tools/clientcmd/api" ) // TestE2EScript runs a Scenario test that runs step-by-step. @@ -79,8 +83,9 @@ func TestE2EScript(t *testing.T) { create.SetupNamespacesAndApplyDefaults(h, script.Objects, project) + var objectsToDelete []*unstructured.Unstructured t.Cleanup(func() { - create.DeleteResources(h, create.CreateDeleteTestOptions{Create: script.Objects}) + create.DeleteResources(h, create.CreateDeleteTestOptions{Create: objectsToDelete}) }) var eventsByStep [][]*test.LogEntry @@ -99,6 +104,20 @@ func TestE2EScript(t *testing.T) { testCommand = "APPLY" } + if obj.GroupVersionKind().Kind == "RunCLI" { + argsObjects := obj.Object["args"].([]any) + var args []string + for _, arg := range argsObjects { + args = append(args, arg.(string)) + } + baseOutputPath := filepath.Join(script.SourceDir, fmt.Sprintf("_cli-%d-", i)) + runCLI(h, args, uniqueID, baseOutputPath) + continue + } + + // Try to delete this object as part of cleanup + objectsToDelete = append(objectsToDelete, obj) + exportResource := obj.DeepCopy() shouldGetKubeObject := true v, ok = obj.Object["WRITE-KUBE-OBJECT"] @@ -119,9 +138,14 @@ func TestE2EScript(t *testing.T) { applyObject(h, obj) create.WaitForReady(h, obj) appliedObjects[k] = obj + case "APPLY-10-SEC": applyObject(h, obj) time.Sleep(10 * time.Second) + + case "READ-OBJECT": + appliedObjects[k] = obj + case "APPLY-NO-WAIT": applyObject(h, obj) appliedObjects[k] = obj @@ -394,3 +418,85 @@ func isConfigConnectorContextObject(gvk schema.GroupVersionKind) bool { } return false } + +func createKubeconfigFromRestConfig(restConfig *rest.Config) ([]byte, error) { + clusters := make(map[string]*clientcmdapi.Cluster) + clusters["default-cluster"] = &clientcmdapi.Cluster{ + Server: restConfig.Host, + CertificateAuthorityData: restConfig.CAData, + } + contexts := make(map[string]*clientcmdapi.Context) + contexts["default-context"] = &clientcmdapi.Context{ + Cluster: "default-cluster", + AuthInfo: "default-user", + } + authInfos := make(map[string]*clientcmdapi.AuthInfo) + authInfos["default-user"] = &clientcmdapi.AuthInfo{ + ClientCertificateData: restConfig.CertData, + ClientKeyData: restConfig.KeyData, + } + clientConfig := clientcmdapi.Config{ + Kind: "Config", + APIVersion: "v1", + Clusters: clusters, + Contexts: contexts, + CurrentContext: "default-context", + AuthInfos: authInfos, + } + return clientcmd.Write(clientConfig) +} + +// runCLI runs the config-connector CLI tool with the specified arguments +func runCLI(h *create.Harness, args []string, uniqueID string, baseOutputPath string) { + project := h.Project + t := h.T + + var options cmd.TestInvocationOptions + + for i, arg := range args { + // Replace any substitutions in the args + arg = strings.ReplaceAll(arg, "${projectId}", project.ProjectID) + arg = strings.ReplaceAll(arg, "${uniqueId}", uniqueID) + args[i] = arg + } + + // Split the args into flags and positional arguments, so we can add more flags + + // Add some flags for kubeconfig and impersonation + { + tempDir := t.TempDir() + p := filepath.Join(tempDir, "kubeconfig") + + kubeconfig, err := createKubeconfigFromRestConfig(h.GetRESTConfig()) + if err != nil { + t.Fatalf("error creating kubeconfig: %v", err) + } + if err := os.WriteFile(p, kubeconfig, 0644); err != nil { + t.Fatalf("error writing kubeconfig to %q: %v", p, err) + } + + args = append(args, "--kubeconfig="+p) + args = append(args, "--as=admin") + args = append(args, "--as-group=system:masters") + } + + options.Args = []string{"config-connector"} + options.Args = append(options.Args, args...) + + t.Logf("running cli with args %+v", options.Args) + if err := cmd.ExecuteFromTest(&options); err != nil { + t.Errorf("cli execution (args=%+v) failed: %v", options.Args, err) + } + + stdout := options.Stdout.String() + t.Logf("stdout: %v", stdout) + stdout = strings.ReplaceAll(stdout, project.ProjectID, "${projectID}") + stdout = strings.ReplaceAll(stdout, uniqueID, "${uniqueId}") + test.CompareGoldenFile(t, baseOutputPath+"stdout.log", stdout) + + stderr := options.Stderr.String() + t.Logf("stderr: %v", stderr) + stderr = strings.ReplaceAll(stderr, project.ProjectID, "${projectID}") + stderr = strings.ReplaceAll(stderr, uniqueID, "${uniqueId}") + test.CompareGoldenFile(t, baseOutputPath+"stderr.log", stderr) +} diff --git a/tests/e2e/testdata/scenarios/README.md b/tests/e2e/testdata/scenarios/README.md index 8fca0a27c8..425f25d54f 100644 --- a/tests/e2e/testdata/scenarios/README.md +++ b/tests/e2e/testdata/scenarios/README.md @@ -23,6 +23,9 @@ a top-level field `TEST` on the object: we stop the test after 10s and capture the error log. This action can be used to test the expected error state. +* Setting `TEST: READ-OBJECT` skips the apply; we read the current value of the + object without changing it. + * Setting `TEST: DELETE` will delete the KCC object and wait for the deletion to complete; it will automatically skip the GCP export and the kube export. It suffices to set @@ -38,7 +41,7 @@ a top-level field `TEST` on the object: object will still be deleted from the kube-apiserver. It suffices to set apiVersion / kind / namespace / name. -* Setting `TEST: WAIT-FOR-HTTP-REQUEST`along with `VALUE_PRESENT: your value` will apply the object +* Setting `TEST: WAIT-FOR-HTTP-REQUEST` along with `VALUE_PRESENT: your value` will apply the object and inspect the http log to check that the value in VALUE_PRESENT appears. The step will wait ~ seconds for that value to show up. From 27d9dba38b8fb496fb088cdd6e9185e202fc6a3c Mon Sep 17 00:00:00 2001 From: justinsb Date: Wed, 24 Apr 2024 20:46:10 -0400 Subject: [PATCH 4/5] tests: Create change-state-into-spec powertool test scenario --- .../clear_state_into_spec/_cli-2-stderr.log | 1 + .../clear_state_into_spec/_cli-2-stdout.log | 12 + .../clear_state_into_spec/_http00.log | 305 ++++++++++++++++++ .../clear_state_into_spec/_http01.log | 305 ++++++++++++++++++ .../clear_state_into_spec/_object00.yaml | 52 +++ .../clear_state_into_spec/_object01.yaml | 49 +++ .../clear_state_into_spec/_object03.yaml | 50 +++ .../clear_state_into_spec/script.yaml | 62 ++++ 8 files changed, 836 insertions(+) create mode 100644 tests/e2e/testdata/scenarios/clear_state_into_spec/_cli-2-stderr.log create mode 100644 tests/e2e/testdata/scenarios/clear_state_into_spec/_cli-2-stdout.log create mode 100644 tests/e2e/testdata/scenarios/clear_state_into_spec/_http00.log create mode 100644 tests/e2e/testdata/scenarios/clear_state_into_spec/_http01.log create mode 100644 tests/e2e/testdata/scenarios/clear_state_into_spec/_object00.yaml create mode 100644 tests/e2e/testdata/scenarios/clear_state_into_spec/_object01.yaml create mode 100644 tests/e2e/testdata/scenarios/clear_state_into_spec/_object03.yaml create mode 100644 tests/e2e/testdata/scenarios/clear_state_into_spec/script.yaml diff --git a/tests/e2e/testdata/scenarios/clear_state_into_spec/_cli-2-stderr.log b/tests/e2e/testdata/scenarios/clear_state_into_spec/_cli-2-stderr.log new file mode 100644 index 0000000000..7cd5e0568b --- /dev/null +++ b/tests/e2e/testdata/scenarios/clear_state_into_spec/_cli-2-stderr.log @@ -0,0 +1 @@ +skipping field removal of array field (may be an indication of undetermined ownership)%!(EXTRA string=path, fieldpath.Path=.spec.lifecycleRule) diff --git a/tests/e2e/testdata/scenarios/clear_state_into_spec/_cli-2-stdout.log b/tests/e2e/testdata/scenarios/clear_state_into_spec/_cli-2-stdout.log new file mode 100644 index 0000000000..fb5f8d9316 --- /dev/null +++ b/tests/e2e/testdata/scenarios/clear_state_into_spec/_cli-2-stdout.log @@ -0,0 +1,12 @@ + + + StorageBucket ${projectID}/storagebucket-merge-${uniqueId}: + metadata: + annotations: + cnrm.cloud.google.com/state-into-spec: merge -> absent + spec: + publicAccessPrevention: inherited -> + storageClass: STANDARD -> + + +applying changes diff --git a/tests/e2e/testdata/scenarios/clear_state_into_spec/_http00.log b/tests/e2e/testdata/scenarios/clear_state_into_spec/_http00.log new file mode 100644 index 0000000000..d7e0db8eff --- /dev/null +++ b/tests/e2e/testdata/scenarios/clear_state_into_spec/_http00.log @@ -0,0 +1,305 @@ +GET https://storage.googleapis.com/storage/v1/b/storagebucket-merge-${uniqueId}?alt=json&prettyPrint=false +User-Agent: google-api-go-client/0.5 Terraform/ (+https://www.terraform.io) Terraform-Plugin-SDK/2.10.1 terraform-provider-google-beta/kcc/controller-manager +X-Goog-Api-Client: gl-go/1.22.0 gdcl/0.160.0 + +404 Not Found +Cache-Control: no-cache, no-store, max-age=0, must-revalidate +Content-Type: application/json; charset=UTF-8 +Expires: Mon, 01 Jan 1990 00:00:00 GMT +Pragma: no-cache +Server: UploadServer +Vary: Origin +Vary: X-Origin + +{ + "error": { + "code": 404, + "errors": [ + { + "domain": "global", + "message": "The specified bucket does not exist.", + "reason": "notFound" + } + ], + "message": "The specified bucket does not exist." + } +} + +--- + +POST https://storage.googleapis.com/storage/v1/b?alt=json&prettyPrint=false&project=${projectId} +Content-Type: application/json +User-Agent: google-api-go-client/0.5 Terraform/ (+https://www.terraform.io) Terraform-Plugin-SDK/2.10.1 terraform-provider-google-beta/kcc/controller-manager +X-Goog-Api-Client: gl-go/1.22.0 gdcl/0.160.0 + +{ + "iamConfiguration": { + "uniformBucketLevelAccess": { + "enabled": false + } + }, + "labels": { + "label-one": "value-one", + "managed-by-cnrm": "true" + }, + "lifecycle": { + "rule": [ + { + "action": { + "type": "Delete" + }, + "condition": { + "age": 7 + } + } + ] + }, + "name": "storagebucket-merge-${uniqueId}", + "storageClass": "STANDARD", + "versioning": { + "enabled": false + } +} + +200 OK +Cache-Control: no-cache, no-store, max-age=0, must-revalidate +Content-Type: application/json; charset=UTF-8 +Expires: Mon, 01 Jan 1990 00:00:00 GMT +Pragma: no-cache +Server: UploadServer +Vary: Origin +Vary: X-Origin + +{ + "etag": "abcdef0123A=", + "iamConfiguration": { + "bucketPolicyOnly": { + "enabled": false + }, + "publicAccessPrevention": "inherited", + "uniformBucketLevelAccess": { + "enabled": false + } + }, + "id": "000000000000000000000", + "kind": "storage#bucket", + "labels": { + "label-one": "value-one", + "managed-by-cnrm": "true" + }, + "lifecycle": { + "rule": [ + { + "action": { + "type": "Delete" + }, + "condition": { + "age": 7 + } + } + ] + }, + "location": "US", + "locationType": "multi-region", + "metageneration": "1", + "name": "storagebucket-merge-${uniqueId}", + "projectNumber": "${projectNumber}", + "rpo": "DEFAULT", + "selfLink": "https://www.googleapis.com/storage/v1/b/storagebucket-merge-${uniqueId}", + "softDeletePolicy": { + "effectiveTime": "2024-04-01T12:34:56.123456Z", + "retentionDurationSeconds": "604800" + }, + "storageClass": "STANDARD", + "timeCreated": "2024-04-01T12:34:56.123456Z", + "updated": "2024-04-01T12:34:56.123456Z", + "versioning": { + "enabled": false + } +} + +--- + +GET https://storage.googleapis.com/storage/v1/b/storagebucket-merge-${uniqueId}?alt=json&prettyPrint=false +User-Agent: google-api-go-client/0.5 Terraform/ (+https://www.terraform.io) Terraform-Plugin-SDK/2.10.1 terraform-provider-google-beta/kcc/controller-manager +X-Goog-Api-Client: gl-go/1.22.0 gdcl/0.160.0 + +200 OK +Cache-Control: private, max-age=0, must-revalidate, no-transform +Content-Type: application/json; charset=UTF-8 +Expires: {now+0m} +Server: UploadServer +Vary: Origin +Vary: X-Origin + +{ + "etag": "abcdef0123A=", + "iamConfiguration": { + "bucketPolicyOnly": { + "enabled": false + }, + "publicAccessPrevention": "inherited", + "uniformBucketLevelAccess": { + "enabled": false + } + }, + "id": "000000000000000000000", + "kind": "storage#bucket", + "labels": { + "label-one": "value-one", + "managed-by-cnrm": "true" + }, + "lifecycle": { + "rule": [ + { + "action": { + "type": "Delete" + }, + "condition": { + "age": 7 + } + } + ] + }, + "location": "US", + "locationType": "multi-region", + "metageneration": "1", + "name": "storagebucket-merge-${uniqueId}", + "projectNumber": "${projectNumber}", + "rpo": "DEFAULT", + "selfLink": "https://www.googleapis.com/storage/v1/b/storagebucket-merge-${uniqueId}", + "softDeletePolicy": { + "effectiveTime": "2024-04-01T12:34:56.123456Z", + "retentionDurationSeconds": "604800" + }, + "storageClass": "STANDARD", + "timeCreated": "2024-04-01T12:34:56.123456Z", + "updated": "2024-04-01T12:34:56.123456Z", + "versioning": { + "enabled": false + } +} + +--- + +GET https://storage.googleapis.com/storage/v1/b/storagebucket-merge-${uniqueId}?alt=json&prettyPrint=false +User-Agent: google-api-go-client/0.5 Terraform/ (+https://www.terraform.io) Terraform-Plugin-SDK/2.10.1 terraform-provider-google-beta/kcc/controller-manager +X-Goog-Api-Client: gl-go/1.22.0 gdcl/0.160.0 + +200 OK +Cache-Control: private, max-age=0, must-revalidate, no-transform +Content-Type: application/json; charset=UTF-8 +Expires: {now+0m} +Server: UploadServer +Vary: Origin +Vary: X-Origin + +{ + "etag": "abcdef0123A=", + "iamConfiguration": { + "bucketPolicyOnly": { + "enabled": false + }, + "publicAccessPrevention": "inherited", + "uniformBucketLevelAccess": { + "enabled": false + } + }, + "id": "000000000000000000000", + "kind": "storage#bucket", + "labels": { + "label-one": "value-one", + "managed-by-cnrm": "true" + }, + "lifecycle": { + "rule": [ + { + "action": { + "type": "Delete" + }, + "condition": { + "age": 7 + } + } + ] + }, + "location": "US", + "locationType": "multi-region", + "metageneration": "1", + "name": "storagebucket-merge-${uniqueId}", + "projectNumber": "${projectNumber}", + "rpo": "DEFAULT", + "selfLink": "https://www.googleapis.com/storage/v1/b/storagebucket-merge-${uniqueId}", + "softDeletePolicy": { + "effectiveTime": "2024-04-01T12:34:56.123456Z", + "retentionDurationSeconds": "604800" + }, + "storageClass": "STANDARD", + "timeCreated": "2024-04-01T12:34:56.123456Z", + "updated": "2024-04-01T12:34:56.123456Z", + "versioning": { + "enabled": false + } +} + +--- + +GET https://storage.googleapis.com/storage/v1/b/storagebucket-merge-${uniqueId}?alt=json&prettyPrint=false +User-Agent: google-api-go-client/0.5 Terraform/ (+https://www.terraform.io) Terraform-Plugin-SDK/2.10.1 terraform-provider-google-beta/kcc/controller-manager +X-Goog-Api-Client: gl-go/1.22.0 gdcl/0.160.0 + +200 OK +Cache-Control: private, max-age=0, must-revalidate, no-transform +Content-Type: application/json; charset=UTF-8 +Expires: {now+0m} +Server: UploadServer +Vary: Origin +Vary: X-Origin + +{ + "etag": "abcdef0123A=", + "iamConfiguration": { + "bucketPolicyOnly": { + "enabled": false + }, + "publicAccessPrevention": "inherited", + "uniformBucketLevelAccess": { + "enabled": false + } + }, + "id": "000000000000000000000", + "kind": "storage#bucket", + "labels": { + "label-one": "value-one", + "managed-by-cnrm": "true" + }, + "lifecycle": { + "rule": [ + { + "action": { + "type": "Delete" + }, + "condition": { + "age": 7 + } + } + ] + }, + "location": "US", + "locationType": "multi-region", + "metageneration": "1", + "name": "storagebucket-merge-${uniqueId}", + "projectNumber": "${projectNumber}", + "rpo": "DEFAULT", + "selfLink": "https://www.googleapis.com/storage/v1/b/storagebucket-merge-${uniqueId}", + "softDeletePolicy": { + "effectiveTime": "2024-04-01T12:34:56.123456Z", + "retentionDurationSeconds": "604800" + }, + "storageClass": "STANDARD", + "timeCreated": "2024-04-01T12:34:56.123456Z", + "updated": "2024-04-01T12:34:56.123456Z", + "versioning": { + "enabled": false + } +} \ No newline at end of file diff --git a/tests/e2e/testdata/scenarios/clear_state_into_spec/_http01.log b/tests/e2e/testdata/scenarios/clear_state_into_spec/_http01.log new file mode 100644 index 0000000000..b1b7f47010 --- /dev/null +++ b/tests/e2e/testdata/scenarios/clear_state_into_spec/_http01.log @@ -0,0 +1,305 @@ +GET https://storage.googleapis.com/storage/v1/b/storagebucket-absent-${uniqueId}?alt=json&prettyPrint=false +User-Agent: google-api-go-client/0.5 Terraform/ (+https://www.terraform.io) Terraform-Plugin-SDK/2.10.1 terraform-provider-google-beta/kcc/controller-manager +X-Goog-Api-Client: gl-go/1.22.0 gdcl/0.160.0 + +404 Not Found +Cache-Control: no-cache, no-store, max-age=0, must-revalidate +Content-Type: application/json; charset=UTF-8 +Expires: Mon, 01 Jan 1990 00:00:00 GMT +Pragma: no-cache +Server: UploadServer +Vary: Origin +Vary: X-Origin + +{ + "error": { + "code": 404, + "errors": [ + { + "domain": "global", + "message": "The specified bucket does not exist.", + "reason": "notFound" + } + ], + "message": "The specified bucket does not exist." + } +} + +--- + +POST https://storage.googleapis.com/storage/v1/b?alt=json&prettyPrint=false&project=${projectId} +Content-Type: application/json +User-Agent: google-api-go-client/0.5 Terraform/ (+https://www.terraform.io) Terraform-Plugin-SDK/2.10.1 terraform-provider-google-beta/kcc/controller-manager +X-Goog-Api-Client: gl-go/1.22.0 gdcl/0.160.0 + +{ + "iamConfiguration": { + "uniformBucketLevelAccess": { + "enabled": false + } + }, + "labels": { + "label-one": "value-one", + "managed-by-cnrm": "true" + }, + "lifecycle": { + "rule": [ + { + "action": { + "type": "Delete" + }, + "condition": { + "age": 7 + } + } + ] + }, + "name": "storagebucket-absent-${uniqueId}", + "storageClass": "STANDARD", + "versioning": { + "enabled": false + } +} + +200 OK +Cache-Control: no-cache, no-store, max-age=0, must-revalidate +Content-Type: application/json; charset=UTF-8 +Expires: Mon, 01 Jan 1990 00:00:00 GMT +Pragma: no-cache +Server: UploadServer +Vary: Origin +Vary: X-Origin + +{ + "etag": "abcdef0123A=", + "iamConfiguration": { + "bucketPolicyOnly": { + "enabled": false + }, + "publicAccessPrevention": "inherited", + "uniformBucketLevelAccess": { + "enabled": false + } + }, + "id": "000000000000000000000", + "kind": "storage#bucket", + "labels": { + "label-one": "value-one", + "managed-by-cnrm": "true" + }, + "lifecycle": { + "rule": [ + { + "action": { + "type": "Delete" + }, + "condition": { + "age": 7 + } + } + ] + }, + "location": "US", + "locationType": "multi-region", + "metageneration": "1", + "name": "storagebucket-absent-${uniqueId}", + "projectNumber": "${projectNumber}", + "rpo": "DEFAULT", + "selfLink": "https://www.googleapis.com/storage/v1/b/storagebucket-absent-${uniqueId}", + "softDeletePolicy": { + "effectiveTime": "2024-04-01T12:34:56.123456Z", + "retentionDurationSeconds": "604800" + }, + "storageClass": "STANDARD", + "timeCreated": "2024-04-01T12:34:56.123456Z", + "updated": "2024-04-01T12:34:56.123456Z", + "versioning": { + "enabled": false + } +} + +--- + +GET https://storage.googleapis.com/storage/v1/b/storagebucket-absent-${uniqueId}?alt=json&prettyPrint=false +User-Agent: google-api-go-client/0.5 Terraform/ (+https://www.terraform.io) Terraform-Plugin-SDK/2.10.1 terraform-provider-google-beta/kcc/controller-manager +X-Goog-Api-Client: gl-go/1.22.0 gdcl/0.160.0 + +200 OK +Cache-Control: private, max-age=0, must-revalidate, no-transform +Content-Type: application/json; charset=UTF-8 +Expires: {now+0m} +Server: UploadServer +Vary: Origin +Vary: X-Origin + +{ + "etag": "abcdef0123A=", + "iamConfiguration": { + "bucketPolicyOnly": { + "enabled": false + }, + "publicAccessPrevention": "inherited", + "uniformBucketLevelAccess": { + "enabled": false + } + }, + "id": "000000000000000000000", + "kind": "storage#bucket", + "labels": { + "label-one": "value-one", + "managed-by-cnrm": "true" + }, + "lifecycle": { + "rule": [ + { + "action": { + "type": "Delete" + }, + "condition": { + "age": 7 + } + } + ] + }, + "location": "US", + "locationType": "multi-region", + "metageneration": "1", + "name": "storagebucket-absent-${uniqueId}", + "projectNumber": "${projectNumber}", + "rpo": "DEFAULT", + "selfLink": "https://www.googleapis.com/storage/v1/b/storagebucket-absent-${uniqueId}", + "softDeletePolicy": { + "effectiveTime": "2024-04-01T12:34:56.123456Z", + "retentionDurationSeconds": "604800" + }, + "storageClass": "STANDARD", + "timeCreated": "2024-04-01T12:34:56.123456Z", + "updated": "2024-04-01T12:34:56.123456Z", + "versioning": { + "enabled": false + } +} + +--- + +GET https://storage.googleapis.com/storage/v1/b/storagebucket-absent-${uniqueId}?alt=json&prettyPrint=false +User-Agent: google-api-go-client/0.5 Terraform/ (+https://www.terraform.io) Terraform-Plugin-SDK/2.10.1 terraform-provider-google-beta/kcc/controller-manager +X-Goog-Api-Client: gl-go/1.22.0 gdcl/0.160.0 + +200 OK +Cache-Control: private, max-age=0, must-revalidate, no-transform +Content-Type: application/json; charset=UTF-8 +Expires: {now+0m} +Server: UploadServer +Vary: Origin +Vary: X-Origin + +{ + "etag": "abcdef0123A=", + "iamConfiguration": { + "bucketPolicyOnly": { + "enabled": false + }, + "publicAccessPrevention": "inherited", + "uniformBucketLevelAccess": { + "enabled": false + } + }, + "id": "000000000000000000000", + "kind": "storage#bucket", + "labels": { + "label-one": "value-one", + "managed-by-cnrm": "true" + }, + "lifecycle": { + "rule": [ + { + "action": { + "type": "Delete" + }, + "condition": { + "age": 7 + } + } + ] + }, + "location": "US", + "locationType": "multi-region", + "metageneration": "1", + "name": "storagebucket-absent-${uniqueId}", + "projectNumber": "${projectNumber}", + "rpo": "DEFAULT", + "selfLink": "https://www.googleapis.com/storage/v1/b/storagebucket-absent-${uniqueId}", + "softDeletePolicy": { + "effectiveTime": "2024-04-01T12:34:56.123456Z", + "retentionDurationSeconds": "604800" + }, + "storageClass": "STANDARD", + "timeCreated": "2024-04-01T12:34:56.123456Z", + "updated": "2024-04-01T12:34:56.123456Z", + "versioning": { + "enabled": false + } +} + +--- + +GET https://storage.googleapis.com/storage/v1/b/storagebucket-absent-${uniqueId}?alt=json&prettyPrint=false +User-Agent: google-api-go-client/0.5 Terraform/ (+https://www.terraform.io) Terraform-Plugin-SDK/2.10.1 terraform-provider-google-beta/kcc/controller-manager +X-Goog-Api-Client: gl-go/1.22.0 gdcl/0.160.0 + +200 OK +Cache-Control: private, max-age=0, must-revalidate, no-transform +Content-Type: application/json; charset=UTF-8 +Expires: {now+0m} +Server: UploadServer +Vary: Origin +Vary: X-Origin + +{ + "etag": "abcdef0123A=", + "iamConfiguration": { + "bucketPolicyOnly": { + "enabled": false + }, + "publicAccessPrevention": "inherited", + "uniformBucketLevelAccess": { + "enabled": false + } + }, + "id": "000000000000000000000", + "kind": "storage#bucket", + "labels": { + "label-one": "value-one", + "managed-by-cnrm": "true" + }, + "lifecycle": { + "rule": [ + { + "action": { + "type": "Delete" + }, + "condition": { + "age": 7 + } + } + ] + }, + "location": "US", + "locationType": "multi-region", + "metageneration": "1", + "name": "storagebucket-absent-${uniqueId}", + "projectNumber": "${projectNumber}", + "rpo": "DEFAULT", + "selfLink": "https://www.googleapis.com/storage/v1/b/storagebucket-absent-${uniqueId}", + "softDeletePolicy": { + "effectiveTime": "2024-04-01T12:34:56.123456Z", + "retentionDurationSeconds": "604800" + }, + "storageClass": "STANDARD", + "timeCreated": "2024-04-01T12:34:56.123456Z", + "updated": "2024-04-01T12:34:56.123456Z", + "versioning": { + "enabled": false + } +} \ No newline at end of file diff --git a/tests/e2e/testdata/scenarios/clear_state_into_spec/_object00.yaml b/tests/e2e/testdata/scenarios/clear_state_into_spec/_object00.yaml new file mode 100644 index 0000000000..2ee54e5f7e --- /dev/null +++ b/tests/e2e/testdata/scenarios/clear_state_into_spec/_object00.yaml @@ -0,0 +1,52 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: storage.cnrm.cloud.google.com/v1beta1 +kind: StorageBucket +metadata: + annotations: + cnrm.cloud.google.com/management-conflict-prevention-policy: none + cnrm.cloud.google.com/project-id: ${projectId} + cnrm.cloud.google.com/state-into-spec: merge + finalizers: + - cnrm.cloud.google.com/finalizer + - cnrm.cloud.google.com/deletion-defender + generation: 2 + labels: + label-one: value-one + name: storagebucket-merge-${uniqueId} + namespace: ${projectId} +spec: + lifecycleRule: + - action: + type: Delete + condition: + age: 7 + withState: ANY + location: US + publicAccessPrevention: inherited + resourceID: storagebucket-merge-${uniqueId} + storageClass: STANDARD + versioning: + enabled: false +status: + conditions: + - lastTransitionTime: "1970-01-01T00:00:00Z" + message: The resource is up to date + reason: UpToDate + status: "True" + type: Ready + observedGeneration: 2 + selfLink: https://www.googleapis.com/storage/v1/b/storagebucket-merge-${uniqueId} + url: gs://storagebucket-merge-${uniqueId} \ No newline at end of file diff --git a/tests/e2e/testdata/scenarios/clear_state_into_spec/_object01.yaml b/tests/e2e/testdata/scenarios/clear_state_into_spec/_object01.yaml new file mode 100644 index 0000000000..f1ddc328c7 --- /dev/null +++ b/tests/e2e/testdata/scenarios/clear_state_into_spec/_object01.yaml @@ -0,0 +1,49 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: storage.cnrm.cloud.google.com/v1beta1 +kind: StorageBucket +metadata: + annotations: + cnrm.cloud.google.com/management-conflict-prevention-policy: none + cnrm.cloud.google.com/project-id: ${projectId} + cnrm.cloud.google.com/state-into-spec: absent + finalizers: + - cnrm.cloud.google.com/finalizer + - cnrm.cloud.google.com/deletion-defender + generation: 2 + labels: + label-one: value-one + name: storagebucket-absent-${uniqueId} + namespace: ${projectId} +spec: + lifecycleRule: + - action: + type: Delete + condition: + age: 7 + location: US + resourceID: storagebucket-absent-${uniqueId} + versioning: + enabled: false +status: + conditions: + - lastTransitionTime: "1970-01-01T00:00:00Z" + message: The resource is up to date + reason: UpToDate + status: "True" + type: Ready + observedGeneration: 2 + selfLink: https://www.googleapis.com/storage/v1/b/storagebucket-absent-${uniqueId} + url: gs://storagebucket-absent-${uniqueId} \ No newline at end of file diff --git a/tests/e2e/testdata/scenarios/clear_state_into_spec/_object03.yaml b/tests/e2e/testdata/scenarios/clear_state_into_spec/_object03.yaml new file mode 100644 index 0000000000..a1cf9e3977 --- /dev/null +++ b/tests/e2e/testdata/scenarios/clear_state_into_spec/_object03.yaml @@ -0,0 +1,50 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: storage.cnrm.cloud.google.com/v1beta1 +kind: StorageBucket +metadata: + annotations: + cnrm.cloud.google.com/management-conflict-prevention-policy: none + cnrm.cloud.google.com/project-id: ${projectId} + cnrm.cloud.google.com/state-into-spec: absent + finalizers: + - cnrm.cloud.google.com/finalizer + - cnrm.cloud.google.com/deletion-defender + generation: 3 + labels: + label-one: value-one + name: storagebucket-merge-${uniqueId} + namespace: ${projectId} +spec: + lifecycleRule: + - action: + type: Delete + condition: + age: 7 + withState: ANY + location: US + resourceID: storagebucket-merge-${uniqueId} + versioning: + enabled: false +status: + conditions: + - lastTransitionTime: "1970-01-01T00:00:00Z" + message: The resource is up to date + reason: UpToDate + status: "True" + type: Ready + observedGeneration: 2 + selfLink: https://www.googleapis.com/storage/v1/b/storagebucket-merge-${uniqueId} + url: gs://storagebucket-merge-${uniqueId} \ No newline at end of file diff --git a/tests/e2e/testdata/scenarios/clear_state_into_spec/script.yaml b/tests/e2e/testdata/scenarios/clear_state_into_spec/script.yaml new file mode 100644 index 0000000000..a72650c766 --- /dev/null +++ b/tests/e2e/testdata/scenarios/clear_state_into_spec/script.yaml @@ -0,0 +1,62 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + + +apiVersion: storage.cnrm.cloud.google.com/v1beta1 +kind: StorageBucket +metadata: + labels: + label-one: "value-one" + name: storagebucket-merge-${uniqueId} +spec: + versioning: + enabled: false + lifecycleRule: + - action: + type: Delete + condition: + age: 7 + +--- + +apiVersion: storage.cnrm.cloud.google.com/v1beta1 +kind: StorageBucket +metadata: + labels: + label-one: "value-one" + annotations: + cnrm.cloud.google.com/state-into-spec: "absent" + name: storagebucket-absent-${uniqueId} +spec: + versioning: + enabled: false + lifecycleRule: + - action: + type: Delete + condition: + age: 7 + +--- + +kind: RunCLI +args: [ "powertools", "change-state-into-spec", "--namespace=${projectId}", "--kind=StorageBucket", "--name=storagebucket-merge-${uniqueId}" ] + +--- + +TEST: READ-OBJECT +apiVersion: storage.cnrm.cloud.google.com/v1beta1 +kind: StorageBucket +metadata: + name: storagebucket-merge-${uniqueId} From 12a1e42f387eea61d1fd6eebeb0265b60e208c43 Mon Sep 17 00:00:00 2001 From: justinsb Date: Wed, 24 Apr 2024 23:18:07 -0400 Subject: [PATCH 5/5] tests: Create test scenario for force-set-field powertool Changing the location of a StorageBucket is a good example usage. --- .../_cli-1-stdout.log | 8 + .../powertool_set_bucket_location/_http00.log | 305 ++++++++++++++++++ .../_object00.yaml | 52 +++ .../_object02.yaml | 52 +++ .../powertool_set_bucket_location/script.yaml | 41 +++ 5 files changed, 458 insertions(+) create mode 100644 tests/e2e/testdata/scenarios/powertool_set_bucket_location/_cli-1-stdout.log create mode 100644 tests/e2e/testdata/scenarios/powertool_set_bucket_location/_http00.log create mode 100644 tests/e2e/testdata/scenarios/powertool_set_bucket_location/_object00.yaml create mode 100644 tests/e2e/testdata/scenarios/powertool_set_bucket_location/_object02.yaml create mode 100644 tests/e2e/testdata/scenarios/powertool_set_bucket_location/script.yaml diff --git a/tests/e2e/testdata/scenarios/powertool_set_bucket_location/_cli-1-stdout.log b/tests/e2e/testdata/scenarios/powertool_set_bucket_location/_cli-1-stdout.log new file mode 100644 index 0000000000..430684033f --- /dev/null +++ b/tests/e2e/testdata/scenarios/powertool_set_bucket_location/_cli-1-stdout.log @@ -0,0 +1,8 @@ + + + StorageBucket ${projectID}/storagebucket-${uniqueId}: + spec: + location: US -> EU + + +applying changes diff --git a/tests/e2e/testdata/scenarios/powertool_set_bucket_location/_http00.log b/tests/e2e/testdata/scenarios/powertool_set_bucket_location/_http00.log new file mode 100644 index 0000000000..2cfb2dfe90 --- /dev/null +++ b/tests/e2e/testdata/scenarios/powertool_set_bucket_location/_http00.log @@ -0,0 +1,305 @@ +GET https://storage.googleapis.com/storage/v1/b/storagebucket-${uniqueId}?alt=json&prettyPrint=false +User-Agent: google-api-go-client/0.5 Terraform/ (+https://www.terraform.io) Terraform-Plugin-SDK/2.10.1 terraform-provider-google-beta/kcc/controller-manager +X-Goog-Api-Client: gl-go/1.22.0 gdcl/0.160.0 + +404 Not Found +Cache-Control: no-cache, no-store, max-age=0, must-revalidate +Content-Type: application/json; charset=UTF-8 +Expires: Mon, 01 Jan 1990 00:00:00 GMT +Pragma: no-cache +Server: UploadServer +Vary: Origin +Vary: X-Origin + +{ + "error": { + "code": 404, + "errors": [ + { + "domain": "global", + "message": "The specified bucket does not exist.", + "reason": "notFound" + } + ], + "message": "The specified bucket does not exist." + } +} + +--- + +POST https://storage.googleapis.com/storage/v1/b?alt=json&prettyPrint=false&project=${projectId} +Content-Type: application/json +User-Agent: google-api-go-client/0.5 Terraform/ (+https://www.terraform.io) Terraform-Plugin-SDK/2.10.1 terraform-provider-google-beta/kcc/controller-manager +X-Goog-Api-Client: gl-go/1.22.0 gdcl/0.160.0 + +{ + "iamConfiguration": { + "uniformBucketLevelAccess": { + "enabled": false + } + }, + "labels": { + "label-one": "value-one", + "managed-by-cnrm": "true" + }, + "lifecycle": { + "rule": [ + { + "action": { + "type": "Delete" + }, + "condition": { + "age": 7 + } + } + ] + }, + "name": "storagebucket-${uniqueId}", + "storageClass": "STANDARD", + "versioning": { + "enabled": false + } +} + +200 OK +Cache-Control: no-cache, no-store, max-age=0, must-revalidate +Content-Type: application/json; charset=UTF-8 +Expires: Mon, 01 Jan 1990 00:00:00 GMT +Pragma: no-cache +Server: UploadServer +Vary: Origin +Vary: X-Origin + +{ + "etag": "abcdef0123A=", + "iamConfiguration": { + "bucketPolicyOnly": { + "enabled": false + }, + "publicAccessPrevention": "inherited", + "uniformBucketLevelAccess": { + "enabled": false + } + }, + "id": "000000000000000000000", + "kind": "storage#bucket", + "labels": { + "label-one": "value-one", + "managed-by-cnrm": "true" + }, + "lifecycle": { + "rule": [ + { + "action": { + "type": "Delete" + }, + "condition": { + "age": 7 + } + } + ] + }, + "location": "US", + "locationType": "multi-region", + "metageneration": "1", + "name": "storagebucket-${uniqueId}", + "projectNumber": "${projectNumber}", + "rpo": "DEFAULT", + "selfLink": "https://www.googleapis.com/storage/v1/b/storagebucket-${uniqueId}", + "softDeletePolicy": { + "effectiveTime": "2024-04-01T12:34:56.123456Z", + "retentionDurationSeconds": "604800" + }, + "storageClass": "STANDARD", + "timeCreated": "2024-04-01T12:34:56.123456Z", + "updated": "2024-04-01T12:34:56.123456Z", + "versioning": { + "enabled": false + } +} + +--- + +GET https://storage.googleapis.com/storage/v1/b/storagebucket-${uniqueId}?alt=json&prettyPrint=false +User-Agent: google-api-go-client/0.5 Terraform/ (+https://www.terraform.io) Terraform-Plugin-SDK/2.10.1 terraform-provider-google-beta/kcc/controller-manager +X-Goog-Api-Client: gl-go/1.22.0 gdcl/0.160.0 + +200 OK +Cache-Control: private, max-age=0, must-revalidate, no-transform +Content-Type: application/json; charset=UTF-8 +Expires: {now+0m} +Server: UploadServer +Vary: Origin +Vary: X-Origin + +{ + "etag": "abcdef0123A=", + "iamConfiguration": { + "bucketPolicyOnly": { + "enabled": false + }, + "publicAccessPrevention": "inherited", + "uniformBucketLevelAccess": { + "enabled": false + } + }, + "id": "000000000000000000000", + "kind": "storage#bucket", + "labels": { + "label-one": "value-one", + "managed-by-cnrm": "true" + }, + "lifecycle": { + "rule": [ + { + "action": { + "type": "Delete" + }, + "condition": { + "age": 7 + } + } + ] + }, + "location": "US", + "locationType": "multi-region", + "metageneration": "1", + "name": "storagebucket-${uniqueId}", + "projectNumber": "${projectNumber}", + "rpo": "DEFAULT", + "selfLink": "https://www.googleapis.com/storage/v1/b/storagebucket-${uniqueId}", + "softDeletePolicy": { + "effectiveTime": "2024-04-01T12:34:56.123456Z", + "retentionDurationSeconds": "604800" + }, + "storageClass": "STANDARD", + "timeCreated": "2024-04-01T12:34:56.123456Z", + "updated": "2024-04-01T12:34:56.123456Z", + "versioning": { + "enabled": false + } +} + +--- + +GET https://storage.googleapis.com/storage/v1/b/storagebucket-${uniqueId}?alt=json&prettyPrint=false +User-Agent: google-api-go-client/0.5 Terraform/ (+https://www.terraform.io) Terraform-Plugin-SDK/2.10.1 terraform-provider-google-beta/kcc/controller-manager +X-Goog-Api-Client: gl-go/1.22.0 gdcl/0.160.0 + +200 OK +Cache-Control: private, max-age=0, must-revalidate, no-transform +Content-Type: application/json; charset=UTF-8 +Expires: {now+0m} +Server: UploadServer +Vary: Origin +Vary: X-Origin + +{ + "etag": "abcdef0123A=", + "iamConfiguration": { + "bucketPolicyOnly": { + "enabled": false + }, + "publicAccessPrevention": "inherited", + "uniformBucketLevelAccess": { + "enabled": false + } + }, + "id": "000000000000000000000", + "kind": "storage#bucket", + "labels": { + "label-one": "value-one", + "managed-by-cnrm": "true" + }, + "lifecycle": { + "rule": [ + { + "action": { + "type": "Delete" + }, + "condition": { + "age": 7 + } + } + ] + }, + "location": "US", + "locationType": "multi-region", + "metageneration": "1", + "name": "storagebucket-${uniqueId}", + "projectNumber": "${projectNumber}", + "rpo": "DEFAULT", + "selfLink": "https://www.googleapis.com/storage/v1/b/storagebucket-${uniqueId}", + "softDeletePolicy": { + "effectiveTime": "2024-04-01T12:34:56.123456Z", + "retentionDurationSeconds": "604800" + }, + "storageClass": "STANDARD", + "timeCreated": "2024-04-01T12:34:56.123456Z", + "updated": "2024-04-01T12:34:56.123456Z", + "versioning": { + "enabled": false + } +} + +--- + +GET https://storage.googleapis.com/storage/v1/b/storagebucket-${uniqueId}?alt=json&prettyPrint=false +User-Agent: google-api-go-client/0.5 Terraform/ (+https://www.terraform.io) Terraform-Plugin-SDK/2.10.1 terraform-provider-google-beta/kcc/controller-manager +X-Goog-Api-Client: gl-go/1.22.0 gdcl/0.160.0 + +200 OK +Cache-Control: private, max-age=0, must-revalidate, no-transform +Content-Type: application/json; charset=UTF-8 +Expires: {now+0m} +Server: UploadServer +Vary: Origin +Vary: X-Origin + +{ + "etag": "abcdef0123A=", + "iamConfiguration": { + "bucketPolicyOnly": { + "enabled": false + }, + "publicAccessPrevention": "inherited", + "uniformBucketLevelAccess": { + "enabled": false + } + }, + "id": "000000000000000000000", + "kind": "storage#bucket", + "labels": { + "label-one": "value-one", + "managed-by-cnrm": "true" + }, + "lifecycle": { + "rule": [ + { + "action": { + "type": "Delete" + }, + "condition": { + "age": 7 + } + } + ] + }, + "location": "US", + "locationType": "multi-region", + "metageneration": "1", + "name": "storagebucket-${uniqueId}", + "projectNumber": "${projectNumber}", + "rpo": "DEFAULT", + "selfLink": "https://www.googleapis.com/storage/v1/b/storagebucket-${uniqueId}", + "softDeletePolicy": { + "effectiveTime": "2024-04-01T12:34:56.123456Z", + "retentionDurationSeconds": "604800" + }, + "storageClass": "STANDARD", + "timeCreated": "2024-04-01T12:34:56.123456Z", + "updated": "2024-04-01T12:34:56.123456Z", + "versioning": { + "enabled": false + } +} \ No newline at end of file diff --git a/tests/e2e/testdata/scenarios/powertool_set_bucket_location/_object00.yaml b/tests/e2e/testdata/scenarios/powertool_set_bucket_location/_object00.yaml new file mode 100644 index 0000000000..38bec58f18 --- /dev/null +++ b/tests/e2e/testdata/scenarios/powertool_set_bucket_location/_object00.yaml @@ -0,0 +1,52 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: storage.cnrm.cloud.google.com/v1beta1 +kind: StorageBucket +metadata: + annotations: + cnrm.cloud.google.com/management-conflict-prevention-policy: none + cnrm.cloud.google.com/project-id: ${projectId} + cnrm.cloud.google.com/state-into-spec: merge + finalizers: + - cnrm.cloud.google.com/finalizer + - cnrm.cloud.google.com/deletion-defender + generation: 2 + labels: + label-one: value-one + name: storagebucket-${uniqueId} + namespace: ${projectId} +spec: + lifecycleRule: + - action: + type: Delete + condition: + age: 7 + withState: ANY + location: US + publicAccessPrevention: inherited + resourceID: storagebucket-${uniqueId} + storageClass: STANDARD + versioning: + enabled: false +status: + conditions: + - lastTransitionTime: "1970-01-01T00:00:00Z" + message: The resource is up to date + reason: UpToDate + status: "True" + type: Ready + observedGeneration: 2 + selfLink: https://www.googleapis.com/storage/v1/b/storagebucket-${uniqueId} + url: gs://storagebucket-${uniqueId} \ No newline at end of file diff --git a/tests/e2e/testdata/scenarios/powertool_set_bucket_location/_object02.yaml b/tests/e2e/testdata/scenarios/powertool_set_bucket_location/_object02.yaml new file mode 100644 index 0000000000..c99b8a5026 --- /dev/null +++ b/tests/e2e/testdata/scenarios/powertool_set_bucket_location/_object02.yaml @@ -0,0 +1,52 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: storage.cnrm.cloud.google.com/v1beta1 +kind: StorageBucket +metadata: + annotations: + cnrm.cloud.google.com/management-conflict-prevention-policy: none + cnrm.cloud.google.com/project-id: ${projectId} + cnrm.cloud.google.com/state-into-spec: merge + finalizers: + - cnrm.cloud.google.com/finalizer + - cnrm.cloud.google.com/deletion-defender + generation: 3 + labels: + label-one: value-one + name: storagebucket-${uniqueId} + namespace: ${projectId} +spec: + lifecycleRule: + - action: + type: Delete + condition: + age: 7 + withState: ANY + location: EU + publicAccessPrevention: inherited + resourceID: storagebucket-${uniqueId} + storageClass: STANDARD + versioning: + enabled: false +status: + conditions: + - lastTransitionTime: "1970-01-01T00:00:00Z" + message: The resource is up to date + reason: UpToDate + status: "True" + type: Ready + observedGeneration: 2 + selfLink: https://www.googleapis.com/storage/v1/b/storagebucket-${uniqueId} + url: gs://storagebucket-${uniqueId} \ No newline at end of file diff --git a/tests/e2e/testdata/scenarios/powertool_set_bucket_location/script.yaml b/tests/e2e/testdata/scenarios/powertool_set_bucket_location/script.yaml new file mode 100644 index 0000000000..2f13efcd23 --- /dev/null +++ b/tests/e2e/testdata/scenarios/powertool_set_bucket_location/script.yaml @@ -0,0 +1,41 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: storage.cnrm.cloud.google.com/v1beta1 +kind: StorageBucket +metadata: + labels: + label-one: "value-one" + name: storagebucket-${uniqueId} +spec: + versioning: + enabled: false + lifecycleRule: + - action: + type: Delete + condition: + age: 7 + +--- + +kind: RunCLI +args: [ "powertools", "force-set-field", "--namespace=${projectId}", "--kind=StorageBucket", "--name=storagebucket-${uniqueId}", "spec.location=EU" ] + +--- + +TEST: READ-OBJECT +apiVersion: storage.cnrm.cloud.google.com/v1beta1 +kind: StorageBucket +metadata: + name: storagebucket-${uniqueId}