diff --git a/.github/workflows/presubmit.yaml b/.github/workflows/presubmit.yaml index 370d46e44f..313a429ec1 100644 --- a/.github/workflows/presubmit.yaml +++ b/.github/workflows/presubmit.yaml @@ -78,6 +78,25 @@ jobs: with: name: artifacts path: /tmp/artifacts/ + pause-tests: + runs-on: ubuntu-22.04 + timeout-minutes: 60 + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-go@v4 + with: + go-version: "1.21.5" + - name: "Run mock tests" + run: | + ./scripts/github-actions/ga-pause-test.sh + env: + GOPATH: /home/runner/go + ARTIFACTS: /tmp/artifacts + - name: "Upload artifacts" + uses: actions/upload-artifact@v3 + with: + name: artifacts + path: /tmp/artifacts/ concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} diff --git a/config/tests/samples/create/samples.go b/config/tests/samples/create/samples.go index 215c74940d..0a744626d2 100644 --- a/config/tests/samples/create/samples.go +++ b/config/tests/samples/create/samples.go @@ -97,6 +97,9 @@ type CreateDeleteTestOptions struct { //nolint:revive // SkipWaitForDelete true means that we don't wait to query that a resource has been deleted. SkipWaitForDelete bool + + // SkipWaitForReady true is mainly used for Paused resources as we don't emit an event for those yet. + SkipWaitForReady bool } func RunCreateDeleteTest(t *Harness, opt CreateDeleteTestOptions) { @@ -109,7 +112,9 @@ func RunCreateDeleteTest(t *Harness, opt CreateDeleteTestOptions) { } } - waitForReady(t, opt.Create) + if !opt.SkipWaitForReady { + waitForReady(t, opt.Create) + } if len(opt.Updates) != 0 { // treat as a patch @@ -118,7 +123,10 @@ func RunCreateDeleteTest(t *Harness, opt CreateDeleteTestOptions) { t.Fatalf("error updating resource: %v", err) } } - waitForReady(t, opt.Updates) + + if !opt.SkipWaitForReady { + waitForReady(t, opt.Updates) + } } // Clean up resources on success if CleanupResources flag is true diff --git a/pkg/controller/dcl/controller.go b/pkg/controller/dcl/controller.go index a4bd36b288..43281c14d3 100644 --- a/pkg/controller/dcl/controller.go +++ b/pkg/controller/dcl/controller.go @@ -21,6 +21,7 @@ import ( "sync" "time" + "github.com/GoogleCloudPlatform/k8s-config-connector/operator/pkg/apis/core/v1beta1" corekccv1alpha1 "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/apis/core/v1alpha1" "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/jitter" "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/lifecyclehandler" @@ -205,6 +206,35 @@ func (r *Reconciler) Reconcile(ctx context.Context, req reconcile.Request) (res if err := r.applyChangesForBackwardsCompatibility(ctx, resource); err != nil { return reconcile.Result{}, fmt.Errorf("error applying changes to resource '%v' for backwards compatibility: %w", k8s.GetNamespacedName(resource), err) } + + cc, ccc, err := resourceactuation.FetchLiveKCCState(ctx, r.mgr.GetClient(), req.NamespacedName) + if err != nil { + return reconcile.Result{}, err + } + + am := resourceactuation.DecideActuationMode(cc, ccc) + switch am { + case v1beta1.Reconciling: + r.logger.V(2).Info("Actuating a resource as actuation mode is \"Reconciling\"", "resource", req.NamespacedName) + case v1beta1.Paused: + jitteredPeriod, err := jitter.GenerateJitteredReenqueuePeriod(r.schemaRef.GVK, nil, r.converter.MetadataLoader, u) + if err != nil { + return reconcile.Result{}, err + } + + if resource.GetDeletionTimestamp().IsZero() { + // add finalizers for deletion defender to make sure we don't delete cloud provider resources when uninstalling + if err := r.EnsureFinalizers(ctx, resource.Original, &resource.Resource, k8s.ControllerFinalizerName, k8s.DeletionDefenderFinalizerName); err != nil { + return reconcile.Result{}, err + } + } + + r.logger.Info("Skipping actuation of resource as actuation mode is \"Paused\"", "resource", req.NamespacedName, "time to next reconciliation", jitteredPeriod) + return reconcile.Result{RequeueAfter: jitteredPeriod}, nil + default: + return reconcile.Result{}, fmt.Errorf("unknown actuation mode %v", am) + } + // Apply pre-actuation transformation. if err := resourceoverrides.Handler.PreActuationTransform(&resource.Resource); err != nil { return reconcile.Result{}, r.HandlePreActuationTransformFailed(ctx, &resource.Resource, fmt.Errorf("error applying pre-actuation transformation to resource '%v': %w", req.NamespacedName.String(), err)) diff --git a/pkg/controller/direct/directbase/directbase_controller.go b/pkg/controller/direct/directbase/directbase_controller.go index 7ac050c099..c93f065813 100644 --- a/pkg/controller/direct/directbase/directbase_controller.go +++ b/pkg/controller/direct/directbase/directbase_controller.go @@ -21,12 +21,14 @@ import ( "strings" "time" + "github.com/GoogleCloudPlatform/k8s-config-connector/operator/pkg/apis/core/v1beta1" kcciamclient "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/iam/iamclient" "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/jitter" "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/lifecyclehandler" "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/metrics" "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/predicate" "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/ratelimiter" + "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/resourceactuation" "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/resourcewatcher" "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/execution" "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/k8s" @@ -180,6 +182,28 @@ func (r *DirectReconciler) Reconcile(ctx context.Context, request reconcile.Requ func (r *reconcileContext) doReconcile(ctx context.Context, u *unstructured.Unstructured) (requeue bool, err error) { logger := log.FromContext(ctx) + cc, ccc, err := resourceactuation.FetchLiveKCCState(ctx, r.Reconciler.Client, r.NamespacedName) + if err != nil { + return true, err + } + + am := resourceactuation.DecideActuationMode(cc, ccc) + switch am { + case v1beta1.Reconciling: + logger.V(2).Info("Actuating a resource as actuation mode is \"Reconciling\"", "resource", r.NamespacedName) + case v1beta1.Paused: + logger.Info("Skipping actuation of resource as actuation mode is \"Paused\"", "resource", r.NamespacedName) + + // add finalizers for deletion defender to make sure we don't delete cloud provider resources when uninstalling + if u.GetDeletionTimestamp().IsZero() { + k8s.EnsureFinalizers(u, k8s.ControllerFinalizerName, k8s.DeletionDefenderFinalizerName) + } + + return false, nil + default: + return false, fmt.Errorf("unknown actuation mode %v", am) + } + adapter, err := r.Reconciler.model.AdapterForObject(ctx, u) if err != nil { return false, r.handleUpdateFailed(ctx, u, err) diff --git a/pkg/controller/iam/auditconfig/iamauditconfig_controller.go b/pkg/controller/iam/auditconfig/iamauditconfig_controller.go index 1ecc3bb1ee..73864ec15a 100644 --- a/pkg/controller/iam/auditconfig/iamauditconfig_controller.go +++ b/pkg/controller/iam/auditconfig/iamauditconfig_controller.go @@ -20,6 +20,7 @@ import ( "fmt" "time" + "github.com/GoogleCloudPlatform/k8s-config-connector/operator/pkg/apis/core/v1beta1" iamv1beta1 "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/apis/iam/v1beta1" condition "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/apis/k8s/v1alpha1" kcciamclient "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/iam/iamclient" @@ -28,6 +29,7 @@ import ( "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/metrics" "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/predicate" "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/ratelimiter" + "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/resourceactuation" "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/resourcewatcher" "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/dcl/conversion" "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/execution" @@ -175,6 +177,29 @@ func (r *Reconciler) handleDefaults(ctx context.Context, auditConfig *iamv1beta1 func (r *reconcileContext) doReconcile(auditConfig *iamv1beta1.IAMAuditConfig) (requeue bool, err error) { defer execution.RecoverWithInternalError(&err) + + cc, ccc, err := resourceactuation.FetchLiveKCCState(r.Ctx, r.Reconciler.Client, r.NamespacedName) + if err != nil { + return true, err + } + + am := resourceactuation.DecideActuationMode(cc, ccc) + switch am { + case v1beta1.Reconciling: + logger.V(2).Info("Actuating a resource as actuation mode is \"Reconciling\"", "resource", r.NamespacedName) + case v1beta1.Paused: + logger.Info("Skipping actuation of resource as actuation mode is \"Paused\"", "resource", r.NamespacedName) + + // add finalizers for deletion defender to make sure we don't delete cloud provider resources when uninstalling + if auditConfig.GetDeletionTimestamp().IsZero() { + k8s.EnsureFinalizers(auditConfig, k8s.ControllerFinalizerName, k8s.DeletionDefenderFinalizerName) + } + + return false, nil + default: + return false, fmt.Errorf("unknown actuation mode %v", am) + } + if !auditConfig.DeletionTimestamp.IsZero() { if !k8s.HasFinalizer(auditConfig, k8s.ControllerFinalizerName) { // Resource has no controller finalizer; no finalization necessary diff --git a/pkg/controller/iam/partialpolicy/iampartialpolicy_controller.go b/pkg/controller/iam/partialpolicy/iampartialpolicy_controller.go index b070e4167f..38bffd5c21 100644 --- a/pkg/controller/iam/partialpolicy/iampartialpolicy_controller.go +++ b/pkg/controller/iam/partialpolicy/iampartialpolicy_controller.go @@ -21,6 +21,7 @@ import ( "reflect" "time" + "github.com/GoogleCloudPlatform/k8s-config-connector/operator/pkg/apis/core/v1beta1" iamv1beta1 "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/apis/iam/v1beta1" condition "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/apis/k8s/v1alpha1" "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/iam/iamclient" @@ -30,6 +31,7 @@ import ( "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/metrics" "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/predicate" "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/ratelimiter" + "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/resourceactuation" "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/resourcewatcher" "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/dcl/conversion" "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/execution" @@ -187,6 +189,29 @@ func (r *ReconcileIAMPartialPolicy) handleDefaults(ctx context.Context, pp *iamv func (r *reconcileContext) doReconcile(pp *iamv1beta1.IAMPartialPolicy) (requeue bool, err error) { defer execution.RecoverWithInternalError(&err) + + cc, ccc, err := resourceactuation.FetchLiveKCCState(r.Ctx, r.Reconciler.Client, r.NamespacedName) + if err != nil { + return true, err + } + + am := resourceactuation.DecideActuationMode(cc, ccc) + switch am { + case v1beta1.Reconciling: + logger.V(2).Info("Actuating a resource as actuation mode is \"Reconciling\"", "resource", r.NamespacedName) + case v1beta1.Paused: + logger.Info("Skipping actuation of resource as actuation mode is \"Paused\"", "resource", r.NamespacedName) + + // add finalizers for deletion defender to make sure we don't delete cloud provider resources when uninstalling + if pp.GetDeletionTimestamp().IsZero() { + k8s.EnsureFinalizers(pp, k8s.ControllerFinalizerName, k8s.DeletionDefenderFinalizerName) + } + + return false, nil + default: + return false, fmt.Errorf("unknown actuation mode %v", am) + } + if !pp.DeletionTimestamp.IsZero() { return r.finalizeDeletion(pp) } diff --git a/pkg/controller/iam/policy/iampolicy_controller.go b/pkg/controller/iam/policy/iampolicy_controller.go index daf7fcde99..c183c64e72 100644 --- a/pkg/controller/iam/policy/iampolicy_controller.go +++ b/pkg/controller/iam/policy/iampolicy_controller.go @@ -20,6 +20,7 @@ import ( "fmt" "time" + "github.com/GoogleCloudPlatform/k8s-config-connector/operator/pkg/apis/core/v1beta1" iamv1beta1 "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/apis/iam/v1beta1" condition "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/apis/k8s/v1alpha1" "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/iam/iamclient" @@ -29,6 +30,7 @@ import ( "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/metrics" "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/predicate" "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/ratelimiter" + "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/resourceactuation" "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/resourcewatcher" "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/dcl/conversion" "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/execution" @@ -186,6 +188,29 @@ func (r *ReconcileIAMPolicy) handleDefaults(ctx context.Context, policy *iamv1be func (r *reconcileContext) doReconcile(policy *iamv1beta1.IAMPolicy) (requeue bool, err error) { defer execution.RecoverWithInternalError(&err) + + cc, ccc, err := resourceactuation.FetchLiveKCCState(r.Ctx, r.Reconciler.Client, r.NamespacedName) + if err != nil { + return true, err + } + + am := resourceactuation.DecideActuationMode(cc, ccc) + switch am { + case v1beta1.Reconciling: + logger.V(2).Info("Actuating a resource as actuation mode is \"Reconciling\"", "resource", r.NamespacedName) + case v1beta1.Paused: + logger.Info("Skipping actuation of resource as actuation mode is \"Paused\"", "resource", r.NamespacedName) + + // add finalizers for deletion defender to make sure we don't delete cloud provider resources when uninstalling + if policy.GetDeletionTimestamp().IsZero() { + k8s.EnsureFinalizers(policy, k8s.ControllerFinalizerName, k8s.DeletionDefenderFinalizerName) + } + + return false, nil + default: + return false, fmt.Errorf("unknown actuation mode %v", am) + } + if !policy.DeletionTimestamp.IsZero() { if !k8s.HasFinalizer(policy, k8s.ControllerFinalizerName) { // Resource has no controller finalizer; no finalization necessary diff --git a/pkg/controller/iam/policymember/iampolicymember_controller.go b/pkg/controller/iam/policymember/iampolicymember_controller.go index 4b2810f073..d5d89d1697 100644 --- a/pkg/controller/iam/policymember/iampolicymember_controller.go +++ b/pkg/controller/iam/policymember/iampolicymember_controller.go @@ -20,6 +20,7 @@ import ( "fmt" "time" + opcorev1beta1 "github.com/GoogleCloudPlatform/k8s-config-connector/operator/pkg/apis/core/v1beta1" "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/apis/iam/v1beta1" iamv1beta1 "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/apis/iam/v1beta1" condition "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/apis/k8s/v1alpha1" @@ -29,6 +30,7 @@ import ( "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/metrics" "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/predicate" kccratelimiter "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/ratelimiter" + "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/resourceactuation" "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/resourcewatcher" "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/dcl/conversion" "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/execution" @@ -188,6 +190,29 @@ func (r *Reconciler) handleDefaults(ctx context.Context, policyMember *iamv1beta func (r *reconcileContext) doReconcile(policyMember *iamv1beta1.IAMPolicyMember) (requeue bool, err error) { defer execution.RecoverWithInternalError(&err) + + cc, ccc, err := resourceactuation.FetchLiveKCCState(r.Ctx, r.Reconciler.Client, r.NamespacedName) + if err != nil { + return true, err + } + + am := resourceactuation.DecideActuationMode(cc, ccc) + switch am { + case opcorev1beta1.Reconciling: + logger.V(2).Info("Actuating a resource as actuation mode is \"Reconciling\"", "resource", r.NamespacedName) + case opcorev1beta1.Paused: + logger.Info("Skipping actuation of resource as actuation mode is \"Paused\"", "resource", r.NamespacedName) + + // add finalizers for deletion defender to make sure we don't delete cloud provider resources when uninstalling + if policyMember.GetDeletionTimestamp().IsZero() { + k8s.EnsureFinalizers(policyMember, k8s.ControllerFinalizerName, k8s.DeletionDefenderFinalizerName) + } + + return false, nil + default: + return false, fmt.Errorf("unknown actuation mode %v", am) + } + if !policyMember.DeletionTimestamp.IsZero() { if !k8s.HasFinalizer(policyMember, k8s.ControllerFinalizerName) { // Resource has no controller finalizer; no finalization necessary diff --git a/pkg/controller/tf/controller.go b/pkg/controller/tf/controller.go index 008a9c53ff..b56037d405 100644 --- a/pkg/controller/tf/controller.go +++ b/pkg/controller/tf/controller.go @@ -200,7 +200,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req reconcile.Request) (res return reconcile.Result{}, err } - // add finalizers for deletion defender + // add finalizers for deletion defender to make sure we don't delete cloud provider resources when uninstalling if resource.GetDeletionTimestamp().IsZero() { if err := r.EnsureFinalizers(ctx, resource.Original, &resource.Resource, k8s.ControllerFinalizerName, k8s.DeletionDefenderFinalizerName); err != nil { return reconcile.Result{}, err diff --git a/scripts/github-actions/ga-pause-test.sh b/scripts/github-actions/ga-pause-test.sh new file mode 100755 index 0000000000..1c04464e82 --- /dev/null +++ b/scripts/github-actions/ga-pause-test.sh @@ -0,0 +1,29 @@ +#!/bin/bash +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -o errexit +set -o nounset +set -o pipefail +REPO_ROOT="$(git rev-parse --show-toplevel)" +source ${REPO_ROOT}/scripts/shared-vars-public.sh +cd ${REPO_ROOT} +source ${REPO_ROOT}/scripts/fetch_ext_bins.sh && \ + fetch_tools && \ + setup_envs + +cd ${REPO_ROOT}/ +echo "Running mock e2e pause tests..." +E2E_KUBE_TARGET=envtest \ + RUN_E2E=1 GOLDEN_REQUEST_CHECKS=1 E2E_GCP_TARGET=mock \ + go test -test.count=1 -timeout 3600s -v ./tests/e2e -run TestPauseInSeries 2>&1 \ No newline at end of file diff --git a/tests/e2e/unified_test.go b/tests/e2e/unified_test.go index a0f6d4c8a7..aa33dfbeae 100644 --- a/tests/e2e/unified_test.go +++ b/tests/e2e/unified_test.go @@ -23,6 +23,7 @@ import ( "testing" "github.com/GoogleCloudPlatform/k8s-config-connector/config/tests/samples/create" + opcorev1beta1 "github.com/GoogleCloudPlatform/k8s-config-connector/operator/pkg/apis/core/v1beta1" "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/cli/cmd/export" "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/test" testcontroller "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/test/controller" @@ -30,7 +31,6 @@ import ( "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/test/resourcefixture" testvariable "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/test/resourcefixture/variable" testyaml "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/test/yaml" - opcorev1beta1 "github.com/GoogleCloudPlatform/k8s-config-connector/operator/pkg/apis/core/v1beta1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" @@ -160,6 +160,9 @@ func testFixturesInSeries(ctx context.Context, t *testing.T, testName string, te create.SetupNamespacesAndApplyDefaults(h, opt.Create, project) opt.CleanupResources = false // We delete explicitly below + if testPause { + opt.SkipWaitForReady = true // Paused resources don't send out an event yet. + } create.RunCreateDeleteTest(h, opt) for _, exportResource := range exportResources { @@ -387,11 +390,11 @@ func assertNoRequest(t *testing.T, got string, normalizers ...func(s string) str got = normalizer(got) } - if strings.Contains(got,"POST") { + if strings.Contains(got, "POST") { t.Fatalf("unexpected POST in log: %s", got) } - if strings.Contains(got,"GET") { + if strings.Contains(got, "GET") { t.Fatalf("unexpected GET in log: %s", got) } } @@ -402,7 +405,6 @@ func bytesToUnstructured(t *testing.T, bytes []byte, testID string, project test return test.ToUnstructWithNamespace(t, updatedBytes, testID) } - func createPausedCC(ctx context.Context, t *testing.T, c client.Client) { t.Helper()