Merge pull request GoogleCloudPlatform#1348 from acpana/acpana/more-p…

…ause

feat: more controllers supporting pause

.github/workflows/presubmit.yaml

@@ -78,6 +78,25 @@ jobs:
         with:
           name: artifacts
           path: /tmp/artifacts/
+  pause-tests:
+    runs-on: ubuntu-22.04
+    timeout-minutes: 60
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-go@v4
+        with:
+          go-version: "1.21.5"
+      - name: "Run mock tests"
+        run: |
+          ./scripts/github-actions/ga-pause-test.sh
+        env:
+          GOPATH: /home/runner/go
+          ARTIFACTS: /tmp/artifacts
+      - name: "Upload artifacts"
+        uses: actions/upload-artifact@v3
+        with:
+          name: artifacts
+          path: /tmp/artifacts/
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}

config/tests/samples/create/samples.go

@@ -97,6 +97,9 @@ type CreateDeleteTestOptions struct { //nolint:revive
 
 	// SkipWaitForDelete true means that we don't wait to query that a resource has been deleted.
 	SkipWaitForDelete bool
+
+	// SkipWaitForReady true is mainly used for Paused resources as we don't emit an event for those yet.
+	SkipWaitForReady bool
 }
 
 func RunCreateDeleteTest(t *Harness, opt CreateDeleteTestOptions) {
@@ -109,7 +112,9 @@ func RunCreateDeleteTest(t *Harness, opt CreateDeleteTestOptions) {
 		}
 	}
 
-	waitForReady(t, opt.Create)
+	if !opt.SkipWaitForReady {
+		waitForReady(t, opt.Create)
+	}
 
 	if len(opt.Updates) != 0 {
 		// treat as a patch
@@ -118,7 +123,10 @@ func RunCreateDeleteTest(t *Harness, opt CreateDeleteTestOptions) {
 				t.Fatalf("error updating resource: %v", err)
 			}
 		}
-		waitForReady(t, opt.Updates)
+
+		if !opt.SkipWaitForReady {
+			waitForReady(t, opt.Updates)
+		}
 	}
 
 	// Clean up resources on success if CleanupResources flag is true

pkg/controller/dcl/controller.go

@@ -21,6 +21,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/GoogleCloudPlatform/k8s-config-connector/operator/pkg/apis/core/v1beta1"
 	corekccv1alpha1 "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/apis/core/v1alpha1"
 	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/jitter"
 	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/lifecyclehandler"
@@ -205,6 +206,35 @@ func (r *Reconciler) Reconcile(ctx context.Context, req reconcile.Request) (res
 	if err := r.applyChangesForBackwardsCompatibility(ctx, resource); err != nil {
 		return reconcile.Result{}, fmt.Errorf("error applying changes to resource '%v' for backwards compatibility: %w", k8s.GetNamespacedName(resource), err)
 	}
+
+	cc, ccc, err := resourceactuation.FetchLiveKCCState(ctx, r.mgr.GetClient(), req.NamespacedName)
+	if err != nil {
+		return reconcile.Result{}, err
+	}
+
+	am := resourceactuation.DecideActuationMode(cc, ccc)
+	switch am {
+	case v1beta1.Reconciling:
+		r.logger.V(2).Info("Actuating a resource as actuation mode is \"Reconciling\"", "resource", req.NamespacedName)
+	case v1beta1.Paused:
+		jitteredPeriod, err := jitter.GenerateJitteredReenqueuePeriod(r.schemaRef.GVK, nil, r.converter.MetadataLoader, u)
+		if err != nil {
+			return reconcile.Result{}, err
+		}
+
+		if resource.GetDeletionTimestamp().IsZero() {
+			// add finalizers for deletion defender to make sure we don't delete cloud provider resources when uninstalling
+			if err := r.EnsureFinalizers(ctx, resource.Original, &resource.Resource, k8s.ControllerFinalizerName, k8s.DeletionDefenderFinalizerName); err != nil {
+				return reconcile.Result{}, err
+			}
+		}
+
+		r.logger.Info("Skipping actuation of resource as actuation mode is \"Paused\"", "resource", req.NamespacedName, "time to next reconciliation", jitteredPeriod)
+		return reconcile.Result{RequeueAfter: jitteredPeriod}, nil
+	default:
+		return reconcile.Result{}, fmt.Errorf("unknown actuation mode %v", am)
+	}
+
 	// Apply pre-actuation transformation.
 	if err := resourceoverrides.Handler.PreActuationTransform(&resource.Resource); err != nil {
 		return reconcile.Result{}, r.HandlePreActuationTransformFailed(ctx, &resource.Resource, fmt.Errorf("error applying pre-actuation transformation to resource '%v': %w", req.NamespacedName.String(), err))

pkg/controller/direct/directbase/directbase_controller.go

@@ -21,12 +21,14 @@ import (
 	"strings"
 	"time"
 
+	"github.com/GoogleCloudPlatform/k8s-config-connector/operator/pkg/apis/core/v1beta1"
 	kcciamclient "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/iam/iamclient"
 	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/jitter"
 	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/lifecyclehandler"
 	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/metrics"
 	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/predicate"
 	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/ratelimiter"
+	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/resourceactuation"
 	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/resourcewatcher"
 	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/execution"
 	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/k8s"
@@ -180,6 +182,28 @@ func (r *DirectReconciler) Reconcile(ctx context.Context, request reconcile.Requ
 func (r *reconcileContext) doReconcile(ctx context.Context, u *unstructured.Unstructured) (requeue bool, err error) {
 	logger := log.FromContext(ctx)
 
+	cc, ccc, err := resourceactuation.FetchLiveKCCState(ctx, r.Reconciler.Client, r.NamespacedName)
+	if err != nil {
+		return true, err
+	}
+
+	am := resourceactuation.DecideActuationMode(cc, ccc)
+	switch am {
+	case v1beta1.Reconciling:
+		logger.V(2).Info("Actuating a resource as actuation mode is \"Reconciling\"", "resource", r.NamespacedName)
+	case v1beta1.Paused:
+		logger.Info("Skipping actuation of resource as actuation mode is \"Paused\"", "resource", r.NamespacedName)
+
+		// add finalizers for deletion defender to make sure we don't delete cloud provider resources when uninstalling
+		if u.GetDeletionTimestamp().IsZero() {
+			k8s.EnsureFinalizers(u, k8s.ControllerFinalizerName, k8s.DeletionDefenderFinalizerName)
+		}
+
+		return false, nil
+	default:
+		return false, fmt.Errorf("unknown actuation mode %v", am)
+	}
+
 	adapter, err := r.Reconciler.model.AdapterForObject(ctx, u)
 	if err != nil {
 		return false, r.handleUpdateFailed(ctx, u, err)

pkg/controller/iam/auditconfig/iamauditconfig_controller.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"time"
 
+	"github.com/GoogleCloudPlatform/k8s-config-connector/operator/pkg/apis/core/v1beta1"
 	iamv1beta1 "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/apis/iam/v1beta1"
 	condition "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/apis/k8s/v1alpha1"
 	kcciamclient "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/iam/iamclient"
@@ -28,6 +29,7 @@ import (
 	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/metrics"
 	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/predicate"
 	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/ratelimiter"
+	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/resourceactuation"
 	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/resourcewatcher"
 	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/dcl/conversion"
 	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/execution"
@@ -175,6 +177,29 @@ func (r *Reconciler) handleDefaults(ctx context.Context, auditConfig *iamv1beta1
 
 func (r *reconcileContext) doReconcile(auditConfig *iamv1beta1.IAMAuditConfig) (requeue bool, err error) {
 	defer execution.RecoverWithInternalError(&err)
+
+	cc, ccc, err := resourceactuation.FetchLiveKCCState(r.Ctx, r.Reconciler.Client, r.NamespacedName)
+	if err != nil {
+		return true, err
+	}
+
+	am := resourceactuation.DecideActuationMode(cc, ccc)
+	switch am {
+	case v1beta1.Reconciling:
+		logger.V(2).Info("Actuating a resource as actuation mode is \"Reconciling\"", "resource", r.NamespacedName)
+	case v1beta1.Paused:
+		logger.Info("Skipping actuation of resource as actuation mode is \"Paused\"", "resource", r.NamespacedName)
+
+		// add finalizers for deletion defender to make sure we don't delete cloud provider resources when uninstalling
+		if auditConfig.GetDeletionTimestamp().IsZero() {
+			k8s.EnsureFinalizers(auditConfig, k8s.ControllerFinalizerName, k8s.DeletionDefenderFinalizerName)
+		}
+
+		return false, nil
+	default:
+		return false, fmt.Errorf("unknown actuation mode %v", am)
+	}
+
 	if !auditConfig.DeletionTimestamp.IsZero() {
 		if !k8s.HasFinalizer(auditConfig, k8s.ControllerFinalizerName) {
 			// Resource has no controller finalizer; no finalization necessary

pkg/controller/iam/partialpolicy/iampartialpolicy_controller.go

@@ -21,6 +21,7 @@ import (
 	"reflect"
 	"time"
 
+	"github.com/GoogleCloudPlatform/k8s-config-connector/operator/pkg/apis/core/v1beta1"
 	iamv1beta1 "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/apis/iam/v1beta1"
 	condition "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/apis/k8s/v1alpha1"
 	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/iam/iamclient"
@@ -30,6 +31,7 @@ import (
 	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/metrics"
 	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/predicate"
 	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/ratelimiter"
+	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/resourceactuation"
 	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/resourcewatcher"
 	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/dcl/conversion"
 	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/execution"
@@ -187,6 +189,29 @@ func (r *ReconcileIAMPartialPolicy) handleDefaults(ctx context.Context, pp *iamv
 
 func (r *reconcileContext) doReconcile(pp *iamv1beta1.IAMPartialPolicy) (requeue bool, err error) {
 	defer execution.RecoverWithInternalError(&err)
+
+	cc, ccc, err := resourceactuation.FetchLiveKCCState(r.Ctx, r.Reconciler.Client, r.NamespacedName)
+	if err != nil {
+		return true, err
+	}
+
+	am := resourceactuation.DecideActuationMode(cc, ccc)
+	switch am {
+	case v1beta1.Reconciling:
+		logger.V(2).Info("Actuating a resource as actuation mode is \"Reconciling\"", "resource", r.NamespacedName)
+	case v1beta1.Paused:
+		logger.Info("Skipping actuation of resource as actuation mode is \"Paused\"", "resource", r.NamespacedName)
+
+		// add finalizers for deletion defender to make sure we don't delete cloud provider resources when uninstalling
+		if pp.GetDeletionTimestamp().IsZero() {
+			k8s.EnsureFinalizers(pp, k8s.ControllerFinalizerName, k8s.DeletionDefenderFinalizerName)
+		}
+
+		return false, nil
+	default:
+		return false, fmt.Errorf("unknown actuation mode %v", am)
+	}
+
 	if !pp.DeletionTimestamp.IsZero() {
 		return r.finalizeDeletion(pp)
 	}

pkg/controller/iam/policy/iampolicy_controller.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"time"
 
+	"github.com/GoogleCloudPlatform/k8s-config-connector/operator/pkg/apis/core/v1beta1"
 	iamv1beta1 "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/apis/iam/v1beta1"
 	condition "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/apis/k8s/v1alpha1"
 	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/iam/iamclient"
@@ -29,6 +30,7 @@ import (
 	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/metrics"
 	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/predicate"
 	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/ratelimiter"
+	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/resourceactuation"
 	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/resourcewatcher"
 	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/dcl/conversion"
 	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/execution"
@@ -186,6 +188,29 @@ func (r *ReconcileIAMPolicy) handleDefaults(ctx context.Context, policy *iamv1be
 
 func (r *reconcileContext) doReconcile(policy *iamv1beta1.IAMPolicy) (requeue bool, err error) {
 	defer execution.RecoverWithInternalError(&err)
+
+	cc, ccc, err := resourceactuation.FetchLiveKCCState(r.Ctx, r.Reconciler.Client, r.NamespacedName)
+	if err != nil {
+		return true, err
+	}
+
+	am := resourceactuation.DecideActuationMode(cc, ccc)
+	switch am {
+	case v1beta1.Reconciling:
+		logger.V(2).Info("Actuating a resource as actuation mode is \"Reconciling\"", "resource", r.NamespacedName)
+	case v1beta1.Paused:
+		logger.Info("Skipping actuation of resource as actuation mode is \"Paused\"", "resource", r.NamespacedName)
+
+		// add finalizers for deletion defender to make sure we don't delete cloud provider resources when uninstalling
+		if policy.GetDeletionTimestamp().IsZero() {
+			k8s.EnsureFinalizers(policy, k8s.ControllerFinalizerName, k8s.DeletionDefenderFinalizerName)
+		}
+
+		return false, nil
+	default:
+		return false, fmt.Errorf("unknown actuation mode %v", am)
+	}
+
 	if !policy.DeletionTimestamp.IsZero() {
 		if !k8s.HasFinalizer(policy, k8s.ControllerFinalizerName) {
 			// Resource has no controller finalizer; no finalization necessary

pkg/controller/iam/policymember/iampolicymember_controller.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"time"
 
+	opcorev1beta1 "github.com/GoogleCloudPlatform/k8s-config-connector/operator/pkg/apis/core/v1beta1"
 	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/apis/iam/v1beta1"
 	iamv1beta1 "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/apis/iam/v1beta1"
 	condition "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/apis/k8s/v1alpha1"
@@ -29,6 +30,7 @@ import (
 	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/metrics"
 	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/predicate"
 	kccratelimiter "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/ratelimiter"
+	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/resourceactuation"
 	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/resourcewatcher"
 	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/dcl/conversion"
 	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/execution"
@@ -188,6 +190,29 @@ func (r *Reconciler) handleDefaults(ctx context.Context, policyMember *iamv1beta
 
 func (r *reconcileContext) doReconcile(policyMember *iamv1beta1.IAMPolicyMember) (requeue bool, err error) {
 	defer execution.RecoverWithInternalError(&err)
+
+	cc, ccc, err := resourceactuation.FetchLiveKCCState(r.Ctx, r.Reconciler.Client, r.NamespacedName)
+	if err != nil {
+		return true, err
+	}
+
+	am := resourceactuation.DecideActuationMode(cc, ccc)
+	switch am {
+	case opcorev1beta1.Reconciling:
+		logger.V(2).Info("Actuating a resource as actuation mode is \"Reconciling\"", "resource", r.NamespacedName)
+	case opcorev1beta1.Paused:
+		logger.Info("Skipping actuation of resource as actuation mode is \"Paused\"", "resource", r.NamespacedName)
+
+		// add finalizers for deletion defender to make sure we don't delete cloud provider resources when uninstalling
+		if policyMember.GetDeletionTimestamp().IsZero() {
+			k8s.EnsureFinalizers(policyMember, k8s.ControllerFinalizerName, k8s.DeletionDefenderFinalizerName)
+		}
+
+		return false, nil
+	default:
+		return false, fmt.Errorf("unknown actuation mode %v", am)
+	}
+
 	if !policyMember.DeletionTimestamp.IsZero() {
 		if !k8s.HasFinalizer(policyMember, k8s.ControllerFinalizerName) {
 			// Resource has no controller finalizer; no finalization necessary

pkg/controller/tf/controller.go

@@ -200,7 +200,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req reconcile.Request) (res
 			return reconcile.Result{}, err
 		}
 
-		// add finalizers for deletion defender
+		// add finalizers for deletion defender to make sure we don't delete cloud provider resources when uninstalling
 		if resource.GetDeletionTimestamp().IsZero() {
 			if err := r.EnsureFinalizers(ctx, resource.Original, &resource.Resource, k8s.ControllerFinalizerName, k8s.DeletionDefenderFinalizerName); err != nil {
 				return reconcile.Result{}, err

scripts/github-actions/ga-pause-test.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+set -o errexit
+set -o nounset
+set -o pipefail
+REPO_ROOT="$(git rev-parse --show-toplevel)"
+source ${REPO_ROOT}/scripts/shared-vars-public.sh
+cd ${REPO_ROOT}
+source ${REPO_ROOT}/scripts/fetch_ext_bins.sh && \
+	fetch_tools && \
+	setup_envs
+
+cd ${REPO_ROOT}/
+echo "Running mock e2e pause tests..."
+E2E_KUBE_TARGET=envtest \
+	RUN_E2E=1 GOLDEN_REQUEST_CHECKS=1 E2E_GCP_TARGET=mock \
+	go test -test.count=1 -timeout 3600s -v ./tests/e2e -run TestPauseInSeries 2>&1