Skip to content

Commit

Permalink
Merge pull request GoogleCloudPlatform#2559 from jasonvigil/cloudsql-…
Browse files Browse the repository at this point in the history
…rolloutgate

Add predicate to partially enable SQLInstance direct reconciler
  • Loading branch information
google-oss-prow[bot] authored Aug 27, 2024
2 parents 031aac4 + f5f22cf commit ec9aceb
Show file tree
Hide file tree
Showing 9 changed files with 229 additions and 14 deletions.
2 changes: 1 addition & 1 deletion apis/sql/v1beta1/sqlinstance_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -595,7 +595,7 @@ type SQLInstanceStatus struct {
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
// +kubebuilder:resource:categories=gcp,shortName=gcpsqlinstance;gcpsqlinstances
// +kubebuilder:subresource:status
// +kubebuilder:metadata:labels="cnrm.cloud.google.com/managed-by-kcc=true";"cnrm.cloud.google.com/stability-level=stable";"cnrm.cloud.google.com/system=true";"cnrm.cloud.google.com/tf2crd=true"
// +kubebuilder:metadata:labels="cnrm.cloud.google.com/managed-by-kcc=true";"cnrm.cloud.google.com/stability-level=stable";"cnrm.cloud.google.com/system=true"
// +kubebuilder:printcolumn:name="Age",JSONPath=".metadata.creationTimestamp",type="date"
// +kubebuilder:printcolumn:name="Ready",JSONPath=".status.conditions[?(@.type=='Ready')].status",type="string",description="When 'True', the most recent reconcile of the resource succeeded"
// +kubebuilder:printcolumn:name="Status",JSONPath=".status.conditions[?(@.type=='Ready')].reason",type="string",description="The reason for the value in 'Ready'"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ metadata:
cnrm.cloud.google.com/managed-by-kcc: "true"
cnrm.cloud.google.com/stability-level: stable
cnrm.cloud.google.com/system: "true"
cnrm.cloud.google.com/tf2crd: "true"
name: sqlinstances.sql.cnrm.cloud.google.com
spec:
group: sql.cnrm.cloud.google.com
Expand Down

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

17 changes: 12 additions & 5 deletions pkg/controller/direct/directbase/directbase_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ import (
"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/jitter"
"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/lifecyclehandler"
"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/metrics"
"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/predicate"
kccpredicate "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/predicate"
"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/ratelimiter"
"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/resourceactuation"
"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/resourcewatcher"
Expand All @@ -48,6 +48,7 @@ import (
"sigs.k8s.io/controller-runtime/pkg/handler"
"sigs.k8s.io/controller-runtime/pkg/log"
"sigs.k8s.io/controller-runtime/pkg/manager"
"sigs.k8s.io/controller-runtime/pkg/predicate"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
"sigs.k8s.io/controller-runtime/pkg/source"
)
Expand All @@ -64,7 +65,7 @@ func AddController(mgr manager.Manager, gvk schema.GroupVersionKind, model Model
if err != nil {
return err
}
return add(mgr, reconciler)
return add(mgr, reconciler, deps.ReconcilePredicate)
}

// NewReconciler returns a new reconcile.Reconciler.
Expand Down Expand Up @@ -97,16 +98,21 @@ func NewReconciler(mgr manager.Manager, immediateReconcileRequests chan event.Ge
}

// add adds a new Controller to mgr with r as the reconcile.Reconciler.
func add(mgr manager.Manager, r *DirectReconciler) error {
func add(mgr manager.Manager, r *DirectReconciler, reconcilePredicate predicate.Predicate) error {
obj := &unstructured.Unstructured{}
obj.SetGroupVersionKind(r.gvk)

predicateList := []predicate.Predicate{kccpredicate.UnderlyingResourceOutOfSyncPredicate{}}
if reconcilePredicate != nil {
predicateList = append(predicateList, reconcilePredicate)
}

_, err := builder.
ControllerManagedBy(mgr).
Named(r.controllerName).
WithOptions(crcontroller.Options{MaxConcurrentReconciles: k8s.ControllerMaxConcurrentReconciles, RateLimiter: ratelimiter.NewRateLimiter()}).
WatchesRawSource(&source.Channel{Source: r.immediateReconcileRequests}, &handler.EnqueueRequestForObject{}).
For(obj, builder.OnlyMetadata, builder.WithPredicates(predicate.UnderlyingResourceOutOfSyncPredicate{})).
For(obj, builder.OnlyMetadata, builder.WithPredicates(predicateList...)).
Build(r)
if err != nil {
return fmt.Errorf("error creating new controller: %w", err)
Expand All @@ -117,7 +123,8 @@ func add(mgr manager.Manager, r *DirectReconciler) error {
var _ reconcile.Reconciler = &DirectReconciler{}

type Deps struct {
JitterGenerator jitter.Generator
JitterGenerator jitter.Generator
ReconcilePredicate predicate.Predicate
}

// DirectReconciler is a reconciler for reconciling resources that support the Model/Adapter pattern.
Expand Down
19 changes: 19 additions & 0 deletions pkg/controller/direct/registry/registry.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@ import (

"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/config"
"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/direct/directbase"
"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/predicate"

"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/klog/v2"
)
Expand All @@ -34,6 +36,7 @@ type registration struct {
gvk schema.GroupVersionKind
factory ModelFactoryFunc
model directbase.Model
rg predicate.ReconcileGate
}

type ModelFactoryFunc func(ctx context.Context, config *config.ControllerConfig) (directbase.Model, error)
Expand All @@ -46,6 +49,11 @@ func GetModel(gk schema.GroupKind) (directbase.Model, error) {
return registration.model, nil
}

func GetReconcileGate(gk schema.GroupKind) predicate.ReconcileGate {
registration := singleton.registrations[gk]
return registration.rg
}

func PreferredGVK(gk schema.GroupKind) (schema.GroupVersionKind, bool) {
registration := singleton.registrations[gk]
if registration == nil {
Expand Down Expand Up @@ -93,6 +101,17 @@ func RegisterModel(gvk schema.GroupVersionKind, modelFn ModelFactoryFunc) {
}
}

func RegisterModelWithReconcileGate(gvk schema.GroupVersionKind, modelFn ModelFactoryFunc, rg predicate.ReconcileGate) {
if singleton.registrations == nil {
singleton.registrations = make(map[schema.GroupKind]*registration)
}
singleton.registrations[gvk.GroupKind()] = &registration{
gvk: gvk,
factory: modelFn,
rg: rg,
}
}

func IsDirectByGK(gk schema.GroupKind) bool {
registration := singleton.registrations[gk]
return registration != nil
Expand Down
17 changes: 16 additions & 1 deletion pkg/controller/direct/sql/sqlinstance_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,14 +31,29 @@ import (
"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/config"
"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/direct/directbase"
"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/direct/registry"
kccpredicate "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/predicate"
"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/k8s"
"github.com/googleapis/gax-go/v2"
)

const ctrlName = "sqlinstance-controller"

func init() {
registry.RegisterModel(krm.SQLInstanceGVK, newSQLInstanceModel)
rg := &SQLInstanceReconcileGate{}
registry.RegisterModelWithReconcileGate(krm.SQLInstanceGVK, newSQLInstanceModel, rg)
}

type SQLInstanceReconcileGate struct{}

var _ kccpredicate.ReconcileGate = &SQLInstanceReconcileGate{}

func (*SQLInstanceReconcileGate) ShouldReconcile(o *unstructured.Unstructured) bool {
obj := &krm.SQLInstance{}
if err := runtime.DefaultUnstructuredConverter.FromUnstructured(o.Object, &obj); err != nil {
return false
}
// Run the direct reconciler only when spec.cloneSource is specified
return obj.Spec.CloneSource != nil
}

func newSQLInstanceModel(ctx context.Context, config *config.ControllerConfig) (directbase.Model, error) {
Expand Down
149 changes: 149 additions & 0 deletions pkg/controller/predicate/reconcilegate.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,149 @@
// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package predicate

import (
"context"

"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/types"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/event"
k8spredicate "sigs.k8s.io/controller-runtime/pkg/predicate"
)

// ReconcileGate allows controllers to select which resources they are enabled for, based on
// features of the resource to-be-reconciled. This allows for partially enabling a controller.
type ReconcileGate interface {
// ShouldReconcile returns true if the reconciler should be used to for the resource.
ShouldReconcile(o *unstructured.Unstructured) bool
}

// ReconcilePredicate generates a controller-runtime predicate based on a ReconcileGate.
type ReconcilePredicate struct {
gvk schema.GroupVersionKind
rg ReconcileGate
c client.Client
}

var _ k8spredicate.Predicate = &ReconcilePredicate{}

func NewReconcilePredicate(c client.Client, gvk schema.GroupVersionKind, rg ReconcileGate) *ReconcilePredicate {
return &ReconcilePredicate{
c: c,
gvk: gvk,
rg: rg,
}
}

func (p *ReconcilePredicate) Create(e event.CreateEvent) bool {
obj, err := getUnstructuredObjWithGVK(p.c, e.Object, p.gvk)
if err != nil {
return false
}
return p.rg.ShouldReconcile(obj)
}

func (p *ReconcilePredicate) Delete(e event.DeleteEvent) bool {
obj, err := getUnstructuredObjWithGVK(p.c, e.Object, p.gvk)
if err != nil {
return false
}
return p.rg.ShouldReconcile(obj)
}

func (p *ReconcilePredicate) Update(e event.UpdateEvent) bool {
obj, err := getUnstructuredObjWithGVK(p.c, e.ObjectNew, p.gvk)
if err != nil {
return false
}
return p.rg.ShouldReconcile(obj)
}

func (p *ReconcilePredicate) Generic(e event.GenericEvent) bool {
obj, err := getUnstructuredObjWithGVK(p.c, e.Object, p.gvk)
if err != nil {
return false
}
return p.rg.ShouldReconcile(obj)
}

// InverseReconcilePredicate generates a controller-runtime predicate based on the inverse of a ReconcileGate.
type InverseReconcilePredicate struct {
gvk schema.GroupVersionKind
rg ReconcileGate
c client.Client
}

var _ k8spredicate.Predicate = &InverseReconcilePredicate{}

func NewInverseReconcilePredicate(c client.Client, gvk schema.GroupVersionKind, rg ReconcileGate) *InverseReconcilePredicate {
return &InverseReconcilePredicate{
c: c,
gvk: gvk,
rg: rg,
}
}

func (p *InverseReconcilePredicate) Create(e event.CreateEvent) bool {
obj, err := getUnstructuredObjWithGVK(p.c, e.Object, p.gvk)
if err != nil {
return false
}
return !p.rg.ShouldReconcile(obj)
}

func (p *InverseReconcilePredicate) Delete(e event.DeleteEvent) bool {
obj, err := getUnstructuredObjWithGVK(p.c, e.Object, p.gvk)
if err != nil {
return false
}
return !p.rg.ShouldReconcile(obj)
}

func (p *InverseReconcilePredicate) Update(e event.UpdateEvent) bool {
obj, err := getUnstructuredObjWithGVK(p.c, e.ObjectNew, p.gvk)
if err != nil {
return false
}
return !p.rg.ShouldReconcile(obj)
}

func (p *InverseReconcilePredicate) Generic(e event.GenericEvent) bool {
obj, err := getUnstructuredObjWithGVK(p.c, e.Object, p.gvk)
if err != nil {
return false
}
return !p.rg.ShouldReconcile(obj)
}

// getUnstructuredObjWithGVK uses the provided client to fetch an object (as unstructured) with a given GVK.
// This helper fn is necessary for the ReconcileGate predicates because both the direct and terraform
// controllers specify the option `builder.OnlyMetadata` to only cache metadata about the objects they are
// watching. Therefore, the objects provided to the predicate fns are only `metav1.PartialObjectMetadata`
// types, and do not contain any of the spec values.
func getUnstructuredObjWithGVK(c client.Client, o client.Object, gvk schema.GroupVersionKind) (*unstructured.Unstructured, error) {
obj := &unstructured.Unstructured{}
obj.SetGroupVersionKind(gvk)
key := types.NamespacedName{
Name: o.GetName(),
Namespace: o.GetNamespace(),
}
if err := c.Get(context.Background(), key, obj); err != nil {
return nil, err
}
return obj, nil
}
25 changes: 23 additions & 2 deletions pkg/controller/registration/registration_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ import (
"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/iam/policy"
"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/iam/policymember"
"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/jitter"
kccpredicate "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/predicate"
"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/tf"
"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/unmanageddetector"
"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/crd/crdgeneration"
Expand Down Expand Up @@ -247,7 +248,7 @@ func registerDefaultController(r *ReconcileRegistration, config *config.Controll
}
// register controllers for tf-based CRDs
if val, ok := crd.Labels[crdgeneration.TF2CRDLabel]; ok && val == "true" {
su, err := tf.Add(r.mgr, crd, r.provider, r.smLoader, r.defaulters, r.jitterGenerator)
su, err := tf.Add(r.mgr, crd, r.provider, r.smLoader, r.defaulters, r.jitterGenerator, nil)
if err != nil {
return nil, fmt.Errorf("error adding terraform controller for %v to a manager: %w", crd.Spec.Names.Kind, err)
}
Expand All @@ -259,9 +260,29 @@ func registerDefaultController(r *ReconcileRegistration, config *config.Controll
if err != nil {
return nil, err
}
if err := directbase.AddController(r.mgr, gvk, model, directbase.Deps{JitterGenerator: r.jitterGenerator}); err != nil {
deps := directbase.Deps{
JitterGenerator: r.jitterGenerator,
}
rg := registry.GetReconcileGate(gvk.GroupKind())
if rg != nil {
// If reconcile gate is enabled for this gvk, generate a controller-runtime predicate that will
// run the direct reconciler only when the reconcile gate returns true.
rp := kccpredicate.NewReconcilePredicate(r.mgr.GetClient(), gvk, rg)
deps.ReconcilePredicate = rp
}
if err := directbase.AddController(r.mgr, gvk, model, deps); err != nil {
return nil, fmt.Errorf("error adding direct controller for %v to a manager: %w", crd.Spec.Names.Kind, err)
}
if rg != nil {
// If reconcile gate is enabled for this gvk, generate a controller-runtime predicate that will
// run the terraform-based reconciler when the reconcile gate returns false.
irp := kccpredicate.NewInverseReconcilePredicate(r.mgr.GetClient(), gvk, rg)
su, err := tf.Add(r.mgr, crd, r.provider, r.smLoader, r.defaulters, r.jitterGenerator, irp)
if err != nil {
return nil, fmt.Errorf("error adding terraform controller for %v to a manager: %w", crd.Spec.Names.Kind, err)
}
return su, nil
}
return schemaUpdater, nil
}
logger.Error(fmt.Errorf("unrecognized CRD: %v", crd.Spec.Names.Kind), "skipping controller registration", "group", gvk.Group, "version", gvk.Version, "kind", gvk.Kind)
Expand Down
Loading

0 comments on commit ec9aceb

Please sign in to comment.