Skip to content

Commit

Permalink
dataplane konnect extension reconciler (#714)
Browse files Browse the repository at this point in the history
* feat: DataPlaneKonnectExtension customization

Signed-off-by: Mattia Lavacca <[email protected]>

---------

Signed-off-by: Mattia Lavacca <[email protected]>
Co-authored-by: Grzegorz Burzyński <[email protected]>
  • Loading branch information
mlavacca and czeslavo authored Oct 10, 2024
1 parent 7e72f7f commit 0780829
Show file tree
Hide file tree
Showing 24 changed files with 899 additions and 65 deletions.
3 changes: 3 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,9 @@
[#669](https://github.com/Kong/gateway-operator/pull/669)
- Allow setting `KonnectGatewayControlPlane`s group membership
[#697](https://github.com/Kong/gateway-operator/pull/697)
- Apply Konnect-related customizations to `DataPlane`s that properly reference `DataPlaneKonnectExtension`
resources.
[#714](https://github.com/Kong/gateway-operator/pull/714)

### Fixed

Expand Down
10 changes: 8 additions & 2 deletions api/v1alpha1/dataplane_konnect_extension_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,11 @@ func init() {
SchemeBuilder.Register(&DataPlaneKonnectExtension{}, &DataPlaneKonnectExtensionList{})
}

const (
// DataPlaneKonnectExtensionKind holds the kind for the DataPlaneKonnectExtension.
DataPlaneKonnectExtensionKind = "DataPlaneKonnectExtension"
)

// +genclient
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
// +kubebuilder:object:root=true
Expand Down Expand Up @@ -60,6 +65,7 @@ type DataPlaneKonnectExtensionList struct {
type DataPlaneKonnectExtensionSpec struct {
// ControlPlaneRef is a reference to a ControlPlane this DataPlaneKonnectExtension is associated with.
// +kubebuilder:validation:Required
// +kubebuilder:validation:XValidation:rule="self.type == 'konnectID'", message="Only konnectID type currently supported as controlPlaneRef."
ControlPlaneRef configurationv1alpha1.ControlPlaneRef `json:"controlPlaneRef"`

// ControlPlaneRegion is the region of the Konnect Control Plane.
Expand Down Expand Up @@ -94,9 +100,9 @@ type DataPlaneKonnectExtensionSpec struct {
// KonnectControlPlaneAPIAuthConfiguration contains the configuration to authenticate with Konnect API ControlPlane.
// +apireference:kgo:include
type KonnectControlPlaneAPIAuthConfiguration struct {
// ClusterCertificateSecretName is a name of the Secret containing the Konnect Control Plane's cluster certificate.
// ClusterCertificateSecretRef is the reference to the Secret containing the Konnect Control Plane's cluster certificate.
// +kubebuilder:validation:Required
ClusterCertificateSecretName ClusterCertificateSecretRef `json:"clusterCertificateSecretRef"`
ClusterCertificateSecretRef ClusterCertificateSecretRef `json:"clusterCertificateSecretRef"`
}

// ClusterCertificateSecretRef contains the reference to the Secret containing the Konnect Control Plane's cluster certificate.
Expand Down
2 changes: 1 addition & 1 deletion api/v1alpha1/zz_generated.deepcopy.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,8 @@ spec:
- type
type: object
x-kubernetes-validations:
- message: Only konnectID type currently supported as controlPlaneRef.
rule: self.type == 'konnectID'
- message: when type is konnectNamespacedRef, konnectNamespacedRef
must be set
rule: 'self.type == ''konnectNamespacedRef'' ? has(self.konnectNamespacedRef)
Expand Down Expand Up @@ -123,8 +125,8 @@ spec:
API authentication.
properties:
clusterCertificateSecretRef:
description: ClusterCertificateSecretName is a name of the Secret
containing the Konnect Control Plane's cluster certificate.
description: ClusterCertificateSecretRef is the reference to the
Secret containing the Konnect Control Plane's cluster certificate.
properties:
name:
description: Name is the name of the Secret containing the
Expand Down
17 changes: 13 additions & 4 deletions config/rbac/role/role.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -240,6 +240,7 @@ rules:
resources:
- aigateways/status
- controlplanes/status
- dataplanekonnectextensions/status
- dataplanes/status
- kongplugininstallations/status
verbs:
Expand All @@ -249,20 +250,28 @@ rules:
- apiGroups:
- gateway-operator.konghq.com
resources:
- gatewayconfigurations
- dataplanekonnectextensions
- kongplugininstallations
verbs:
- get
- list
- patch
- update
- watch
- apiGroups:
- gateway-operator.konghq.com
resources:
- kongplugininstallations
- dataplanekonnectextensions/finalizers
verbs:
- get
- list
- patch
- update
- apiGroups:
- gateway-operator.konghq.com
resources:
- gatewayconfigurations
verbs:
- get
- list
- watch
- apiGroups:
- gateway.networking.k8s.io
Expand Down
13 changes: 13 additions & 0 deletions controller/dataplane/controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,19 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
return ctrl.Result{}, markErr
}

log.Trace(logger, "applying extensions", dataplane)
patched, requeue, err := applyExtensions(ctx, r.Client, logger, dataplane)
if err != nil {
if !requeue {
log.Debug(logger, "failed to apply extensions", dataplane, "error:", err)
return ctrl.Result{}, nil
}
return ctrl.Result{}, err
}
if patched {
return ctrl.Result{}, nil
}

log.Trace(logger, "exposing DataPlane deployment admin API via headless service", dataplane)
res, dataplaneAdminService, err := ensureAdminServiceForDataPlane(ctx, r.Client, dataplane,
client.MatchingLabels{
Expand Down
2 changes: 2 additions & 0 deletions controller/dataplane/controller_rbac.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@ package dataplane
// +kubebuilder:rbac:groups=gateway-operator.konghq.com,resources=dataplanes,verbs=get;list;watch;update;patch
// +kubebuilder:rbac:groups=gateway-operator.konghq.com,resources=dataplanes/status,verbs=get;update;patch
// +kubebuilder:rbac:groups=gateway-operator.konghq.com,resources=dataplanes/finalizers,verbs=update
// +kubebuilder:rbac:groups=gateway-operator.konghq.com,resources=dataplanekonnectextensions,verbs=get;list;watch;update;patch
// +kubebuilder:rbac:groups=gateway-operator.konghq.com,resources=dataplanekonnectextensions/status,verbs=get;update;patch
// +kubebuilder:rbac:groups=apps,resources=deployments,verbs=create;get;list;watch;update;patch;delete
// +kubebuilder:rbac:groups=apps,resources=deployments/status,verbs=get
// +kubebuilder:rbac:groups=core,resources=services,verbs=create;get;list;watch;update;patch;delete
Expand Down
46 changes: 46 additions & 0 deletions controller/dataplane/controller_utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,10 @@ package dataplane
import (
"context"
"encoding/json"
"errors"
"fmt"
"os"
"strings"

"github.com/go-logr/logr"
appsv1 "k8s.io/api/apps/v1"
Expand Down Expand Up @@ -308,3 +310,47 @@ func isDeploymentReady(deploymentStatus appsv1.DeploymentStatus) (metav1.Conditi
return metav1.ConditionFalse, false
}
}

// -----------------------------------------------------------------------------
// DataPlane - Private Functions - extensions
// -----------------------------------------------------------------------------

// applyExtensions patches the dataplane spec by taking into account customizations from the referenced extensions.
// In case any extension is referenced, it adds a resolvedRefs condition to the dataplane, indicating the status of the
// extension reference. it returns 3 values:
// - patched: a boolean indicating if the dataplane was patched. If the dataplane was patched, a reconciliation loop will be automatically re-triggered.
// - requeue: a boolean indicating if the dataplane should be requeued. If the error was unexpected (e.g., because of API server error), the dataplane should be requeued.
// In case the error is related to a misconfiguration, the dataplane does not need to be requeued, and feedback is provided into the dataplane status.
// - err: an error in case of failure.
func applyExtensions(ctx context.Context, cl client.Client, logger logr.Logger, dataplane *operatorv1beta1.DataPlane) (patched bool, requeue bool, err error) {
if len(dataplane.Spec.Extensions) == 0 {
return false, false, nil
}
condition := k8sutils.NewConditionWithGeneration(consts.ResolvedRefsType, metav1.ConditionTrue, consts.ResolvedRefsReason, "", dataplane.GetGeneration())
err = applyDataPlaneKonnectExtension(ctx, cl, dataplane)
if err != nil {
switch {
case errors.Is(err, ErrCrossNamespaceReference):
condition.Status = metav1.ConditionFalse
condition.Reason = string(consts.RefNotPermittedReason)
condition.Message = strings.ReplaceAll(err.Error(), "\n", " - ")
case errors.Is(err, ErrKonnectExtensionNotFound):
condition.Status = metav1.ConditionFalse
condition.Reason = string(consts.InvalidExtensionRefReason)
condition.Message = strings.ReplaceAll(err.Error(), "\n", " - ")
case errors.Is(err, ErrClusterCertificateNotFound):
condition.Status = metav1.ConditionFalse
condition.Reason = string(consts.InvalidSecretRefReason)
condition.Message = strings.ReplaceAll(err.Error(), "\n", " - ")
default:
return patched, true, err
}
}
newDataPlane := dataplane.DeepCopy()
k8sutils.SetCondition(condition, newDataPlane)
patched, patchErr := patchDataPlaneStatus(ctx, cl, logger, newDataPlane)
if patchErr != nil {
return false, true, fmt.Errorf("failed patching status for DataPlane %s/%s: %w", dataplane.Namespace, dataplane.Name, patchErr)
}
return patched, false, err
}
119 changes: 119 additions & 0 deletions controller/dataplane/dataplanekonnectextension_controller.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,119 @@
package dataplane

import (
"context"
"reflect"

k8serrors "k8s.io/apimachinery/pkg/api/errors"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
"sigs.k8s.io/controller-runtime/pkg/handler"
ctrllog "sigs.k8s.io/controller-runtime/pkg/log"
"sigs.k8s.io/controller-runtime/pkg/reconcile"

operatorv1alpha1 "github.com/kong/gateway-operator/api/v1alpha1"
operatorv1beta1 "github.com/kong/gateway-operator/api/v1beta1"
"github.com/kong/gateway-operator/controller/pkg/ctxinjector"
"github.com/kong/gateway-operator/controller/pkg/log"
operatorerrors "github.com/kong/gateway-operator/internal/errors"
"github.com/kong/gateway-operator/internal/utils/index"
"github.com/kong/gateway-operator/pkg/consts"
)

// -----------------------------------------------------------------------------
// DataKonnectExtensionReconciler
// -----------------------------------------------------------------------------

// DataPlaneKonnectExtensionReconciler reconciles a DataPlaneKonnectExtension object.
type DataPlaneKonnectExtensionReconciler struct {
client.Client
ContextInjector ctxinjector.CtxInjector
// DevelopmentMode indicates if the controller should run in development mode,
// which causes it to e.g. perform less validations.
DevelopmentMode bool
}

// SetupWithManager sets up the controller with the Manager.
func (r *DataPlaneKonnectExtensionReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager) error {
return ctrl.NewControllerManagedBy(mgr).
For(&operatorv1alpha1.DataPlaneKonnectExtension{}).
Watches(&operatorv1beta1.DataPlane{}, handler.EnqueueRequestsFromMapFunc(r.listDataPlaneExtensionsReferenced)).
Complete(r)
}

// listDataPlaneExtensionsReferenced returns a list of all the DataPlaneKonnectExtensions referenced by the DataPlane object.
// Maximum one reference is expected.
func (r *DataPlaneKonnectExtensionReconciler) listDataPlaneExtensionsReferenced(ctx context.Context, obj client.Object) []reconcile.Request {
logger := ctrllog.FromContext(ctx)
dataPlane, ok := obj.(*operatorv1beta1.DataPlane)
if !ok {
logger.Error(
operatorerrors.ErrUnexpectedObject,
"failed to run map funcs",
"expected", "DataPlane", "found", reflect.TypeOf(obj),
)
return nil
}

if len(dataPlane.Spec.Extensions) == 0 {
return nil
}

recs := []reconcile.Request{}

for _, ext := range dataPlane.Spec.Extensions {
namespace := dataPlane.Namespace
if ext.Group != operatorv1alpha1.SchemeGroupVersion.Group ||
ext.Kind != operatorv1alpha1.DataPlaneKonnectExtensionKind {
continue
}
if ext.Namespace != nil && *ext.Namespace != namespace {
continue
}
recs = append(recs, reconcile.Request{
NamespacedName: client.ObjectKey{
Namespace: namespace,
Name: ext.Name,
},
})
}
return recs
}

// Reconcile reconciles a DataPlaneKonnectExtension object.
func (r *DataPlaneKonnectExtensionReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
ctx = r.ContextInjector.InjectKeyValues(ctx)
var konnectExtension operatorv1alpha1.DataPlaneKonnectExtension
if err := r.Client.Get(ctx, req.NamespacedName, &konnectExtension); err != nil {
return ctrl.Result{}, client.IgnoreNotFound(err)
}

logger := log.GetLogger(ctx, operatorv1alpha1.DataPlaneKonnectExtensionKind, r.DevelopmentMode)
var dataPlaneList operatorv1beta1.DataPlaneList
if err := r.List(ctx, &dataPlaneList, client.MatchingFields{
index.DataPlaneKonnectExtensionIndex: konnectExtension.Namespace + "/" + konnectExtension.Name,
}); err != nil {
return ctrl.Result{}, err
}

var updated bool
switch len(dataPlaneList.Items) {
case 0:
updated = controllerutil.RemoveFinalizer(&konnectExtension, consts.DataPlaneExtensionFinalizer)
default:
updated = controllerutil.AddFinalizer(&konnectExtension, consts.DataPlaneExtensionFinalizer)
}
if updated {
if err := r.Client.Update(ctx, &konnectExtension); err != nil {
if k8serrors.IsConflict(err) {
return ctrl.Result{Requeue: true}, nil
}
return ctrl.Result{}, err
}

log.Info(logger, "DataPlaneKonnectExtension finalizer updated", konnectExtension)
}

return ctrl.Result{}, nil
}
9 changes: 9 additions & 0 deletions controller/dataplane/dataplanekonnectextension_rbac.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
package dataplane

// -----------------------------------------------------------------------------
// DataPlaneKonnectExtensionReconciler - RBAC
// -----------------------------------------------------------------------------

// +kubebuilder:rbac:groups=gateway-operator.konghq.com,resources=dataplanes,verbs=get;list;watch
// +kubebuilder:rbac:groups=gateway-operator.konghq.com,resources=dataplanekonnectextensions,verbs=get;list;watch
// +kubebuilder:rbac:groups=gateway-operator.konghq.com,resources=dataplanekonnectextensions/finalizers,verbs=update;patch
Loading

0 comments on commit 0780829

Please sign in to comment.