From a0edd3fa7de49a5c2a2e88d42708697213407b48 Mon Sep 17 00:00:00 2001 From: Mattia Lavacca Date: Tue, 1 Oct 2024 17:10:07 +0200 Subject: [PATCH] feat: DataPlaneKonnectExtension customization Signed-off-by: Mattia Lavacca --- CHANGELOG.md | 3 + .../dataplane_konnect_extension_types.go | 1 + ...konghq.com_dataplanekonnectextensions.yaml | 2 + controller/dataplane/controller.go | 5 + controller/dataplane/controller_utils.go | 8 ++ .../dataplanekonnectextension_controller.go | 117 ++++++++++++++++++ controller/dataplane/konnect_extension.go | 113 +++++++++++++++++ controller/dataplane/owned_deployment.go | 5 +- controller/dataplane/watch.go | 33 +++++ internal/utils/dataplane/config.go | 34 +---- internal/utils/dataplane/config_test.go | 4 +- internal/utils/dataplane/env.go | 52 ++++++++ internal/utils/index/index.go | 41 ++++++ modules/manager/controller_setup.go | 30 +++-- pkg/consts/consts.go | 8 ++ pkg/consts/dataplane.go | 10 ++ pkg/utils/kubernetes/resources/deployments.go | 4 +- 17 files changed, 423 insertions(+), 47 deletions(-) create mode 100644 controller/dataplane/dataplanekonnectextension_controller.go create mode 100644 controller/dataplane/konnect_extension.go create mode 100644 internal/utils/dataplane/env.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 6d408b03c..55da640f2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -101,6 +101,9 @@ [#669](https://github.com/Kong/gateway-operator/pull/669) - Allow setting `KonnectGatewayControlPlane`s group membership [#697](https://github.com/Kong/gateway-operator/pull/697) +- Apply konnect-related customization to `DataPlane`s that properly reference `DataPlaneKonnectExtension` + resources. + [#714](https://github.com/Kong/gateway-operator/pull/714) ### Fixed diff --git a/api/v1alpha1/dataplane_konnect_extension_types.go b/api/v1alpha1/dataplane_konnect_extension_types.go index 23b83ff49..a27d37d6a 100644 --- a/api/v1alpha1/dataplane_konnect_extension_types.go +++ b/api/v1alpha1/dataplane_konnect_extension_types.go @@ -60,6 +60,7 @@ type DataPlaneKonnectExtensionList struct { type DataPlaneKonnectExtensionSpec struct { // ControlPlaneRef is a reference to a ControlPlane this DataPlaneKonnectExtension is associated with. // +kubebuilder:validation:Required + // +kubebuilder:validation:XValidation:rule="self.type == 'konnectID'", message="Only konnectID type currently supported as controlPlaneRef." ControlPlaneRef configurationv1alpha1.ControlPlaneRef `json:"controlPlaneRef"` // ControlPlaneRegion is the region of the Konnect Control Plane. diff --git a/config/crd/bases/gateway-operator.konghq.com_dataplanekonnectextensions.yaml b/config/crd/bases/gateway-operator.konghq.com_dataplanekonnectextensions.yaml index ade8097a5..5efd92abd 100644 --- a/config/crd/bases/gateway-operator.konghq.com_dataplanekonnectextensions.yaml +++ b/config/crd/bases/gateway-operator.konghq.com_dataplanekonnectextensions.yaml @@ -92,6 +92,8 @@ spec: - type type: object x-kubernetes-validations: + - message: Only konnectID type currently supported as controlPlaneRef. + rule: self.type == 'konnectID' - message: when type is konnectNamespacedRef, konnectNamespacedRef must be set rule: 'self.type == ''konnectNamespacedRef'' ? has(self.konnectNamespacedRef) diff --git a/controller/dataplane/controller.go b/controller/dataplane/controller.go index 81c909586..87e5e166e 100644 --- a/controller/dataplane/controller.go +++ b/controller/dataplane/controller.go @@ -90,6 +90,11 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu return ctrl.Result{}, markErr } + log.Trace(logger, "applying extensions", dataplane) + if err = applyExtensions(ctx, r.Client, dataplane); err != nil { + return ctrl.Result{}, err + } + log.Trace(logger, "exposing DataPlane deployment admin API via headless service", dataplane) res, dataplaneAdminService, err := ensureAdminServiceForDataPlane(ctx, r.Client, dataplane, client.MatchingLabels{ diff --git a/controller/dataplane/controller_utils.go b/controller/dataplane/controller_utils.go index 4e0c8c973..a655f57c0 100644 --- a/controller/dataplane/controller_utils.go +++ b/controller/dataplane/controller_utils.go @@ -308,3 +308,11 @@ func isDeploymentReady(deploymentStatus appsv1.DeploymentStatus) (metav1.Conditi return metav1.ConditionFalse, false } } + +// ----------------------------------------------------------------------------- +// DataPlane - Private Functions - extensions +// ----------------------------------------------------------------------------- + +func applyExtensions(ctx context.Context, cl client.Client, dataplane *operatorv1beta1.DataPlane) error { + return applyDataPlaneKonnectExtension(ctx, cl, dataplane) +} diff --git a/controller/dataplane/dataplanekonnectextension_controller.go b/controller/dataplane/dataplanekonnectextension_controller.go new file mode 100644 index 000000000..a6c651c22 --- /dev/null +++ b/controller/dataplane/dataplanekonnectextension_controller.go @@ -0,0 +1,117 @@ +package dataplane + +import ( + "context" + "reflect" + + k8serrors "k8s.io/apimachinery/pkg/api/errors" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/handler" + ctrllog "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + operatorv1alpha1 "github.com/kong/gateway-operator/api/v1alpha1" + operatorv1beta1 "github.com/kong/gateway-operator/api/v1beta1" + "github.com/kong/gateway-operator/controller/pkg/ctxinjector" + "github.com/kong/gateway-operator/controller/pkg/log" + operatorerrors "github.com/kong/gateway-operator/internal/errors" + "github.com/kong/gateway-operator/internal/utils/index" + "github.com/kong/gateway-operator/pkg/consts" +) + +// ----------------------------------------------------------------------------- +// DataKonnectExtensionReconciler +// ----------------------------------------------------------------------------- + +// DataPlaneKonnectExtensionReconciler reconciles a DataPlaneKonnectExtension object. +type DataPlaneKonnectExtensionReconciler struct { + client.Client + ContextInjector ctxinjector.CtxInjector + // DevelopmentMode indicates if the controller should run in development mode, + // which causes it to e.g. perform less validations. + DevelopmentMode bool +} + +// SetupWithManager sets up the controller with the Manager. +func (r *DataPlaneKonnectExtensionReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&operatorv1alpha1.DataPlaneKonnectExtension{}). + Watches(&operatorv1beta1.DataPlane{}, handler.EnqueueRequestsFromMapFunc(r.listDataPlaneExtensionsReferenced)). + Complete(r) +} + +// listDataPlaneExtensionsReferenced returns a list of all the DataPlaneKonnectExtensions referenced by the DataPlane object. +// Maximum one reference is expected. +func (r *DataPlaneKonnectExtensionReconciler) listDataPlaneExtensionsReferenced(ctx context.Context, obj client.Object) []reconcile.Request { + logger := ctrllog.FromContext(ctx) + dataPlane, ok := obj.(*operatorv1beta1.DataPlane) + if !ok { + logger.Error( + operatorerrors.ErrUnexpectedObject, + "failed to run map funcs", + "expected", "DataPlane", "found", reflect.TypeOf(obj), + ) + return nil + } + + if len(dataPlane.Spec.Extensions) == 0 { + return nil + } + + recs := []reconcile.Request{} + for _, extension := range dataPlane.Spec.Extensions { + namespace := dataPlane.Namespace + if extension.Group == operatorv1alpha1.SchemeGroupVersion.Group && + extension.Kind == "DataPlaneKonnectExtension" { + if extension.Namespace != nil { + namespace = *extension.Namespace + } + recs = append(recs, reconcile.Request{ + NamespacedName: client.ObjectKey{ + Namespace: namespace, + Name: extension.Name, + }, + }) + } + } + return recs +} + +// Reconcile reconciles a DataPlaneKonnectExtension object. +func (r *DataPlaneKonnectExtensionReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + ctx = r.ContextInjector.InjectKeyValues(ctx) + var konnectExtension operatorv1alpha1.DataPlaneKonnectExtension + if err := r.Client.Get(ctx, req.NamespacedName, &konnectExtension); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + logger := log.GetLogger(ctx, "dataplaneKonnectExtension", r.DevelopmentMode) + var dataPlaneList operatorv1beta1.DataPlaneList + if err := r.List(ctx, &dataPlaneList, client.MatchingFields{ + index.DataPlaneKonnectExtensionIndex: konnectExtension.Namespace + "/" + konnectExtension.Name, + }); err != nil { + return ctrl.Result{}, err + } + + var updated bool + switch len(dataPlaneList.Items) { + case 0: + updated = controllerutil.RemoveFinalizer(&konnectExtension, consts.DataPlaneExtensionFinalizer) + default: + updated = controllerutil.AddFinalizer(&konnectExtension, consts.DataPlaneExtensionFinalizer) + } + if updated { + if err := r.Client.Update(ctx, &konnectExtension); err != nil { + if k8serrors.IsConflict(err) { + return ctrl.Result{Requeue: true}, nil + } + return ctrl.Result{}, err + } + + log.Info(logger, "DataPlaneKonnectExtension finalizer updated", konnectExtension) + } + + return ctrl.Result{}, nil +} diff --git a/controller/dataplane/konnect_extension.go b/controller/dataplane/konnect_extension.go new file mode 100644 index 000000000..a9f946d36 --- /dev/null +++ b/controller/dataplane/konnect_extension.go @@ -0,0 +1,113 @@ +package dataplane + +import ( + "context" + "strings" + + "github.com/samber/lo" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/kong/gateway-operator/api/v1alpha1" + "github.com/kong/gateway-operator/api/v1beta1" + dputils "github.com/kong/gateway-operator/internal/utils/dataplane" + "github.com/kong/gateway-operator/pkg/consts" + k8sresources "github.com/kong/gateway-operator/pkg/utils/kubernetes/resources" +) + +// applyDataPlaneKonnectExtension gets the DataPlane as argument, and in case it references a KonnectExtension, it +// fetch the referenced extension and applies the necessary changes to the DataPlane spec. +func applyDataPlaneKonnectExtension(ctx context.Context, cl client.Client, dataplane *v1beta1.DataPlane) error { + if len(dataplane.Spec.Extensions) == 0 { + return nil + } + + for _, extensionRef := range dataplane.Spec.Extensions { + if extensionRef.Group != v1alpha1.SchemeGroupVersion.Group || extensionRef.Kind != "DataPlaneKonnectExtension" { + continue + } + namespace := dataplane.Namespace + if extensionRef.Namespace != nil { + namespace = *extensionRef.Namespace + } + + konnectExt := v1alpha1.DataPlaneKonnectExtension{} + if err := cl.Get(ctx, client.ObjectKey{ + Namespace: namespace, + Name: extensionRef.Name, + }, &konnectExt); err != nil { + return err + } + + d := k8sresources.Deployment(appsv1.Deployment{ + Spec: appsv1.DeploymentSpec{ + Template: *dataplane.Spec.Deployment.PodTemplateSpec, + }, + }) + + d.WithVolume(kongInKonnectClusterCertificateVolume()) + d.WithVolumeMount(kongInKonnectClusterCertificateVolumeMount(), consts.DataPlaneProxyContainerName) + d.WithVolume(kongInKonnectClusterCertVolume(konnectExt.Spec.AuthConfiguration.ClusterCertificateSecretName.Name)) + d.WithVolumeMount(kongInKonnectClusterVolumeMount(), consts.DataPlaneProxyContainerName) + + envSet := customizeKongInKonnectDefaults( + dputils.KongInKonnectDefaults, + *konnectExt.Spec.ControlPlaneRef.KonnectID, + konnectExt.Spec.ControlPlaneRegion, + konnectExt.Spec.ServerHostname) + + dputils.FillDataPlaneProxyContainerEnvs(nil, &d.Spec.Template, envSet) + dataplane.Spec.Deployment.PodTemplateSpec = &d.Spec.Template + } + return nil +} + +// customizeKongInKonnectDefaults replaces placeholders in the KongInKonnect env list with the actual values. +func customizeKongInKonnectDefaults(envSet map[string]string, + controlPlane, + region, + server string, +) map[string]string { + newEnvSet := make(map[string]string, len(envSet)) + for k, v := range envSet { + v = strings.ReplaceAll(v, "", controlPlane) + v = strings.ReplaceAll(v, "", region) + v = strings.ReplaceAll(v, "", server) + newEnvSet[k] = v + } + return newEnvSet +} + +func kongInKonnectClusterCertVolume(secretName string) corev1.Volume { + return corev1.Volume{ + Name: consts.KongClusterCertVolume, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: secretName, + DefaultMode: lo.ToPtr(int32(420)), + }, + }, + } +} + +func kongInKonnectClusterVolumeMount() corev1.VolumeMount { + return corev1.VolumeMount{ + Name: consts.KongClusterCertVolume, + MountPath: consts.KongClusterCertVolumeMountPath, + } +} + +func kongInKonnectClusterCertificateVolume() corev1.Volume { + return corev1.Volume{ + Name: consts.ClusterCertificateVolume, + } +} + +func kongInKonnectClusterCertificateVolumeMount() corev1.VolumeMount { + return corev1.VolumeMount{ + Name: consts.ClusterCertificateVolume, + MountPath: consts.ClusterCertificateVolumeMountPath, + ReadOnly: true, + } +} diff --git a/controller/dataplane/owned_deployment.go b/controller/dataplane/owned_deployment.go index b777b4682..4edbc4a8c 100644 --- a/controller/dataplane/owned_deployment.go +++ b/controller/dataplane/owned_deployment.go @@ -141,7 +141,7 @@ func (d *DeploymentBuilder) BuildAndDeploy( return nil, op.Noop, err } // apply default envvars and restore the hacked-out ones - desiredDeployment = applyEnvForDataPlane(existingEnvVars, desiredDeployment) + desiredDeployment = applyEnvForDataPlane(existingEnvVars, desiredDeployment, dputils.KongDefaults) // push the complete Deployment to Kubernetes res, deployment, err := reconcileDataPlaneDeployment(ctx, d.client, d.logger, @@ -200,8 +200,9 @@ func applyDeploymentUserPatchesForDataPlane( func applyEnvForDataPlane( existing []corev1.EnvVar, deployment *k8sresources.Deployment, + envSet map[string]string, ) *k8sresources.Deployment { - dputils.FillDataPlaneProxyContainerEnvs(existing, &deployment.Spec.Template) + dputils.FillDataPlaneProxyContainerEnvs(existing, &deployment.Spec.Template, envSet) return deployment } diff --git a/controller/dataplane/watch.go b/controller/dataplane/watch.go index 1d017d0ee..95adaf2a2 100644 --- a/controller/dataplane/watch.go +++ b/controller/dataplane/watch.go @@ -16,6 +16,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" + operatorv1alpha1 "github.com/kong/gateway-operator/api/v1alpha1" operatorv1beta1 "github.com/kong/gateway-operator/api/v1beta1" "github.com/kong/gateway-operator/internal/utils/index" "github.com/kong/gateway-operator/pkg/consts" @@ -48,6 +49,13 @@ func DataPlaneWatchBuilder(mgr ctrl.Manager) *builder.Builder { &corev1.ConfigMap{}, handler.TypedEnqueueRequestsFromMapFunc(listDataPlanesReferencingKongPluginInstallation(mgr.GetClient())), ), + ). + WatchesRawSource( + source.Kind( + mgr.GetCache(), + &operatorv1alpha1.DataPlaneKonnectExtension{}, + handler.TypedEnqueueRequestsFromMapFunc(listDataPlanesReferencingDataPlaneKonnectExtension(mgr.GetClient())), + ), ) } @@ -79,3 +87,28 @@ func listDataPlanesReferencingKongPluginInstallation( }) } } + +func listDataPlanesReferencingDataPlaneKonnectExtension( + c client.Client, +) handler.TypedMapFunc[*operatorv1alpha1.DataPlaneKonnectExtension, reconcile.Request] { + return func( + ctx context.Context, ext *operatorv1alpha1.DataPlaneKonnectExtension, + ) []reconcile.Request { + logger := ctrllog.FromContext(ctx) + + // Find all DataPlane resources referencing DataPlaneKonnectExtension + // that maps to the DataPlaneKonnectExtension enqueued for reconciliation. + var dataPlaneList operatorv1beta1.DataPlaneList + if err := c.List(ctx, &dataPlaneList, client.MatchingFields{ + index.DataPlaneKonnectExtensionIndex: ext.Namespace + "/" + ext.Name, + }); err != nil { + logger.Error(err, "Failed to list DataPlanes in watch", "DataPlaneKonnectExtension") + return nil + } + return lo.Map(dataPlaneList.Items, func(dp operatorv1beta1.DataPlane, _ int) reconcile.Request { + return reconcile.Request{ + NamespacedName: client.ObjectKeyFromObject(&dp), + } + }) + } +} diff --git a/internal/utils/dataplane/config.go b/internal/utils/dataplane/config.go index b3bf8151b..5b5b631f2 100644 --- a/internal/utils/dataplane/config.go +++ b/internal/utils/dataplane/config.go @@ -1,7 +1,6 @@ package dataplane import ( - "fmt" "sort" "strings" @@ -19,35 +18,6 @@ const ( kongLuaPackagePathDefaultValue = "/opt/?.lua;;" ) -// KongDefaults are the baseline Kong proxy configuration options needed for -// the proxy to function. -var KongDefaults = map[string]string{ - "KONG_ADMIN_ACCESS_LOG": "/dev/stdout", - "KONG_ADMIN_ERROR_LOG": "/dev/stderr", - "KONG_ADMIN_GUI_ACCESS_LOG": "/dev/stdout", - "KONG_ADMIN_GUI_ERROR_LOG": "/dev/stderr", - "KONG_CLUSTER_LISTEN": "off", - "KONG_DATABASE": "off", - "KONG_NGINX_WORKER_PROCESSES": "2", - kongPluginsEnvVarName: kongPluginsDefaultValue, - "KONG_PORTAL_API_ACCESS_LOG": "/dev/stdout", - "KONG_PORTAL_API_ERROR_LOG": "/dev/stderr", - "KONG_PORT_MAPS": "80:8000, 443:8443", - "KONG_PROXY_ACCESS_LOG": "/dev/stdout", - "KONG_PROXY_ERROR_LOG": "/dev/stderr", - "KONG_PROXY_LISTEN": fmt.Sprintf("0.0.0.0:%d reuseport backlog=16384, 0.0.0.0:%d http2 ssl reuseport backlog=16384", consts.DataPlaneProxyPort, consts.DataPlaneProxySSLPort), - "KONG_STATUS_LISTEN": fmt.Sprintf("0.0.0.0:%d", consts.DataPlaneStatusPort), - - "KONG_ADMIN_LISTEN": fmt.Sprintf("0.0.0.0:%d ssl reuseport backlog=16384", consts.DataPlaneAdminAPIPort), - - // MTLS - "KONG_ADMIN_SSL_CERT": "/var/cluster-certificate/tls.crt", - "KONG_ADMIN_SSL_CERT_KEY": "/var/cluster-certificate/tls.key", - "KONG_NGINX_ADMIN_SSL_CLIENT_CERTIFICATE": "/var/cluster-certificate/ca.crt", - "KONG_NGINX_ADMIN_SSL_VERIFY_CLIENT": "on", - "KONG_NGINX_ADMIN_SSL_VERIFY_DEPTH": "3", -} - // ----------------------------------------------------------------------------- // DataPlane Utils - Config // ----------------------------------------------------------------------------- @@ -57,7 +27,7 @@ var KongDefaults = map[string]string{ // PodTemplateSpec. // EnvVars are sorted lexographically as a side effect. // It also returns the updated EnvVar slice. -func FillDataPlaneProxyContainerEnvs(existing []corev1.EnvVar, podTemplateSpec *corev1.PodTemplateSpec) { +func FillDataPlaneProxyContainerEnvs(existing []corev1.EnvVar, podTemplateSpec *corev1.PodTemplateSpec, envSet map[string]string) { if podTemplateSpec == nil { return } @@ -73,7 +43,7 @@ func FillDataPlaneProxyContainerEnvs(existing []corev1.EnvVar, podTemplateSpec * container.Env = append(container.Env, envVar) } } - for k, v := range KongDefaults { + for k, v := range envSet { envVar := corev1.EnvVar{ Name: k, Value: v, diff --git a/internal/utils/dataplane/config_test.go b/internal/utils/dataplane/config_test.go index e119b980d..0098a4165 100644 --- a/internal/utils/dataplane/config_test.go +++ b/internal/utils/dataplane/config_test.go @@ -26,7 +26,7 @@ func TestFillDataPlaneProxyContainerEnvs(t *testing.T) { } t.Run("nil doesn't panic", func(t *testing.T) { - FillDataPlaneProxyContainerEnvs(nil, nil) + FillDataPlaneProxyContainerEnvs(nil, nil, KongDefaults) }) testcases := []struct { @@ -151,7 +151,7 @@ func TestFillDataPlaneProxyContainerEnvs(t *testing.T) { } for _, tc := range testcases { t.Run(tc.name, func(t *testing.T) { - FillDataPlaneProxyContainerEnvs(tc.existing, tc.podTemplateSpec) + FillDataPlaneProxyContainerEnvs(tc.existing, tc.podTemplateSpec, KongDefaults) container := k8sutils.GetPodContainerByName(&tc.podTemplateSpec.Spec, consts.DataPlaneProxyContainerName) require.Equal(t, tc.expected, container.Env) }) diff --git a/internal/utils/dataplane/env.go b/internal/utils/dataplane/env.go new file mode 100644 index 000000000..055853a06 --- /dev/null +++ b/internal/utils/dataplane/env.go @@ -0,0 +1,52 @@ +package dataplane + +import ( + "fmt" + + "github.com/kong/gateway-operator/pkg/consts" +) + +// KongDefaults are the baseline Kong proxy configuration options needed for +// the proxy to function. +var KongDefaults = map[string]string{ + "KONG_ADMIN_ACCESS_LOG": "/dev/stdout", + "KONG_ADMIN_ERROR_LOG": "/dev/stderr", + "KONG_ADMIN_GUI_ACCESS_LOG": "/dev/stdout", + "KONG_ADMIN_GUI_ERROR_LOG": "/dev/stderr", + "KONG_CLUSTER_LISTEN": "off", + "KONG_DATABASE": "off", + "KONG_NGINX_WORKER_PROCESSES": "2", + kongPluginsEnvVarName: kongPluginsDefaultValue, + "KONG_PORTAL_API_ACCESS_LOG": "/dev/stdout", + "KONG_PORTAL_API_ERROR_LOG": "/dev/stderr", + "KONG_PORT_MAPS": "80:8000, 443:8443", + "KONG_PROXY_ACCESS_LOG": "/dev/stdout", + "KONG_PROXY_ERROR_LOG": "/dev/stderr", + "KONG_PROXY_LISTEN": fmt.Sprintf("0.0.0.0:%d reuseport backlog=16384, 0.0.0.0:%d http2 ssl reuseport backlog=16384", consts.DataPlaneProxyPort, consts.DataPlaneProxySSLPort), + "KONG_STATUS_LISTEN": fmt.Sprintf("0.0.0.0:%d", consts.DataPlaneStatusPort), + + "KONG_ADMIN_LISTEN": fmt.Sprintf("0.0.0.0:%d ssl reuseport backlog=16384", consts.DataPlaneAdminAPIPort), + + // MTLS + "KONG_ADMIN_SSL_CERT": "/var/cluster-certificate/tls.crt", + "KONG_ADMIN_SSL_CERT_KEY": "/var/cluster-certificate/tls.key", + "KONG_NGINX_ADMIN_SSL_CLIENT_CERTIFICATE": "/var/cluster-certificate/ca.crt", + "KONG_NGINX_ADMIN_SSL_VERIFY_CLIENT": "on", + "KONG_NGINX_ADMIN_SSL_VERIFY_DEPTH": "3", +} + +// KongInKonnectDefaults are the baseline Kong proxy configuration options needed for +// the proxy to function when configured in Konnect. +var KongInKonnectDefaults = map[string]string{ + "KONG_ROLE": "data_plane", + "KONG_CLUSTER_MTLS": "pki", + "KONG_CLUSTER_CONTROL_PLANE": "..cp0.:443", + "KONG_CLUSTER_SERVER_NAME": "..cp0.", + "KONG_CLUSTER_TELEMETRY_ENDPOINT": "..tp0.:443", + "KONG_CLUSTER_TELEMETRY_SERVER_NAME": "..tp0.", + "KONG_CLUSTER_CERT": "/etc/secrets/kong-cluster-cert/tls.crt", + "KONG_CLUSTER_CERT_KEY": "/etc/secrets/kong-cluster-cert/tls.key", + "KONG_LUA_SSL_TRUSTED_CERTIFICATE": "system", + "KONG_KONNECT_MODE": "on", + "KONG_VITALS": "off", +} diff --git a/internal/utils/index/index.go b/internal/utils/index/index.go index 9a002799c..03a887edf 100644 --- a/internal/utils/index/index.go +++ b/internal/utils/index/index.go @@ -8,6 +8,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" + operatorv1alpha1 "github.com/kong/gateway-operator/api/v1alpha1" operatorv1beta1 "github.com/kong/gateway-operator/api/v1beta1" ) @@ -18,6 +19,10 @@ const ( // KongPluginInstallationsIndex is the key to be used to access the .spec.pluginsToInstall indexed values, // in a form of list of namespace/name strings. KongPluginInstallationsIndex = "KongPluginInstallations" + + // DataPlaneKonnectExtensionIndex is the key to be used to access the .spec.extensions indexed values, + // in a form of list of namespace/name strings. + DataPlaneKonnectExtensionIndex = "DataPlaneKonnectExtension" ) // DataPlaneNameOnControlPlane indexes the ControlPlane .spec.dataplaneName field @@ -71,3 +76,39 @@ func KongPluginInstallationsOnDataPlane(ctx context.Context, c cache.Cache) erro }, ) } + +// DataPlaneOnDataPlaneKonnecExtension indexes the DataPlane .spec.extensions field +// on the "DataPlaneKonnectExtension" key. +func DataPlaneOnDataPlaneKonnecExtension(ctx context.Context, c cache.Cache) error { + if _, err := c.GetInformer(ctx, &operatorv1beta1.DataPlane{}); err != nil { + if meta.IsNoMatchError(err) { + return nil + } + return fmt.Errorf("failed to get informer for v1alpha1 DataPlaneKonnectExtension: %w, disabling indexing DataPlaneKonnectExtensions for DataPlanes' .spec.extensions", err) + } + return c.IndexField( + ctx, + &operatorv1beta1.DataPlane{}, + DataPlaneKonnectExtensionIndex, + func(o client.Object) []string { + dp, ok := o.(*operatorv1beta1.DataPlane) + if !ok { + return nil + } + result := []string{} + if len(dp.Spec.Extensions) > 0 { + for _, ext := range dp.Spec.Extensions { + namespace := dp.Namespace + if ext.Group == operatorv1alpha1.SchemeGroupVersion.Group && + ext.Kind == "DataPlaneKonnectExtension" { + if ext.Namespace != nil { + namespace = *ext.Namespace + } + result = append(result, namespace+"/"+ext.NamespacedRef.Name) + } + } + } + return result + }, + ) +} diff --git a/modules/manager/controller_setup.go b/modules/manager/controller_setup.go index 863ad009e..3ab52f7bc 100644 --- a/modules/manager/controller_setup.go +++ b/modules/manager/controller_setup.go @@ -40,26 +40,26 @@ import ( const ( // GatewayClassControllerName is the name of the GatewayClass controller. GatewayClassControllerName = "GatewayClass" - // GatewayControllerName is the name of the GatewayClass controller. + // GatewayControllerName is the name of the Gateway controller. GatewayControllerName = "Gateway" - // ControlPlaneControllerName is the name of the GatewayClass controller. + // ControlPlaneControllerName is the name of ControlPlane controller. ControlPlaneControllerName = "ControlPlane" - // DataPlaneControllerName is the name of the GatewayClass controller. + // DataPlaneControllerName is the name of the DataPlane controller. DataPlaneControllerName = "DataPlane" - // DataPlaneBlueGreenControllerName is the name of the GatewayClass controller. + // DataPlaneBlueGreenControllerName is the name of the DataPlaneBlueGreen controller. DataPlaneBlueGreenControllerName = "DataPlaneBlueGreen" - // DataPlaneOwnedServiceFinalizerControllerName is the name of the GatewayClass controller. + // DataPlaneOwnedServiceFinalizerControllerName is the name of the DataPlaneOwnedServiceFinalizer controller. DataPlaneOwnedServiceFinalizerControllerName = "DataPlaneOwnedServiceFinalizer" - // DataPlaneOwnedSecretFinalizerControllerName is the name of the GatewayClass controller. + // DataPlaneOwnedSecretFinalizerControllerName is the name of the DataPlaneOwnedSecretFinalizer controller. DataPlaneOwnedSecretFinalizerControllerName = "DataPlaneOwnedSecretFinalizer" - // DataPlaneOwnedDeploymentFinalizerControllerName is the name of the GatewayClass controller. + // DataPlaneOwnedDeploymentFinalizerControllerName is the name of the DataPlaneOwnedDeploymentFinalizer controller. DataPlaneOwnedDeploymentFinalizerControllerName = "DataPlaneOwnedDeploymentFinalizer" - // AIGatewayControllerName is the name of the GatewayClass controller. + // DataPlaneKonnectExtensionControllerName is the name of the DataPlaneKonnectExtension controller. + DataPlaneKonnectExtensionControllerName = "DataPlaneKonnectExtension" + // AIGatewayControllerName is the name of the AIGateway controller. AIGatewayControllerName = "AIGateway" - // KongPluginInstallationControllerName is the name of the KongPluginInstallation controller. KongPluginInstallationControllerName = "KongPluginInstallation" - // KonnectAPIAuthConfigurationControllerName is the name of the KonnectAPIAuthConfiguration controller. KonnectAPIAuthConfigurationControllerName = "KonnectAPIAuthConfiguration" // KonnectGatewayControlPlaneControllerName is the name of the KonnectGatewayControlPlane controller. @@ -177,6 +177,9 @@ func setupIndexes(ctx context.Context, mgr manager.Manager, cfg Config) error { return fmt.Errorf("failed to setup index for KongPluginInstallations on DataPlane: %w", err) } } + if err := index.DataPlaneOnDataPlaneKonnecExtension(ctx, mgr.GetCache()); err != nil { + return fmt.Errorf("failed to setup index for DataPlanes on DataPlaneKonnectExtensions: %w", err) + } } return nil } @@ -343,6 +346,13 @@ func SetupControllers(mgr manager.Manager, c *Config) (map[string]ControllerDef, c.DevelopmentMode, ), }, + DataPlaneKonnectExtensionControllerName: { + Enabled: c.DataPlaneControllerEnabled || c.DataPlaneBlueGreenControllerEnabled, + Controller: &dataplane.DataPlaneKonnectExtensionReconciler{ + Client: mgr.GetClient(), + DevelopmentMode: c.DevelopmentMode, + }, + }, // AIGateway Controller AIGatewayControllerName: { Enabled: c.AIGatewayControllerEnabled, diff --git a/pkg/consts/consts.go b/pkg/consts/consts.go index 23576b6ed..c569a7d9f 100644 --- a/pkg/consts/consts.go +++ b/pkg/consts/consts.go @@ -119,6 +119,14 @@ const ( // CAFieldSecret is the field name in Kubernetes secret - WebhookCertificateConfigSecretName. CAFieldSecret = "ca" + + // KongClusterCertVolume is the name of the volume that holds the certificate the enables + // communication between Kong and Konnect. + KongClusterCertVolume = "kong-cluster-cert" + + // KongClusterCertVolumeMountPath holds the path where the Kong Cluster certificate + // volume will be mounted. + KongClusterCertVolumeMountPath = "/etc/secrets/kong-cluster-cert" ) // ----------------------------------------------------------------------------- diff --git a/pkg/consts/dataplane.go b/pkg/consts/dataplane.go index b50e8d83b..00a1a350e 100644 --- a/pkg/consts/dataplane.go +++ b/pkg/consts/dataplane.go @@ -178,3 +178,13 @@ const ( // to ensure that the resources are not deleted before the DataPlane is deleted. DataPlaneOwnedWaitForOwnerFinalizer = "gateway-operator.konghq.com/wait-for-owner" ) + +// ----------------------------------------------------------------------------- +// Consts - DataPlaneKonnectExtension Finalizers +// ----------------------------------------------------------------------------- + +const ( + // DataPlaneExtensionFinalizer is the finalizer added to DataPlaneKonnectExtension + // to ensure that the resources are not deleted when in use by a dataplane. + DataPlaneExtensionFinalizer = "gateway-operator.konghq.com/extension-in-use" +) diff --git a/pkg/utils/kubernetes/resources/deployments.go b/pkg/utils/kubernetes/resources/deployments.go index 8f9f3c9d0..6323a7bed 100644 --- a/pkg/utils/kubernetes/resources/deployments.go +++ b/pkg/utils/kubernetes/resources/deployments.go @@ -278,7 +278,9 @@ func GenerateNewDeploymentForDataPlane( LabelObjectAsDataPlaneManaged(deployment) for _, opt := range opts { - opt(deployment) + if opt != nil { + opt(deployment) + } } k8sutils.SetOwnerForObject(deployment, dataplane)