feat(konnect): allow setting ControlPlane group members (#697)

* feat(konnect): allow setting ControlPlane group members

* chore: reorder variable declaration

* Update controller/konnect/ops/ops_controlplane.go

Co-authored-by: Grzegorz Burzyński <[email protected]>

* refactor: update status after Konnect update

* refactor: use sourcegraph/conc for concurrency

---------

Co-authored-by: Grzegorz Burzyński <[email protected]>

.mockery.yaml

@@ -12,6 +12,7 @@ packages:
   github.com/kong/gateway-operator/controller/konnect/ops:
     interfaces:
       ControlPlaneSDK:
+      ControlPlaneGroupSDK:
       ServicesSDK:
       RoutesSDK:
       ConsumersSDK:

CHANGELOG.md

@@ -99,6 +99,8 @@
   - HMAC Auth [#687](https://github.com/Kong/gateway-operator/pull/687)
 - Add support for `KongRoute`s bound directly to `KonnectGatewayControlPlane`s (serviceless rotues).
   [#669](https://github.com/Kong/gateway-operator/pull/669)
+- Allow setting `KonnectGatewayControlPlane`s group membership
+  [#697](https://github.com/Kong/gateway-operator/pull/697)
 
 ### Fixed
 

config/samples/konnect_gatewaycontrolplane_group_assignment.yaml

@@ -0,0 +1,61 @@
+kind: KonnectAPIAuthConfiguration
+apiVersion: konnect.konghq.com/v1alpha1
+metadata:
+  name: konnect-api-auth-dev-1
+  namespace: default
+spec:
+  type: token
+  token: kpat_XXXXXXXXXXXXXXXXXXX
+  serverURL: us.api.konghq.tech
+---
+kind: KonnectGatewayControlPlane
+apiVersion: konnect.konghq.com/v1alpha1
+metadata:
+  name: test1
+  namespace: default
+spec:
+  name: test1
+  konnect:
+    authRef:
+      name: konnect-api-auth-dev-1
+---
+kind: KonnectGatewayControlPlane
+apiVersion: konnect.konghq.com/v1alpha1
+metadata:
+  name: test2
+  namespace: default
+spec:
+  name: test2
+  konnect:
+    authRef:
+      name: konnect-api-auth-dev-1
+---
+kind: KonnectGatewayControlPlane
+apiVersion: konnect.konghq.com/v1alpha1
+metadata:
+  name: test3
+  namespace: default
+spec:
+  name: test3
+  konnect:
+    authRef:
+      name: konnect-api-auth-dev-1
+---
+kind: KonnectGatewayControlPlane
+apiVersion: konnect.konghq.com/v1alpha1
+metadata:
+  name: group1
+  namespace: default
+spec:
+  name: group1
+  cluster_type: CLUSTER_TYPE_CONTROL_PLANE_GROUP
+  members:
+  - name: test1
+  - name: test2
+  - name: test3
+  labels:
+    app: group1
+    key1: group1
+  konnect:
+    authRef:
+      name: konnect-api-auth-dev-1

controller/konnect/conditions/conditions.go

@@ -20,6 +20,11 @@ const (
 	// KonnectEntityProgrammedReasonFailedToReconcileConsumerGroupsWithKonnect is the reason for the Programmed condition.
 	// It is set when one or more KongConsumerGroup references could not be reconciled with Konnect.
 	KonnectEntityProgrammedReasonFailedToReconcileConsumerGroupsWithKonnect = "FailedToReconcileConsumerGroupsWithKonnect"
+
+	// KonnectGatewayControlPlaneProgrammedReasonFailedToSetControlPlaneGroupMembers
+	// is the reason for the Programmed condition. It is set when the control plane
+	// group members could not be set.
+	KonnectGatewayControlPlaneProgrammedReasonFailedToSetControlPlaneGroupMembers = "FailedToSetControlPlaneGroupMembers"
 )
 
 const (

controller/konnect/ops/controlplane.go

@@ -7,7 +7,7 @@ import (
 	sdkkonnectops "github.com/Kong/sdk-konnect-go/models/operations"
 )
 
-// ControlPlaneSDK is the interface for the Konnect ControlPlaneSDK SDK.
+// ControlPlaneSDK is the interface for the Konnect ControlPlane SDK.
 type ControlPlaneSDK interface {
 	CreateControlPlane(ctx context.Context, req sdkkonnectcomp.CreateControlPlaneRequest, opts ...sdkkonnectops.Option) (*sdkkonnectops.CreateControlPlaneResponse, error)
 	DeleteControlPlane(ctx context.Context, id string, opts ...sdkkonnectops.Option) (*sdkkonnectops.DeleteControlPlaneResponse, error)

controller/konnect/ops/controlplanegroup.go

@@ -0,0 +1,13 @@
+package ops
+
+import (
+	"context"
+
+	sdkkonnectcomp "github.com/Kong/sdk-konnect-go/models/components"
+	sdkkonnectops "github.com/Kong/sdk-konnect-go/models/operations"
+)
+
+// ControlPlaneGroupSDK is the interface for the Konnect ControlPlaneGroupSDK SDK.
+type ControlPlaneGroupSDK interface {
+	PutControlPlanesIDGroupMemberships(ctx context.Context, id string, groupMembership *sdkkonnectcomp.GroupMembership, opts ...sdkkonnectops.Option) (*sdkkonnectops.PutControlPlanesIDGroupMembershipsResponse, error)
+}

controller/konnect/ops/ops.go

@@ -53,7 +53,7 @@ func Create[
 
 	switch ent := any(e).(type) {
 	case *konnectv1alpha1.KonnectGatewayControlPlane:
-		return e, createControlPlane(ctx, sdk.GetControlPlaneSDK(), ent)
+		return e, createControlPlane(ctx, sdk.GetControlPlaneSDK(), sdk.GetControlPlaneGroupSDK(), cl, ent)
 	case *configurationv1alpha1.KongService:
 		return e, createService(ctx, sdk.GetServicesSDK(), ent)
 	case *configurationv1alpha1.KongRoute:
@@ -226,7 +226,7 @@ func Update[
 
 	switch ent := any(e).(type) {
 	case *konnectv1alpha1.KonnectGatewayControlPlane:
-		return ctrl.Result{}, updateControlPlane(ctx, sdk.GetControlPlaneSDK(), ent)
+		return ctrl.Result{}, updateControlPlane(ctx, sdk.GetControlPlaneSDK(), sdk.GetControlPlaneGroupSDK(), cl, ent)
 	case *configurationv1alpha1.KongService:
 		return ctrl.Result{}, updateService(ctx, sdk.GetServicesSDK(), ent)
 	case *configurationv1alpha1.KongRoute:

controller/konnect/ops/ops_controlplane.go

@@ -3,18 +3,28 @@ package ops
 import (
 	"context"
 	"errors"
+	"fmt"
+	"sort"
 
 	sdkkonnectgo "github.com/Kong/sdk-konnect-go"
 	sdkkonnectcomp "github.com/Kong/sdk-konnect-go/models/components"
 	sdkkonnecterrs "github.com/Kong/sdk-konnect-go/models/sdkerrors"
+	"github.com/samber/lo"
+	"github.com/sourcegraph/conc/iter"
+	corev1 "k8s.io/api/core/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 	ctrllog "sigs.k8s.io/controller-runtime/pkg/log"
 
+	"github.com/kong/gateway-operator/controller/konnect/conditions"
+
 	konnectv1alpha1 "github.com/kong/kubernetes-configuration/api/konnect/v1alpha1"
 )
 
 func createControlPlane(
 	ctx context.Context,
 	sdk ControlPlaneSDK,
+	sdkGroups ControlPlaneGroupSDK,
+	cl client.Client,
 	cp *konnectv1alpha1.KonnectGatewayControlPlane,
 ) error {
 	req := cp.Spec.CreateControlPlaneRequest
@@ -30,6 +40,12 @@ func createControlPlane(
 	}
 
 	cp.Status.SetKonnectID(*resp.ControlPlane.ID)
+
+	if err := setGroupMembers(ctx, cl, cp, sdkGroups); err != nil {
+		SetKonnectEntityProgrammedConditionFalse(cp, conditions.KonnectGatewayControlPlaneProgrammedReasonFailedToSetControlPlaneGroupMembers, err.Error())
+		return err
+	}
+
 	SetKonnectEntityProgrammedCondition(cp)
 
 	return nil
@@ -75,6 +91,8 @@ func deleteControlPlane(
 func updateControlPlane(
 	ctx context.Context,
 	sdk ControlPlaneSDK,
+	sdkGroups ControlPlaneGroupSDK,
+	cl client.Client,
 	cp *konnectv1alpha1.KonnectGatewayControlPlane,
 ) error {
 	id := cp.GetKonnectStatus().GetKonnectID()
@@ -93,7 +111,7 @@ func updateControlPlane(
 			Info("entity not found in Konnect, trying to recreate",
 				"type", cp.GetTypeName(), "id", id,
 			)
-		if err := createControlPlane(ctx, sdk, cp); err != nil {
+		if err := createControlPlane(ctx, sdk, sdkGroups, cl, cp); err != nil {
 			return FailedKonnectOpError[konnectv1alpha1.KonnectGatewayControlPlane]{
 				Op:  UpdateOp,
 				Err: err,
@@ -113,7 +131,70 @@ func updateControlPlane(
 	}
 
 	cp.Status.SetKonnectID(*resp.ControlPlane.ID)
+
+	if err := setGroupMembers(ctx, cl, cp, sdkGroups); err != nil {
+		SetKonnectEntityProgrammedConditionFalse(cp, conditions.KonnectGatewayControlPlaneProgrammedReasonFailedToSetControlPlaneGroupMembers, err.Error())
+		return err
+	}
+
 	SetKonnectEntityProgrammedCondition(cp)
 
 	return nil
 }
+
+func setGroupMembers(
+	ctx context.Context,
+	cl client.Client,
+	cp *konnectv1alpha1.KonnectGatewayControlPlane,
+	sdkGroups ControlPlaneGroupSDK,
+) error {
+	if len(cp.Spec.Members) == 0 ||
+		cp.Spec.ClusterType == nil ||
+		*cp.Spec.ClusterType != sdkkonnectcomp.ClusterTypeClusterTypeControlPlaneGroup {
+		return nil
+	}
+
+	members, err := iter.MapErr(cp.Spec.Members,
+		func(member *corev1.LocalObjectReference) (sdkkonnectcomp.Members, error) {
+			var (
+				memberCP konnectv1alpha1.KonnectGatewayControlPlane
+				nn       = client.ObjectKey{
+					Namespace: cp.Namespace,
+					Name:      member.Name,
+				}
+			)
+			if err := cl.Get(ctx, nn, &memberCP); err != nil {
+				return sdkkonnectcomp.Members{},
+					fmt.Errorf("failed to get control plane group member %s: %w", member.Name, err)
+			}
+			if memberCP.GetKonnectID() == "" {
+				return sdkkonnectcomp.Members{},
+					fmt.Errorf("control plane group %s member %s has no Konnect ID", cp.Name, member.Name)
+			}
+			return sdkkonnectcomp.Members{
+				ID: lo.ToPtr(memberCP.GetKonnectID()),
+			}, nil
+		})
+	if err != nil {
+		return fmt.Errorf("failed to set group members, some members couldn't be found: %w", err)
+	}
+
+	sort.Sort(membersByID(members))
+	gm := sdkkonnectcomp.GroupMembership{
+		Members: members,
+	}
+	_, err = sdkGroups.PutControlPlanesIDGroupMemberships(ctx, cp.GetKonnectID(), &gm)
+	if err != nil {
+		return fmt.Errorf("failed to set members on control plane group %s: %w",
+			client.ObjectKeyFromObject(cp), err,
+		)
+	}
+
+	return nil
+}
+
+type membersByID []sdkkonnectcomp.Members
+
+func (m membersByID) Len() int           { return len(m) }
+func (m membersByID) Less(i, j int) bool { return *m[i].ID < *m[j].ID }
+func (m membersByID) Swap(i, j int)      { m[i], m[j] = m[j], m[i] }