Merge pull request #350 from shapeblue/reconcile-capc-cluster-to-cks

Reconcile CAPC Cluster as CloudStack CKS Cluster (Default: Opt Out)

api/v1beta1/conversion.go

@@ -51,7 +51,7 @@ func Convert_v1beta1_CloudStackCluster_To_v1beta3_CloudStackCluster(in *CloudSta
 //nolint:golint,revive,stylecheck
 func Convert_v1beta3_CloudStackCluster_To_v1beta1_CloudStackCluster(in *v1beta3.CloudStackCluster, out *CloudStackCluster, scope conv.Scope) error {
 	if len(in.Spec.FailureDomains) < 1 {
-		return fmt.Errorf("v1beta3 to v1beta1 conversion not supported when < 1 failure domain is provided. Input CloudStackCluster spec %s", in.Spec)
+		return fmt.Errorf("v1beta3 to v1beta1 conversion not supported when < 1 failure domain is provided. Input CloudStackCluster spec %v", in.Spec)
 	}
 	out.ObjectMeta = in.ObjectMeta
 	out.Spec = CloudStackClusterSpec{

api/v1beta2/cloudstackcluster_conversion.go

@@ -17,6 +17,7 @@ limitations under the License.
 package v1beta2
 
 import (
+	machineryconversion "k8s.io/apimachinery/pkg/conversion"
 	"sigs.k8s.io/cluster-api-provider-cloudstack/api/v1beta3"
 	"sigs.k8s.io/controller-runtime/pkg/conversion"
 )
@@ -30,3 +31,11 @@ func (dst *CloudStackCluster) ConvertFrom(srcRaw conversion.Hub) error { // noli
 	src := srcRaw.(*v1beta3.CloudStackCluster)
 	return Convert_v1beta3_CloudStackCluster_To_v1beta2_CloudStackCluster(src, dst, nil)
 }
+
+func Convert_v1beta3_CloudStackClusterSpec_To_v1beta2_CloudStackClusterSpec(in *v1beta3.CloudStackClusterSpec, out *CloudStackClusterSpec, s machineryconversion.Scope) error { // nolint
+	return autoConvert_v1beta3_CloudStackClusterSpec_To_v1beta2_CloudStackClusterSpec(in, out, s)
+}
+
+func Convert_v1beta3_CloudStackClusterStatus_To_v1beta2_CloudStackClusterStatus(in *v1beta3.CloudStackClusterStatus, out *CloudStackClusterStatus, s machineryconversion.Scope) error { // nolint
+	return autoConvert_v1beta3_CloudStackClusterStatus_To_v1beta2_CloudStackClusterStatus(in, out, s)
+}

api/v1beta3/cloudstackcluster_types.go

@@ -34,6 +34,10 @@ type CloudStackClusterSpec struct {
 
 	// The kubernetes control plane endpoint.
 	ControlPlaneEndpoint clusterv1.APIEndpoint `json:"controlPlaneEndpoint"`
+
+	// SyncWithACS determines if an externalManaged CKS cluster should be created on ACS.
+	// +optional
+	SyncWithACS *bool `json:"syncWithACS,omitempty"`
 }
 
 // The status of the CloudStackCluster object.
@@ -43,6 +47,10 @@ type CloudStackClusterStatus struct {
 	// +optional
 	FailureDomains clusterv1.FailureDomains `json:"failureDomains,omitempty"`
 
+	// Id of CAPC managed kubernetes cluster created in CloudStack
+	// +optional
+	CloudStackClusterID string `json:"cloudStackClusterId"`
+
 	// Reflects the readiness of the CS cluster.
 	Ready bool `json:"ready"`
 }

config/crd/bases/infrastructure.cluster.x-k8s.io_cloudstackclusters.yaml

@@ -415,13 +415,20 @@ spec:
                   - zone
                   type: object
                 type: array
+              syncWithACS:
+                description: SyncWithACS determines if an externalManaged CKS cluster
+                  should be created on ACS.
+                type: boolean
             required:
             - controlPlaneEndpoint
             - failureDomains
             type: object
           status:
             description: The actual cluster state reported by CloudStack.
             properties:
+              cloudStackClusterId:
+                description: Id of CAPC managed kubernetes cluster created in CloudStack
+                type: string
               failureDomains:
                 additionalProperties:
                   description: FailureDomainSpec is the Schema for Cluster API failure

config/manager/manager.yaml

@@ -27,6 +27,7 @@ spec:
         - "--metrics-bind-addr=localhost:8080"
         - "--cloudstackcluster-concurrency=${CAPC_CLOUDSTACKCLUSTER_CONCURRENCY:=10}"
         - "--cloudstackmachine-concurrency=${CAPC_CLOUDSTACKMACHINE_CONCURRENCY:=10}"
+        - "--enable-cloudstack-cks-sync=${CAPC_CLOUDSTACKMACHINE_CKS_SYNC:=false}"
         image: controller:latest
         name: manager
         securityContext:

controllers/cks_cluster_controller.go

@@ -0,0 +1,115 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controllers
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+
+	infrav1 "sigs.k8s.io/cluster-api-provider-cloudstack/api/v1beta3"
+	csCtrlrUtils "sigs.k8s.io/cluster-api-provider-cloudstack/controllers/utils"
+)
+
+const CksClusterFinalizer = "ckscluster.infrastructure.cluster.x-k8s.io"
+
+// RBAC permissions for CloudStackCluster.
+// +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=cloudstackclusters,verbs=get;list;watch;update;patch
+// +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=cloudstackclusters/status,verbs=create;get;update;patch
+// +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=cloudstackclusters/finalizers,verbs=update
+
+// CksClusterReconciliationRunner is a ReconciliationRunner with extensions specific to CloudStackClusters.
+// The runner does the actual reconciliation.
+type CksClusterReconciliationRunner struct {
+	*csCtrlrUtils.ReconciliationRunner
+	FailureDomains        *infrav1.CloudStackFailureDomainList
+	ReconciliationSubject *infrav1.CloudStackCluster
+}
+
+// CksClusterReconciler is the k8s controller manager's interface to reconcile a CloudStackCluster.
+type CksClusterReconciler struct {
+	csCtrlrUtils.ReconcilerBase
+}
+
+// Initialize a new CloudStackCluster reconciliation runner with concrete types and initialized member fields.
+func NewCksClusterReconciliationRunner() *CksClusterReconciliationRunner {
+	// Set concrete type and init pointers.
+	runner := &CksClusterReconciliationRunner{ReconciliationSubject: &infrav1.CloudStackCluster{}}
+	runner.FailureDomains = &infrav1.CloudStackFailureDomainList{}
+	// Setup the base runner. Initializes pointers and links reconciliation methods.
+	runner.ReconciliationRunner = csCtrlrUtils.NewRunner(runner, runner.ReconciliationSubject, "CKSClusterController")
+	runner.CSCluster = runner.ReconciliationSubject
+	return runner
+}
+
+// Reconcile is the method k8s will call upon a reconciliation request.
+func (reconciler *CksClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) (retRes ctrl.Result, retErr error) {
+	r := NewCksClusterReconciliationRunner()
+	r.UsingBaseReconciler(reconciler.ReconcilerBase).ForRequest(req).WithRequestCtx(ctx)
+	r.WithAdditionalCommonStages(r.GetFailureDomains(r.FailureDomains))
+	return r.RunBaseReconciliationStages()
+}
+
+// Reconcile actually reconciles the CloudStackCluster.
+func (r *CksClusterReconciliationRunner) Reconcile() (res ctrl.Result, reterr error) {
+	if r.CSCluster.Spec.SyncWithACS == nil || !*r.CSCluster.Spec.SyncWithACS || len(r.FailureDomains.Items) == 0 {
+		return ctrl.Result{}, nil
+	}
+	// Prevent premature deletion.
+	controllerutil.AddFinalizer(r.ReconciliationSubject, CksClusterFinalizer)
+
+	res, err := r.AsFailureDomainUser(&r.FailureDomains.Items[0].Spec)()
+	if r.ShouldReturn(res, err) {
+		return res, err
+	}
+
+	r.Log.Info("Creating entry with CKS")
+	err = r.CSUser.GetOrCreateCksCluster(r.CAPICluster, r.ReconciliationSubject, &r.FailureDomains.Items[0].Spec)
+	if err != nil {
+		return r.RequeueWithMessage(fmt.Sprintf("Failed creating ExternalManaged CKS cluster on CloudStack. error: %s", err.Error()))
+	}
+	return ctrl.Result{}, nil
+}
+
+// ReconcileDelete cleans up resources used by the cluster and finally removes the CloudStackCluster's finalizers.
+func (r *CksClusterReconciliationRunner) ReconcileDelete() (ctrl.Result, error) {
+	if r.ReconciliationSubject.Status.CloudStackClusterID != "" {
+		if len(r.FailureDomains.Items) == 0 {
+			return ctrl.Result{}, fmt.Errorf("no failure domains found")
+		}
+		res, err := r.AsFailureDomainUser(&r.FailureDomains.Items[0].Spec)()
+		if r.ShouldReturn(res, err) {
+			return res, err
+		}
+		err = r.CSUser.DeleteCksCluster(r.ReconciliationSubject)
+		if err != nil && !strings.Contains(err.Error(), " not found") {
+			return r.RequeueWithMessage(fmt.Sprintf("Deleting cks cluster on CloudStack failed. error: %s", err.Error()))
+		}
+	}
+	controllerutil.RemoveFinalizer(r.ReconciliationSubject, CksClusterFinalizer)
+	return ctrl.Result{}, nil
+}
+
+// SetupWithManager sets up the controller with the Manager.
+func (reconciler *CksClusterReconciler) SetupWithManager(mgr ctrl.Manager) error {
+	return ctrl.NewControllerManagedBy(mgr).
+		For(&infrav1.CloudStackCluster{}).
+		Complete(reconciler)
+}

controllers/cks_cluster_controller_test.go

@@ -0,0 +1,96 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controllers_test
+
+import (
+	"github.com/golang/mock/gomock"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	"k8s.io/apimachinery/pkg/api/errors"
+	infrav1 "sigs.k8s.io/cluster-api-provider-cloudstack/api/v1beta3"
+	"sigs.k8s.io/cluster-api-provider-cloudstack/controllers"
+	"sigs.k8s.io/cluster-api-provider-cloudstack/pkg/cloud"
+	dummies "sigs.k8s.io/cluster-api-provider-cloudstack/test/dummies/v1beta3"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller"
+)
+
+var _ = Describe("CksCloudStackClusterReconciler", func() {
+	Context("With k8s like test environment.", func() {
+		BeforeEach(func() {
+			dummies.SetDummyVars()
+			SetupTestEnvironment()
+			Ω(ClusterReconciler.SetupWithManager(ctx, k8sManager, controller.Options{})).Should(Succeed())  // Register CloudStack ClusterReconciler.
+			Ω(FailureDomainReconciler.SetupWithManager(k8sManager, controller.Options{})).Should(Succeed()) // Register CloudStack FailureDomainReconciler.
+			Ω(CksClusterReconciler.SetupWithManager(k8sManager)).Should(Succeed())                          // Register CloudStack Cks ClusterReconciler.
+		})
+
+		It("Should create a cluster in CKS.", func() {
+			mockCloudClient.EXPECT().GetOrCreateCksCluster(gomock.Any(), gomock.Any(), gomock.Any()).Do(func(_, arg1, _ interface{}) {
+				arg1.(*infrav1.CloudStackCluster).Status.CloudStackClusterID = "cluster-id-123"
+			}).MinTimes(1).Return(nil)
+			mockCloudClient.EXPECT().ResolveZone(gomock.Any()).AnyTimes()
+			mockCloudClient.EXPECT().ResolveNetworkForZone(gomock.Any()).AnyTimes().Do(
+				func(arg1 interface{}) {
+					arg1.(*infrav1.CloudStackZoneSpec).Network.ID = "SomeID"
+					arg1.(*infrav1.CloudStackZoneSpec).Network.Type = cloud.NetworkTypeShared
+				}).MinTimes(1)
+
+			Eventually(func() string {
+				key := client.ObjectKeyFromObject(dummies.CSCluster)
+				if err := k8sClient.Get(ctx, key, dummies.CSCluster); err != nil {
+					return ""
+				}
+				return dummies.CSCluster.Status.CloudStackClusterID
+			}, timeout).WithPolling(pollInterval).Should(Equal("cluster-id-123"))
+
+		})
+	})
+
+	Context("With k8s like test environment.", func() {
+		BeforeEach(func() {
+			dummies.SetDummyVars()
+			dummies.CSCluster.Status.CloudStackClusterID = "cluster-id-123"
+			SetupTestEnvironment()
+			Ω(ClusterReconciler.SetupWithManager(ctx, k8sManager, controller.Options{})).Should(Succeed())  // Register CloudStack ClusterReconciler.
+			Ω(FailureDomainReconciler.SetupWithManager(k8sManager, controller.Options{})).Should(Succeed()) // Register CloudStack FailureDomainReconciler.
+			Ω(CksClusterReconciler.SetupWithManager(k8sManager)).Should(Succeed())                          // Register CloudStack Cks ClusterReconciler.
+		})
+
+		It("Should delete the cluster in CKS.", func() {
+
+			Ω(k8sClient.Delete(ctx, dummies.CSCluster)).Should(Succeed())
+
+			Eventually(func() bool {
+				csCluster := &infrav1.CloudStackCluster{}
+				csClusterKey := client.ObjectKey{Namespace: dummies.ClusterNameSpace, Name: dummies.CSCluster.Name}
+				if err := k8sClient.Get(ctx, csClusterKey, csCluster); err != nil {
+					return errors.IsNotFound(err)
+				}
+				return false
+			}, timeout).WithPolling(pollInterval).Should(BeTrue())
+		})
+
+	})
+
+	Context("Without a k8s test environment.", func() {
+		It("Should create a reconciliation runner with a Cloudstack Cluster as the reconciliation subject.", func() {
+			reconRunner := controllers.NewCksClusterReconciliationRunner()
+			Ω(reconRunner.ReconciliationSubject).ShouldNot(BeNil())
+		})
+	})
+})