diff --git a/api/v1beta1/conversion.go b/api/v1beta1/conversion.go index 236685f6..944a9661 100644 --- a/api/v1beta1/conversion.go +++ b/api/v1beta1/conversion.go @@ -51,7 +51,7 @@ func Convert_v1beta1_CloudStackCluster_To_v1beta3_CloudStackCluster(in *CloudSta //nolint:golint,revive,stylecheck func Convert_v1beta3_CloudStackCluster_To_v1beta1_CloudStackCluster(in *v1beta3.CloudStackCluster, out *CloudStackCluster, scope conv.Scope) error { if len(in.Spec.FailureDomains) < 1 { - return fmt.Errorf("v1beta3 to v1beta1 conversion not supported when < 1 failure domain is provided. Input CloudStackCluster spec %s", in.Spec) + return fmt.Errorf("v1beta3 to v1beta1 conversion not supported when < 1 failure domain is provided. Input CloudStackCluster spec %v", in.Spec) } out.ObjectMeta = in.ObjectMeta out.Spec = CloudStackClusterSpec{ diff --git a/api/v1beta2/cloudstackcluster_conversion.go b/api/v1beta2/cloudstackcluster_conversion.go index 31fd3c20..dbd8a21a 100644 --- a/api/v1beta2/cloudstackcluster_conversion.go +++ b/api/v1beta2/cloudstackcluster_conversion.go @@ -17,6 +17,7 @@ limitations under the License. package v1beta2 import ( + machineryconversion "k8s.io/apimachinery/pkg/conversion" "sigs.k8s.io/cluster-api-provider-cloudstack/api/v1beta3" "sigs.k8s.io/controller-runtime/pkg/conversion" ) @@ -30,3 +31,11 @@ func (dst *CloudStackCluster) ConvertFrom(srcRaw conversion.Hub) error { // noli src := srcRaw.(*v1beta3.CloudStackCluster) return Convert_v1beta3_CloudStackCluster_To_v1beta2_CloudStackCluster(src, dst, nil) } + +func Convert_v1beta3_CloudStackClusterSpec_To_v1beta2_CloudStackClusterSpec(in *v1beta3.CloudStackClusterSpec, out *CloudStackClusterSpec, s machineryconversion.Scope) error { // nolint + return autoConvert_v1beta3_CloudStackClusterSpec_To_v1beta2_CloudStackClusterSpec(in, out, s) +} + +func Convert_v1beta3_CloudStackClusterStatus_To_v1beta2_CloudStackClusterStatus(in *v1beta3.CloudStackClusterStatus, out *CloudStackClusterStatus, s machineryconversion.Scope) error { // nolint + return autoConvert_v1beta3_CloudStackClusterStatus_To_v1beta2_CloudStackClusterStatus(in, out, s) +} diff --git a/api/v1beta2/zz_generated.conversion.go b/api/v1beta2/zz_generated.conversion.go index 026a8bd5..9573ab25 100644 --- a/api/v1beta2/zz_generated.conversion.go +++ b/api/v1beta2/zz_generated.conversion.go @@ -103,21 +103,11 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } - if err := s.AddGeneratedConversionFunc((*v1beta3.CloudStackClusterSpec)(nil), (*CloudStackClusterSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta3_CloudStackClusterSpec_To_v1beta2_CloudStackClusterSpec(a.(*v1beta3.CloudStackClusterSpec), b.(*CloudStackClusterSpec), scope) - }); err != nil { - return err - } if err := s.AddGeneratedConversionFunc((*CloudStackClusterStatus)(nil), (*v1beta3.CloudStackClusterStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1beta2_CloudStackClusterStatus_To_v1beta3_CloudStackClusterStatus(a.(*CloudStackClusterStatus), b.(*v1beta3.CloudStackClusterStatus), scope) }); err != nil { return err } - if err := s.AddGeneratedConversionFunc((*v1beta3.CloudStackClusterStatus)(nil), (*CloudStackClusterStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta3_CloudStackClusterStatus_To_v1beta2_CloudStackClusterStatus(a.(*v1beta3.CloudStackClusterStatus), b.(*CloudStackClusterStatus), scope) - }); err != nil { - return err - } if err := s.AddGeneratedConversionFunc((*CloudStackFailureDomain)(nil), (*v1beta3.CloudStackFailureDomain)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1beta2_CloudStackFailureDomain_To_v1beta3_CloudStackFailureDomain(a.(*CloudStackFailureDomain), b.(*v1beta3.CloudStackFailureDomain), scope) }); err != nil { @@ -358,6 +348,16 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddConversionFunc((*v1beta3.CloudStackClusterSpec)(nil), (*CloudStackClusterSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta3_CloudStackClusterSpec_To_v1beta2_CloudStackClusterSpec(a.(*v1beta3.CloudStackClusterSpec), b.(*CloudStackClusterSpec), scope) + }); err != nil { + return err + } + if err := s.AddConversionFunc((*v1beta3.CloudStackClusterStatus)(nil), (*CloudStackClusterStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta3_CloudStackClusterStatus_To_v1beta2_CloudStackClusterStatus(a.(*v1beta3.CloudStackClusterStatus), b.(*CloudStackClusterStatus), scope) + }); err != nil { + return err + } if err := s.AddConversionFunc((*v1beta3.CloudStackFailureDomainSpec)(nil), (*CloudStackFailureDomainSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1beta3_CloudStackFailureDomainSpec_To_v1beta2_CloudStackFailureDomainSpec(a.(*v1beta3.CloudStackFailureDomainSpec), b.(*CloudStackFailureDomainSpec), scope) }); err != nil { @@ -579,14 +579,10 @@ func autoConvert_v1beta3_CloudStackClusterSpec_To_v1beta2_CloudStackClusterSpec( out.FailureDomains = nil } out.ControlPlaneEndpoint = in.ControlPlaneEndpoint + // WARNING: in.SyncWithACS requires manual conversion: does not exist in peer-type return nil } -// Convert_v1beta3_CloudStackClusterSpec_To_v1beta2_CloudStackClusterSpec is an autogenerated conversion function. -func Convert_v1beta3_CloudStackClusterSpec_To_v1beta2_CloudStackClusterSpec(in *v1beta3.CloudStackClusterSpec, out *CloudStackClusterSpec, s conversion.Scope) error { - return autoConvert_v1beta3_CloudStackClusterSpec_To_v1beta2_CloudStackClusterSpec(in, out, s) -} - func autoConvert_v1beta2_CloudStackClusterStatus_To_v1beta3_CloudStackClusterStatus(in *CloudStackClusterStatus, out *v1beta3.CloudStackClusterStatus, s conversion.Scope) error { out.FailureDomains = *(*v1beta1.FailureDomains)(unsafe.Pointer(&in.FailureDomains)) out.Ready = in.Ready @@ -600,15 +596,11 @@ func Convert_v1beta2_CloudStackClusterStatus_To_v1beta3_CloudStackClusterStatus( func autoConvert_v1beta3_CloudStackClusterStatus_To_v1beta2_CloudStackClusterStatus(in *v1beta3.CloudStackClusterStatus, out *CloudStackClusterStatus, s conversion.Scope) error { out.FailureDomains = *(*v1beta1.FailureDomains)(unsafe.Pointer(&in.FailureDomains)) + // WARNING: in.CloudStackClusterID requires manual conversion: does not exist in peer-type out.Ready = in.Ready return nil } -// Convert_v1beta3_CloudStackClusterStatus_To_v1beta2_CloudStackClusterStatus is an autogenerated conversion function. -func Convert_v1beta3_CloudStackClusterStatus_To_v1beta2_CloudStackClusterStatus(in *v1beta3.CloudStackClusterStatus, out *CloudStackClusterStatus, s conversion.Scope) error { - return autoConvert_v1beta3_CloudStackClusterStatus_To_v1beta2_CloudStackClusterStatus(in, out, s) -} - func autoConvert_v1beta2_CloudStackFailureDomain_To_v1beta3_CloudStackFailureDomain(in *CloudStackFailureDomain, out *v1beta3.CloudStackFailureDomain, s conversion.Scope) error { out.ObjectMeta = in.ObjectMeta if err := Convert_v1beta2_CloudStackFailureDomainSpec_To_v1beta3_CloudStackFailureDomainSpec(&in.Spec, &out.Spec, s); err != nil { diff --git a/api/v1beta3/cloudstackcluster_types.go b/api/v1beta3/cloudstackcluster_types.go index 1b47ff89..455f3994 100644 --- a/api/v1beta3/cloudstackcluster_types.go +++ b/api/v1beta3/cloudstackcluster_types.go @@ -34,6 +34,10 @@ type CloudStackClusterSpec struct { // The kubernetes control plane endpoint. ControlPlaneEndpoint clusterv1.APIEndpoint `json:"controlPlaneEndpoint"` + + // SyncWithACS determines if an externalManaged CKS cluster should be created on ACS. + // +optional + SyncWithACS *bool `json:"syncWithACS,omitempty"` } // The status of the CloudStackCluster object. @@ -43,6 +47,10 @@ type CloudStackClusterStatus struct { // +optional FailureDomains clusterv1.FailureDomains `json:"failureDomains,omitempty"` + // Id of CAPC managed kubernetes cluster created in CloudStack + // +optional + CloudStackClusterID string `json:"cloudStackClusterId"` + // Reflects the readiness of the CS cluster. Ready bool `json:"ready"` } diff --git a/api/v1beta3/zz_generated.deepcopy.go b/api/v1beta3/zz_generated.deepcopy.go index 993a974a..a6919a22 100644 --- a/api/v1beta3/zz_generated.deepcopy.go +++ b/api/v1beta3/zz_generated.deepcopy.go @@ -184,6 +184,11 @@ func (in *CloudStackClusterSpec) DeepCopyInto(out *CloudStackClusterSpec) { copy(*out, *in) } out.ControlPlaneEndpoint = in.ControlPlaneEndpoint + if in.SyncWithACS != nil { + in, out := &in.SyncWithACS, &out.SyncWithACS + *out = new(bool) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudStackClusterSpec. diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_cloudstackclusters.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_cloudstackclusters.yaml index 882a42eb..81bbca78 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_cloudstackclusters.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_cloudstackclusters.yaml @@ -415,6 +415,10 @@ spec: - zone type: object type: array + syncWithACS: + description: SyncWithACS determines if an externalManaged CKS cluster + should be created on ACS. + type: boolean required: - controlPlaneEndpoint - failureDomains @@ -422,6 +426,9 @@ spec: status: description: The actual cluster state reported by CloudStack. properties: + cloudStackClusterId: + description: Id of CAPC managed kubernetes cluster created in CloudStack + type: string failureDomains: additionalProperties: description: FailureDomainSpec is the Schema for Cluster API failure diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index a66ade9d..970ca541 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -27,6 +27,7 @@ spec: - "--metrics-bind-addr=localhost:8080" - "--cloudstackcluster-concurrency=${CAPC_CLOUDSTACKCLUSTER_CONCURRENCY:=10}" - "--cloudstackmachine-concurrency=${CAPC_CLOUDSTACKMACHINE_CONCURRENCY:=10}" + - "--enable-cloudstack-cks-sync=${CAPC_CLOUDSTACKMACHINE_CKS_SYNC:=false}" image: controller:latest name: manager securityContext: diff --git a/controllers/cks_cluster_controller.go b/controllers/cks_cluster_controller.go new file mode 100644 index 00000000..25116518 --- /dev/null +++ b/controllers/cks_cluster_controller.go @@ -0,0 +1,115 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + "fmt" + "strings" + + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + infrav1 "sigs.k8s.io/cluster-api-provider-cloudstack/api/v1beta3" + csCtrlrUtils "sigs.k8s.io/cluster-api-provider-cloudstack/controllers/utils" +) + +const CksClusterFinalizer = "ckscluster.infrastructure.cluster.x-k8s.io" + +// RBAC permissions for CloudStackCluster. +// +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=cloudstackclusters,verbs=get;list;watch;update;patch +// +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=cloudstackclusters/status,verbs=create;get;update;patch +// +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=cloudstackclusters/finalizers,verbs=update + +// CksClusterReconciliationRunner is a ReconciliationRunner with extensions specific to CloudStackClusters. +// The runner does the actual reconciliation. +type CksClusterReconciliationRunner struct { + *csCtrlrUtils.ReconciliationRunner + FailureDomains *infrav1.CloudStackFailureDomainList + ReconciliationSubject *infrav1.CloudStackCluster +} + +// CksClusterReconciler is the k8s controller manager's interface to reconcile a CloudStackCluster. +type CksClusterReconciler struct { + csCtrlrUtils.ReconcilerBase +} + +// Initialize a new CloudStackCluster reconciliation runner with concrete types and initialized member fields. +func NewCksClusterReconciliationRunner() *CksClusterReconciliationRunner { + // Set concrete type and init pointers. + runner := &CksClusterReconciliationRunner{ReconciliationSubject: &infrav1.CloudStackCluster{}} + runner.FailureDomains = &infrav1.CloudStackFailureDomainList{} + // Setup the base runner. Initializes pointers and links reconciliation methods. + runner.ReconciliationRunner = csCtrlrUtils.NewRunner(runner, runner.ReconciliationSubject, "CKSClusterController") + runner.CSCluster = runner.ReconciliationSubject + return runner +} + +// Reconcile is the method k8s will call upon a reconciliation request. +func (reconciler *CksClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) (retRes ctrl.Result, retErr error) { + r := NewCksClusterReconciliationRunner() + r.UsingBaseReconciler(reconciler.ReconcilerBase).ForRequest(req).WithRequestCtx(ctx) + r.WithAdditionalCommonStages(r.GetFailureDomains(r.FailureDomains)) + return r.RunBaseReconciliationStages() +} + +// Reconcile actually reconciles the CloudStackCluster. +func (r *CksClusterReconciliationRunner) Reconcile() (res ctrl.Result, reterr error) { + if r.CSCluster.Spec.SyncWithACS == nil || !*r.CSCluster.Spec.SyncWithACS || len(r.FailureDomains.Items) == 0 { + return ctrl.Result{}, nil + } + // Prevent premature deletion. + controllerutil.AddFinalizer(r.ReconciliationSubject, CksClusterFinalizer) + + res, err := r.AsFailureDomainUser(&r.FailureDomains.Items[0].Spec)() + if r.ShouldReturn(res, err) { + return res, err + } + + r.Log.Info("Creating entry with CKS") + err = r.CSUser.GetOrCreateCksCluster(r.CAPICluster, r.ReconciliationSubject, &r.FailureDomains.Items[0].Spec) + if err != nil { + return r.RequeueWithMessage(fmt.Sprintf("Failed creating ExternalManaged CKS cluster on CloudStack. error: %s", err.Error())) + } + return ctrl.Result{}, nil +} + +// ReconcileDelete cleans up resources used by the cluster and finally removes the CloudStackCluster's finalizers. +func (r *CksClusterReconciliationRunner) ReconcileDelete() (ctrl.Result, error) { + if r.ReconciliationSubject.Status.CloudStackClusterID != "" { + if len(r.FailureDomains.Items) == 0 { + return ctrl.Result{}, fmt.Errorf("no failure domains found") + } + res, err := r.AsFailureDomainUser(&r.FailureDomains.Items[0].Spec)() + if r.ShouldReturn(res, err) { + return res, err + } + err = r.CSUser.DeleteCksCluster(r.ReconciliationSubject) + if err != nil && !strings.Contains(err.Error(), " not found") { + return r.RequeueWithMessage(fmt.Sprintf("Deleting cks cluster on CloudStack failed. error: %s", err.Error())) + } + } + controllerutil.RemoveFinalizer(r.ReconciliationSubject, CksClusterFinalizer) + return ctrl.Result{}, nil +} + +// SetupWithManager sets up the controller with the Manager. +func (reconciler *CksClusterReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&infrav1.CloudStackCluster{}). + Complete(reconciler) +} diff --git a/controllers/cks_cluster_controller_test.go b/controllers/cks_cluster_controller_test.go new file mode 100644 index 00000000..aad1449e --- /dev/null +++ b/controllers/cks_cluster_controller_test.go @@ -0,0 +1,96 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers_test + +import ( + "github.com/golang/mock/gomock" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/api/errors" + infrav1 "sigs.k8s.io/cluster-api-provider-cloudstack/api/v1beta3" + "sigs.k8s.io/cluster-api-provider-cloudstack/controllers" + "sigs.k8s.io/cluster-api-provider-cloudstack/pkg/cloud" + dummies "sigs.k8s.io/cluster-api-provider-cloudstack/test/dummies/v1beta3" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" +) + +var _ = Describe("CksCloudStackClusterReconciler", func() { + Context("With k8s like test environment.", func() { + BeforeEach(func() { + dummies.SetDummyVars() + SetupTestEnvironment() + Ω(ClusterReconciler.SetupWithManager(ctx, k8sManager, controller.Options{})).Should(Succeed()) // Register CloudStack ClusterReconciler. + Ω(FailureDomainReconciler.SetupWithManager(k8sManager, controller.Options{})).Should(Succeed()) // Register CloudStack FailureDomainReconciler. + Ω(CksClusterReconciler.SetupWithManager(k8sManager)).Should(Succeed()) // Register CloudStack Cks ClusterReconciler. + }) + + It("Should create a cluster in CKS.", func() { + mockCloudClient.EXPECT().GetOrCreateCksCluster(gomock.Any(), gomock.Any(), gomock.Any()).Do(func(_, arg1, _ interface{}) { + arg1.(*infrav1.CloudStackCluster).Status.CloudStackClusterID = "cluster-id-123" + }).MinTimes(1).Return(nil) + mockCloudClient.EXPECT().ResolveZone(gomock.Any()).AnyTimes() + mockCloudClient.EXPECT().ResolveNetworkForZone(gomock.Any()).AnyTimes().Do( + func(arg1 interface{}) { + arg1.(*infrav1.CloudStackZoneSpec).Network.ID = "SomeID" + arg1.(*infrav1.CloudStackZoneSpec).Network.Type = cloud.NetworkTypeShared + }).MinTimes(1) + + Eventually(func() string { + key := client.ObjectKeyFromObject(dummies.CSCluster) + if err := k8sClient.Get(ctx, key, dummies.CSCluster); err != nil { + return "" + } + return dummies.CSCluster.Status.CloudStackClusterID + }, timeout).WithPolling(pollInterval).Should(Equal("cluster-id-123")) + + }) + }) + + Context("With k8s like test environment.", func() { + BeforeEach(func() { + dummies.SetDummyVars() + dummies.CSCluster.Status.CloudStackClusterID = "cluster-id-123" + SetupTestEnvironment() + Ω(ClusterReconciler.SetupWithManager(ctx, k8sManager, controller.Options{})).Should(Succeed()) // Register CloudStack ClusterReconciler. + Ω(FailureDomainReconciler.SetupWithManager(k8sManager, controller.Options{})).Should(Succeed()) // Register CloudStack FailureDomainReconciler. + Ω(CksClusterReconciler.SetupWithManager(k8sManager)).Should(Succeed()) // Register CloudStack Cks ClusterReconciler. + }) + + It("Should delete the cluster in CKS.", func() { + + Ω(k8sClient.Delete(ctx, dummies.CSCluster)).Should(Succeed()) + + Eventually(func() bool { + csCluster := &infrav1.CloudStackCluster{} + csClusterKey := client.ObjectKey{Namespace: dummies.ClusterNameSpace, Name: dummies.CSCluster.Name} + if err := k8sClient.Get(ctx, csClusterKey, csCluster); err != nil { + return errors.IsNotFound(err) + } + return false + }, timeout).WithPolling(pollInterval).Should(BeTrue()) + }) + + }) + + Context("Without a k8s test environment.", func() { + It("Should create a reconciliation runner with a Cloudstack Cluster as the reconciliation subject.", func() { + reconRunner := controllers.NewCksClusterReconciliationRunner() + Ω(reconRunner.ReconciliationSubject).ShouldNot(BeNil()) + }) + }) +}) diff --git a/controllers/cks_machine_controller.go b/controllers/cks_machine_controller.go new file mode 100644 index 00000000..574f9322 --- /dev/null +++ b/controllers/cks_machine_controller.go @@ -0,0 +1,119 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + "fmt" + + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + infrav1 "sigs.k8s.io/cluster-api-provider-cloudstack/api/v1beta3" + csCtrlrUtils "sigs.k8s.io/cluster-api-provider-cloudstack/controllers/utils" +) + +const CksMachineFinalizer = "cksMachine.infrastructure.cluster.x-k8s.io" + +// RBAC permissions for CloudStackMachines. +// +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=cloudstackmachines,verbs=get;list;watch +// +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=cloudstackmachines/status,verbs=get +// +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=cloudstackmachines/finalizers,verbs=update + +// CksMachineReconciliationRunner is a ReconciliationRunner with extensions specific to CloudStackMachines. +// The runner does the actual reconciliation. +type CksMachineReconciliationRunner struct { + *csCtrlrUtils.ReconciliationRunner + FailureDomain *infrav1.CloudStackFailureDomain + ReconciliationSubject *infrav1.CloudStackMachine +} + +// CksMachineReconciler is the k8s controller manager's interface to reconcile CloudStackMachines with a CKS cluster. +type CksMachineReconciler struct { + csCtrlrUtils.ReconcilerBase +} + +// Initialize a new CloudStackMachines reconciliation runner with concrete types and initialized member fields. +func NewCksMachineReconciliationRunner() *CksMachineReconciliationRunner { + // Set concrete type and init pointers. + runner := &CksMachineReconciliationRunner{ReconciliationSubject: &infrav1.CloudStackMachine{}} + runner.FailureDomain = &infrav1.CloudStackFailureDomain{} + // Setup the base runner. Initializes pointers and links reconciliation methods. + runner.ReconciliationRunner = csCtrlrUtils.NewRunner(runner, runner.ReconciliationSubject, "CKSMachineController") + return runner +} + +// Reconcile is the method k8s will call upon a reconciliation request. +func (reconciler *CksMachineReconciler) Reconcile(ctx context.Context, req ctrl.Request) (retRes ctrl.Result, retErr error) { + r := NewCksMachineReconciliationRunner() + r.UsingBaseReconciler(reconciler.ReconcilerBase).ForRequest(req).WithRequestCtx(ctx) + r.WithAdditionalCommonStages( + r.GetFailureDomainByName(func() string { return r.ReconciliationSubject.Spec.FailureDomainName }, r.FailureDomain), + r.AsFailureDomainUser(&r.FailureDomain.Spec)) + return r.RunBaseReconciliationStages() +} + +// Reconcile actually reconciles the CloudStackMachine. +func (r *CksMachineReconciliationRunner) Reconcile() (res ctrl.Result, reterr error) { + if r.CSCluster.Spec.SyncWithACS == nil || !*r.CSCluster.Spec.SyncWithACS { + return ctrl.Result{}, nil + } + if r.CSCluster.Status.CloudStackClusterID == "" { + return r.RequeueWithMessage("CloudStackClusterID is not set") + } + + if r.ReconciliationSubject.Spec.InstanceID == nil || *r.ReconciliationSubject.Spec.InstanceID == "" { + return r.RequeueWithMessage("InstanceID is not set") + } + + // Prevent premature deletion. + controllerutil.AddFinalizer(r.ReconciliationSubject, CksMachineFinalizer) + + res, err := r.AsFailureDomainUser(&r.FailureDomain.Spec)() + if r.ShouldReturn(res, err) { + return res, err + } + r.Log.Info("Assigning VM to CKS") + err = r.CSUser.AddVMToCksCluster(r.CSCluster, r.ReconciliationSubject) + if err != nil { + return r.RequeueWithMessage(fmt.Sprintf("Adding VM to CloudStack CKS cluster failed. error: %s", err.Error())) + } + r.Log.Info("Assigned VM to CKS") + return ctrl.Result{}, nil + +} + +// ReconcileDelete cleans up resources used by the cluster and finally removes the CloudStackMachine's finalizers. +func (r *CksMachineReconciliationRunner) ReconcileDelete() (ctrl.Result, error) { + r.Log.Info("Removing VM from CKS") + if r.ReconciliationSubject.Spec.InstanceID != nil && *r.ReconciliationSubject.Spec.InstanceID != "" { + err := r.CSUser.RemoveVMFromCksCluster(r.CSCluster, r.ReconciliationSubject) + if err != nil { + return r.RequeueWithMessage(fmt.Sprintf("Removing VM from CloudStack CKS cluster failed. error: %s", err.Error())) + } + } + r.Log.Info("Removed VM from CKS") + controllerutil.RemoveFinalizer(r.ReconciliationSubject, CksMachineFinalizer) + return ctrl.Result{}, nil +} + +// SetupWithManager sets up the controller with the Manager. +func (reconciler *CksMachineReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&infrav1.CloudStackMachine{}). + Complete(reconciler) +} diff --git a/controllers/cks_machine_controller_test.go b/controllers/cks_machine_controller_test.go new file mode 100644 index 00000000..8847e81e --- /dev/null +++ b/controllers/cks_machine_controller_test.go @@ -0,0 +1,135 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers_test + +import ( + "github.com/golang/mock/gomock" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/utils/pointer" + infrav1 "sigs.k8s.io/cluster-api-provider-cloudstack/api/v1beta3" + "sigs.k8s.io/cluster-api-provider-cloudstack/controllers" + dummies "sigs.k8s.io/cluster-api-provider-cloudstack/test/dummies/v1beta3" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" +) + +var _ = Describe("CksCloudStackMachineReconciler", func() { + Context("With machine controller running.", func() { + BeforeEach(func() { + dummies.SetDummyVars() + dummies.CSCluster.Spec.SyncWithACS = pointer.Bool(true) + dummies.CSCluster.Spec.FailureDomains = dummies.CSCluster.Spec.FailureDomains[:1] + dummies.CSCluster.Spec.FailureDomains[0].Name = dummies.CSFailureDomain1.Spec.Name + dummies.CSCluster.Status.CloudStackClusterID = "cluster-id-123" + + SetupTestEnvironment() // Must happen before setting up managers/reconcilers. + Ω(MachineReconciler.SetupWithManager(ctx, k8sManager, controller.Options{})).Should(Succeed()) // Register the CloudStack MachineReconciler. + Ω(CksClusterReconciler.SetupWithManager(k8sManager)).Should(Succeed()) // Register the CloudStack MachineReconciler. + Ω(CksMachineReconciler.SetupWithManager(k8sManager)).Should(Succeed()) // Register the CloudStack MachineReconciler. + + mockCloudClient.EXPECT().GetOrCreateCksCluster(gomock.Any(), gomock.Any(), gomock.Any()).Do(func(_, arg1, _ interface{}) { + arg1.(*infrav1.CloudStackCluster).Status.CloudStackClusterID = "cluster-id-123" + }).MinTimes(1).Return(nil) + // Point CAPI machine Bootstrap secret ref to dummy bootstrap secret. + dummies.CAPIMachine.Spec.Bootstrap.DataSecretName = &dummies.BootstrapSecret.Name + Ω(k8sClient.Create(ctx, dummies.BootstrapSecret)).Should(Succeed()) + + // Setup a failure domain for the machine reconciler to find. + Ω(k8sClient.Create(ctx, dummies.CSFailureDomain1)).Should(Succeed()) + setClusterReady(k8sClient) + }) + + It("Should call AddVMToCksCluster", func() { + // Mock a call to GetOrCreateVMInstance and set the machine to running. + mockCloudClient.EXPECT().GetOrCreateVMInstance( + gomock.Any(), gomock.Any(), gomock.Any(), + gomock.Any(), gomock.Any(), gomock.Any()).Do( + func(arg1, _, _, _, _, _ interface{}) { + arg1.(*infrav1.CloudStackMachine).Status.InstanceState = "Running" + }).AnyTimes() + + mockCloudClient.EXPECT().AddVMToCksCluster( + gomock.Any(), gomock.Any()).MinTimes(1).Return(nil) + // Have to do this here or the reconcile call to GetOrCreateVMInstance may happen too early. + setupMachineCRDs() + + // Eventually the machine should set ready to true. + Eventually(func() bool { + tempMachine := &infrav1.CloudStackMachine{} + key := client.ObjectKey{Namespace: dummies.ClusterNameSpace, Name: dummies.CSMachine1.Name} + if err := k8sClient.Get(ctx, key, tempMachine); err == nil { + if tempMachine.Status.Ready == true { + return len(tempMachine.ObjectMeta.Finalizers) > 1 + } + } + return false + }, timeout).WithPolling(pollInterval).Should(BeTrue()) + }) + + It("Should call RemoveVMFromCksCluster when CS machine deleted", func() { + // Mock a call to GetOrCreateVMInstance and set the machine to running. + mockCloudClient.EXPECT().GetOrCreateVMInstance( + gomock.Any(), gomock.Any(), gomock.Any(), + gomock.Any(), gomock.Any(), gomock.Any()).Do( + func(arg1, _, _, _, _, _ interface{}) { + arg1.(*infrav1.CloudStackMachine).Status.InstanceState = "Running" + controllerutil.AddFinalizer(arg1.(*infrav1.CloudStackMachine), infrav1.MachineFinalizer) + }).AnyTimes() + + mockCloudClient.EXPECT().AddVMToCksCluster(gomock.Any(), gomock.Any()).MinTimes(1).Return(nil) + + mockCloudClient.EXPECT().DestroyVMInstance(gomock.Any()).MinTimes(1).Return(nil) + mockCloudClient.EXPECT().RemoveVMFromCksCluster( + gomock.Any(), gomock.Any()).MinTimes(1).Return(nil) + // Have to do this here or the reconcile call to GetOrCreateVMInstance may happen too early. + setupMachineCRDs() + + // Eventually the machine should set ready to true. + Eventually(func() bool { + tempMachine := &infrav1.CloudStackMachine{} + key := client.ObjectKey{Namespace: dummies.ClusterNameSpace, Name: dummies.CSMachine1.Name} + if err := k8sClient.Get(ctx, key, tempMachine); err == nil { + if tempMachine.Status.Ready == true { + return len(tempMachine.ObjectMeta.Finalizers) > 1 + } + } + return false + }, timeout).WithPolling(pollInterval).Should(BeTrue()) + + Ω(k8sClient.Delete(ctx, dummies.CSMachine1)).Should(Succeed()) + + Eventually(func() bool { + tempMachine := &infrav1.CloudStackMachine{} + key := client.ObjectKey{Namespace: dummies.ClusterNameSpace, Name: dummies.CSMachine1.Name} + if err := k8sClient.Get(ctx, key, tempMachine); err != nil { + return errors.IsNotFound(err) + } + return false + }, timeout).WithPolling(pollInterval).Should(BeTrue()) + }) + }) + + Context("Without a k8s test environment.", func() { + It("Should create a reconciliation runner with a Cloudstack Machine as the reconciliation subject.", func() { + reconRunner := controllers.NewCksMachineReconciliationRunner() + Ω(reconRunner.ReconciliationSubject).ShouldNot(BeNil()) + }) + }) +}) diff --git a/controllers/cloudstackfailuredomain_controller.go b/controllers/cloudstackfailuredomain_controller.go index 822d4ef7..3084cf6e 100644 --- a/controllers/cloudstackfailuredomain_controller.go +++ b/controllers/cloudstackfailuredomain_controller.go @@ -18,6 +18,8 @@ package controllers import ( "context" + "sort" + "github.com/pkg/errors" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" @@ -26,7 +28,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" - "sort" infrav1 "sigs.k8s.io/cluster-api-provider-cloudstack/api/v1beta3" csCtrlrUtils "sigs.k8s.io/cluster-api-provider-cloudstack/controllers/utils" diff --git a/controllers/controllers_suite_test.go b/controllers/controllers_suite_test.go index a9776f64..c1b2d1df 100644 --- a/controllers/controllers_suite_test.go +++ b/controllers/controllers_suite_test.go @@ -21,16 +21,17 @@ import ( "flag" "fmt" "go/build" - "k8s.io/client-go/tools/record" "os" "os/exec" "path/filepath" "regexp" - "sigs.k8s.io/cluster-api-provider-cloudstack/test/fakes" "strings" "testing" "time" + "k8s.io/client-go/tools/record" + "sigs.k8s.io/cluster-api-provider-cloudstack/test/fakes" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/klog/v2" "k8s.io/klog/v2/klogr" @@ -129,6 +130,10 @@ var ( FailureDomainReconciler *csReconcilers.CloudStackFailureDomainReconciler IsoNetReconciler *csReconcilers.CloudStackIsoNetReconciler AffinityGReconciler *csReconcilers.CloudStackAffinityGroupReconciler + + // CKS Reconcilers + CksClusterReconciler *csReconcilers.CksClusterReconciler + CksMachineReconciler *csReconcilers.CksMachineReconciler ) var _ = BeforeSuite(func() { @@ -224,6 +229,9 @@ func SetupTestEnvironment() { IsoNetReconciler = &csReconcilers.CloudStackIsoNetReconciler{ReconcilerBase: base} AffinityGReconciler = &csReconcilers.CloudStackAffinityGroupReconciler{ReconcilerBase: base} + CksClusterReconciler = &csReconcilers.CksClusterReconciler{ReconcilerBase: base} + CksMachineReconciler = &csReconcilers.CksMachineReconciler{ReconcilerBase: base} + ctx, cancel = context.WithCancel(context.TODO()) // Setup mock clients. @@ -237,6 +245,9 @@ func SetupTestEnvironment() { AffinityGReconciler.CSClient = mockCloudClient FailureDomainReconciler.CSClient = mockCloudClient + CksClusterReconciler.CSClient = mockCloudClient + CksMachineReconciler.CSClient = mockCloudClient + setupClusterCRDs() // See reconciliation results. Left commented as it's noisy otherwise. diff --git a/docs/book/src/development/common.md b/docs/book/src/development/common.md index 043e5e7f..cf1462ff 100644 --- a/docs/book/src/development/common.md +++ b/docs/book/src/development/common.md @@ -33,6 +33,11 @@ # The SSH KeyPair to log into the VM (Optional: you must use clusterctl --flavor *managed-ssh*) export CLOUDSTACK_SSH_KEY_NAME=CAPCKeyPair6 + + # Sync resources created by CAPC in Apache Cloudstack CKS. Default is false. + # Requires setting CAPC_CLOUDSTACKMACHINE_CKS_SYNC=true before initialising the cloudstack provider. + # Or set enable-cloudstack-cks-sync to true in the deployment for capc-controller. + export CLOUDSTACK_SYNC_WITH_ACS=true ``` 2. Generate the CAPC cluster spec yaml file diff --git a/docs/book/src/getting-started.md b/docs/book/src/getting-started.md index 5c10878e..ea4361f0 100644 --- a/docs/book/src/getting-started.md +++ b/docs/book/src/getting-started.md @@ -51,6 +51,11 @@ Run the following command to turn your cluster into a management cluster and loa clusterctl init --infrastructure cloudstack +> If you wish to enable syncing of CAPC resources with cloudstack, set the environment variable +> `CAPC_CLOUDSTACKMACHINE_CKS_SYNC=true` before initializing the cloudstack provider. Or set +> `enable-cloudstack-cks-sync` to true in the deployment for capc-controller. +> This is only supported for Apache CloudStack version 4.19 and above. + [capi-quick-start]: https://cluster-api.sigs.k8s.io/user/quick-start.html diff --git a/main.go b/main.go index d7e88c07..1929a017 100644 --- a/main.go +++ b/main.go @@ -86,6 +86,7 @@ type managerOpts struct { CloudStackMachineConcurrency int CloudStackAffinityGroupConcurrency int CloudStackFailureDomainConcurrency int + EnableCloudStackCksSync bool } func setFlags() *managerOpts { @@ -154,6 +155,12 @@ func setFlags() *managerOpts { 5, "Maximum concurrent reconciles for CloudStackFailureDomain resources", ) + flag.BoolVar( + &opts.EnableCloudStackCksSync, + "enable-cloudstack-cks-sync", + false, + "Enable syncing of CloudStack clusters and machines with CKS clusters and machines", + ) return opts } @@ -262,4 +269,14 @@ func setupReconcilers(ctx context.Context, base utils.ReconcilerBase, opts manag setupLog.Error(err, "unable to create controller", "controller", "CloudStackFailureDomain") os.Exit(1) } + if opts.EnableCloudStackCksSync { + if err := (&controllers.CksClusterReconciler{ReconcilerBase: base}).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "CKSClusterController") + os.Exit(1) + } + if err := (&controllers.CksMachineReconciler{ReconcilerBase: base}).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "CKSMachineController") + os.Exit(1) + } + } } diff --git a/pkg/cloud/cks_cluster.go b/pkg/cloud/cks_cluster.go new file mode 100644 index 00000000..279a0713 --- /dev/null +++ b/pkg/cloud/cks_cluster.go @@ -0,0 +1,143 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cloud + +import ( + "fmt" + "strings" + + "github.com/apache/cloudstack-go/v2/cloudstack" + "github.com/pkg/errors" + infrav1 "sigs.k8s.io/cluster-api-provider-cloudstack/api/v1beta3" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" +) + +type ClusterIface interface { + GetOrCreateCksCluster(*clusterv1.Cluster, *infrav1.CloudStackCluster, *infrav1.CloudStackFailureDomainSpec) error + DeleteCksCluster(*infrav1.CloudStackCluster) error + AddVMToCksCluster(*infrav1.CloudStackCluster, *infrav1.CloudStackMachine) error + RemoveVMFromCksCluster(*infrav1.CloudStackCluster, *infrav1.CloudStackMachine) error +} + +type ClustertypeSetter interface { + SetClustertype(string) +} + +func withExternalManaged() cloudstack.OptionFunc { + return func(cs *cloudstack.CloudStackClient, p interface{}) error { + ps, ok := p.(ClustertypeSetter) + if !ok { + return errors.New("invalid params type") + } + ps.SetClustertype("ExternalManaged") + return nil + } +} + +func (c *client) GetOrCreateCksCluster(cluster *clusterv1.Cluster, csCluster *infrav1.CloudStackCluster, fd *infrav1.CloudStackFailureDomainSpec) error { + // Get cluster + if csCluster.Status.CloudStackClusterID != "" { + externalManagedCluster, count, err := c.cs.Kubernetes.GetKubernetesClusterByID(csCluster.Status.CloudStackClusterID, withExternalManaged(), cloudstack.WithProject(c.user.Project.ID)) + if err != nil { + return err + } else if count > 0 { + csCluster.Status.CloudStackClusterID = externalManagedCluster.Id + return nil + } + } + + // Check if a cluster exists with the same name + clusterName := fmt.Sprintf("%s - %s - %s", cluster.GetName(), csCluster.GetName(), csCluster.GetUID()) + externalManagedCluster, count, err := c.cs.Kubernetes.GetKubernetesClusterByName(clusterName, withExternalManaged(), cloudstack.WithProject(c.user.Project.ID)) + if err != nil && !strings.Contains(err.Error(), "No match found for ") { + return err + } + if count > 0 { + csCluster.Status.CloudStackClusterID = externalManagedCluster.Id + } else if err == nil || (err != nil && strings.Contains(err.Error(), "No match found for ")) { + // Create cluster + accountName := csCluster.Spec.FailureDomains[0].Account + if accountName == "" { + userParams := c.cs.User.NewGetUserParams(c.config.APIKey) + user, err := c.cs.User.GetUser(userParams) + if err != nil { + return err + } + accountName = user.Account + } + // NewCreateKubernetesClusterParams(description string, kubernetesversionid string, name string, serviceofferingid string, size int64, zoneid string) *CreateKubernetesClusterParams + params := c.cs.Kubernetes.NewCreateKubernetesClusterParams(fmt.Sprintf("%s managed by CAPC", clusterName), "", clusterName, "", 0, fd.Zone.ID) + + setIfNotEmpty(c.user.Project.ID, params.SetProjectid) + setIfNotEmpty(accountName, params.SetAccount) + setIfNotEmpty(c.user.Domain.ID, params.SetDomainid) + setIfNotEmpty(fd.Zone.Network.ID, params.SetNetworkid) + setIfNotEmpty(csCluster.Spec.ControlPlaneEndpoint.Host, params.SetExternalloadbalanceripaddress) + params.ResetKubernetesversionid() + params.ResetServiceofferingid() + params.SetClustertype("ExternalManaged") + + cloudStackCKSCluster, err := c.cs.Kubernetes.CreateKubernetesCluster(params) + if err != nil { + return err + } + csCluster.Status.CloudStackClusterID = cloudStackCKSCluster.Id + } + return nil +} + +func (c *client) DeleteCksCluster(csCluster *infrav1.CloudStackCluster) error { + if csCluster.Status.CloudStackClusterID != "" { + csCksCluster, count, err := c.cs.Kubernetes.GetKubernetesClusterByID( + csCluster.Status.CloudStackClusterID, withExternalManaged(), cloudstack.WithProject(c.user.Project.ID)) + if err != nil && strings.Contains(err.Error(), " not found") { + return nil + } + if count != 0 { + params := c.cs.Kubernetes.NewDeleteKubernetesClusterParams(csCksCluster.Id) + _, err = c.cs.Kubernetes.DeleteKubernetesCluster(params) + if err != nil { + return err + } + } + csCluster.Status.CloudStackClusterID = "" + } + return nil +} + +func (c *client) AddVMToCksCluster(csCluster *infrav1.CloudStackCluster, csMachine *infrav1.CloudStackMachine) error { + if csCluster.Status.CloudStackClusterID != "" { + params := c.cs.Kubernetes.NewAddVirtualMachinesToKubernetesClusterParams(csCluster.Status.CloudStackClusterID, []string{*csMachine.Spec.InstanceID}) + if csMachine.Labels != nil { + _, ok := csMachine.Labels[clusterv1.MachineControlPlaneLabel] + params.SetIscontrolnode(ok) + } + + _, err := c.cs.Kubernetes.AddVirtualMachinesToKubernetesCluster(params) + return err + } + return nil +} + +func (c *client) RemoveVMFromCksCluster(csCluster *infrav1.CloudStackCluster, csMachine *infrav1.CloudStackMachine) error { + if csCluster.Status.CloudStackClusterID != "" { + params := c.cs.Kubernetes.NewRemoveVirtualMachinesFromKubernetesClusterParams(csCluster.Status.CloudStackClusterID, []string{*csMachine.Spec.InstanceID}) + _, err := c.cs.Kubernetes.RemoveVirtualMachinesFromKubernetesCluster(params) + return err + } + return nil +} diff --git a/pkg/cloud/client.go b/pkg/cloud/client.go index 12e61176..76c50256 100644 --- a/pkg/cloud/client.go +++ b/pkg/cloud/client.go @@ -37,6 +37,7 @@ import ( //go:generate ../../hack/tools/bin/mockgen -destination=../mocks/mock_client.go -package=mocks sigs.k8s.io/cluster-api-provider-cloudstack/pkg/cloud Client type Client interface { + ClusterIface VMIface NetworkIface AffinityGroupIface diff --git a/templates/cluster-template-managed-ssh.yaml b/templates/cluster-template-managed-ssh.yaml index 19917d75..a036adb8 100644 --- a/templates/cluster-template-managed-ssh.yaml +++ b/templates/cluster-template-managed-ssh.yaml @@ -23,6 +23,7 @@ kind: CloudStackCluster metadata: name: ${CLUSTER_NAME} spec: + syncWithACS: ${CLOUDSTACK_SYNC_WITH_ACS=false} controlPlaneEndpoint: host: ${CLUSTER_ENDPOINT_IP} port: ${CLUSTER_ENDPOINT_PORT=6443} diff --git a/templates/cluster-template-ssh-material.yaml b/templates/cluster-template-ssh-material.yaml index 1475488f..3c85db2b 100644 --- a/templates/cluster-template-ssh-material.yaml +++ b/templates/cluster-template-ssh-material.yaml @@ -23,6 +23,7 @@ kind: CloudStackCluster metadata: name: ${CLUSTER_NAME} spec: + syncWithACS: ${CLOUDSTACK_SYNC_WITH_ACS=false} controlPlaneEndpoint: host: ${CLUSTER_ENDPOINT_IP} port: ${CLUSTER_ENDPOINT_PORT=6443} diff --git a/templates/cluster-template-with-disk-offering.yaml b/templates/cluster-template-with-disk-offering.yaml index bc32a6a5..486c96ef 100644 --- a/templates/cluster-template-with-disk-offering.yaml +++ b/templates/cluster-template-with-disk-offering.yaml @@ -23,6 +23,7 @@ kind: CloudStackCluster metadata: name: ${CLUSTER_NAME} spec: + syncWithACS: ${CLOUDSTACK_SYNC_WITH_ACS=false} controlPlaneEndpoint: host: ${CLUSTER_ENDPOINT_IP} port: ${CLUSTER_ENDPOINT_PORT=6443} diff --git a/templates/cluster-template-with-kube-vip.yaml b/templates/cluster-template-with-kube-vip.yaml index e33fa0b8..47e23652 100644 --- a/templates/cluster-template-with-kube-vip.yaml +++ b/templates/cluster-template-with-kube-vip.yaml @@ -23,6 +23,7 @@ kind: CloudStackCluster metadata: name: ${CLUSTER_NAME} spec: + syncWithACS: ${CLOUDSTACK_SYNC_WITH_ACS=false} controlPlaneEndpoint: host: ${CLUSTER_ENDPOINT_IP} port: ${CLUSTER_ENDPOINT_PORT=6443} diff --git a/templates/cluster-template.yaml b/templates/cluster-template.yaml index c3e703b7..c9555ee5 100644 --- a/templates/cluster-template.yaml +++ b/templates/cluster-template.yaml @@ -23,6 +23,7 @@ kind: CloudStackCluster metadata: name: ${CLUSTER_NAME} spec: + syncWithACS: ${CLOUDSTACK_SYNC_WITH_ACS=false} controlPlaneEndpoint: host: ${CLUSTER_ENDPOINT_IP} port: ${CLUSTER_ENDPOINT_PORT=6443} diff --git a/test/dummies/v1beta3/vars.go b/test/dummies/v1beta3/vars.go index e633abf5..087603bd 100644 --- a/test/dummies/v1beta3/vars.go +++ b/test/dummies/v1beta3/vars.go @@ -80,6 +80,7 @@ var ( // Declare exported dummy vars. LBRuleID string PublicIPID string EndPointHost string + SyncWithACS *bool EndPointPort int32 CSConf *simpleyaml.Yaml DiskOffering infrav1.CloudStackResourceDiskOffering @@ -273,6 +274,7 @@ func SetDummyCAPCClusterVars() { CSClusterKind = "CloudStackCluster" ClusterName = "test-cluster" EndPointHost = "EndpointHost" + SyncWithACS := pointer.Bool(true) EndPointPort = int32(5309) PublicIPID = "FakePublicIPID" ClusterNameSpace = "default" @@ -333,6 +335,7 @@ func SetDummyCAPCClusterVars() { Spec: infrav1.CloudStackClusterSpec{ ControlPlaneEndpoint: clusterv1.APIEndpoint{Host: EndPointHost, Port: EndPointPort}, FailureDomains: []infrav1.CloudStackFailureDomainSpec{CSFailureDomain1.Spec, CSFailureDomain2.Spec}, + SyncWithACS: SyncWithACS, }, Status: infrav1.CloudStackClusterStatus{}, } diff --git a/test/e2e/common.go b/test/e2e/common.go index 53518904..ea537200 100644 --- a/test/e2e/common.go +++ b/test/e2e/common.go @@ -251,6 +251,17 @@ func DownloadMetricsFromCAPCManager(ctx context.Context, bootstrapKubeconfigPath return result, nil } +func GetACSVersion(client *cloudstack.CloudStackClient) (string, error) { + msServersResp, err := client.InfrastructureUsage.ListManagementServersMetrics(client.InfrastructureUsage.NewListManagementServersMetricsParams()) + if err != nil { + return "", err + } + if msServersResp.Count == 0 { + return "", errors.New("no management servers found") + } + return msServersResp.ManagementServersMetrics[0].Version, nil +} + func DestroyOneMachine(client *cloudstack.CloudStackClient, clusterName string, machineType string) { matcher := clusterName + "-" + machineType diff --git a/test/e2e/config/cloudstack.yaml b/test/e2e/config/cloudstack.yaml index 076be161..8c428453 100644 --- a/test/e2e/config/cloudstack.yaml +++ b/test/e2e/config/cloudstack.yaml @@ -98,6 +98,7 @@ providers: - sourcePath: "../data/infrastructure-cloudstack/v1beta3/cluster-template-invalid-ip.yaml" - sourcePath: "../data/infrastructure-cloudstack/v1beta3/cluster-template-kubernetes-version-upgrade-before.yaml" - sourcePath: "../data/infrastructure-cloudstack/v1beta3/cluster-template-kubernetes-version-upgrade-after.yaml" + - sourcePath: "../data/infrastructure-cloudstack/v1beta3/cluster-template-k8s-cks.yaml" - sourcePath: "../data/shared/v1beta1_provider/metadata.yaml" versions: - name: v1.0.0 @@ -159,6 +160,7 @@ variables: CONFORMANCE_CONFIGURATION: "./data/kubetest/conformance.yaml" CONFORMANCE_WORKER_MACHINE_COUNT: "3" CONFORMANCE_CONTROL_PLANE_MACHINE_COUNT: "1" + CAPC_CLOUDSTACKMACHINE_CKS_SYNC: "true" intervals: conformance/wait-control-plane: ["20m", "10s"] diff --git a/test/e2e/data/infrastructure-cloudstack/v1beta3/bases/cluster-with-kcp.yaml b/test/e2e/data/infrastructure-cloudstack/v1beta3/bases/cluster-with-kcp.yaml index bdd9d70b..2b719aa9 100644 --- a/test/e2e/data/infrastructure-cloudstack/v1beta3/bases/cluster-with-kcp.yaml +++ b/test/e2e/data/infrastructure-cloudstack/v1beta3/bases/cluster-with-kcp.yaml @@ -35,6 +35,7 @@ spec: name : ${CLOUDSTACK_ZONE_NAME} network: name: ${CLOUDSTACK_NETWORK_NAME} + syncWithACS: false --- kind: KubeadmControlPlane apiVersion: controlplane.cluster.x-k8s.io/v1beta1 diff --git a/test/e2e/data/infrastructure-cloudstack/v1beta3/cluster-template-k8s-cks/cloudstack-cluster.yaml b/test/e2e/data/infrastructure-cloudstack/v1beta3/cluster-template-k8s-cks/cloudstack-cluster.yaml new file mode 100644 index 00000000..7bb1a966 --- /dev/null +++ b/test/e2e/data/infrastructure-cloudstack/v1beta3/cluster-template-k8s-cks/cloudstack-cluster.yaml @@ -0,0 +1,19 @@ +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta3 +kind: CloudStackCluster +metadata: + name: ${CLUSTER_NAME} +spec: + controlPlaneEndpoint: + host: "" + port: 6443 + failureDomains: + - name: ${CLOUDSTACK_FD1_NAME} + acsEndpoint: + name: ${CLOUDSTACK_FD1_SECRET_NAME} + namespace: default + zone: + name : ${CLOUDSTACK_ZONE_NAME} + network: + name: ${CLOUDSTACK_NETWORK_NAME} + syncWithACS: true diff --git a/test/e2e/data/infrastructure-cloudstack/v1beta3/cluster-template-k8s-cks/kustomization.yaml b/test/e2e/data/infrastructure-cloudstack/v1beta3/cluster-template-k8s-cks/kustomization.yaml new file mode 100644 index 00000000..7a6d2984 --- /dev/null +++ b/test/e2e/data/infrastructure-cloudstack/v1beta3/cluster-template-k8s-cks/kustomization.yaml @@ -0,0 +1,6 @@ +bases: + - ../bases/cluster-with-kcp.yaml + - ../bases/md.yaml + +patchesStrategicMerge: +- ./cloudstack-cluster.yaml diff --git a/test/e2e/go.mod b/test/e2e/go.mod index 003539a6..72edffc8 100644 --- a/test/e2e/go.mod +++ b/test/e2e/go.mod @@ -4,7 +4,7 @@ go 1.21 require ( github.com/Shopify/toxiproxy/v2 v2.5.0 - github.com/apache/cloudstack-go/v2 v2.15.0 + github.com/apache/cloudstack-go/v2 v2.16.1 github.com/blang/semver v3.5.1+incompatible github.com/onsi/ginkgo/v2 v2.11.0 github.com/onsi/gomega v1.27.8 @@ -97,12 +97,12 @@ require ( github.com/stoewer/go-strcase v1.2.0 // indirect github.com/subosito/gotenv v1.4.2 // indirect github.com/valyala/fastjson v1.6.4 // indirect - golang.org/x/crypto v0.14.0 // indirect - golang.org/x/net v0.17.0 // indirect + golang.org/x/crypto v0.21.0 // indirect + golang.org/x/net v0.23.0 // indirect golang.org/x/oauth2 v0.6.0 // indirect - golang.org/x/sys v0.13.0 // indirect - golang.org/x/term v0.13.0 // indirect - golang.org/x/text v0.13.0 // indirect + golang.org/x/sys v0.18.0 // indirect + golang.org/x/term v0.18.0 // indirect + golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.3.0 // indirect golang.org/x/tools v0.9.3 // indirect gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect diff --git a/test/e2e/go.sum b/test/e2e/go.sum index e84ef48a..5fe3f956 100644 --- a/test/e2e/go.sum +++ b/test/e2e/go.sum @@ -66,8 +66,8 @@ github.com/alessio/shellescape v1.4.1/go.mod h1:PZAiSCk0LJaZkiCSkPv8qIobYglO3FPp github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 h1:yL7+Jz0jTC6yykIK/Wh74gnTJnrGr5AyrNMXuA0gves= github.com/antlr/antlr4/runtime/Go/antlr v1.4.10/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY= -github.com/apache/cloudstack-go/v2 v2.15.0 h1:oojn1qx0+wBwrFSSmA2rL8XjWd4BXqwYo0RVCrAXoHk= -github.com/apache/cloudstack-go/v2 v2.15.0/go.mod h1:Mc+tXpujtslBuZFk5atoGT2LanVxOrXS2GGgidAoz1A= +github.com/apache/cloudstack-go/v2 v2.16.1 h1:2wOE4RKEjWPRZNO7ZNnZYmR3JJ+JJPQwhoc7W1fkiK4= +github.com/apache/cloudstack-go/v2 v2.16.1/go.mod h1:cZsgFe+VmrgLBm7QjeHTJBXYe8E5+yGYkdfwGb+Pu9c= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= @@ -540,8 +540,8 @@ golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= -golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= -golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= +golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -622,8 +622,8 @@ golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= -golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= +golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -706,13 +706,13 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= -golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= -golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= -golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= +golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= +golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -723,8 +723,8 @@ golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= -golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/test/e2e/kubernetes_cks_cluster.go b/test/e2e/kubernetes_cks_cluster.go new file mode 100644 index 00000000..87d43dd0 --- /dev/null +++ b/test/e2e/kubernetes_cks_cluster.go @@ -0,0 +1,136 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/apache/cloudstack-go/v2/cloudstack" + "github.com/blang/semver" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + "k8s.io/utils/pointer" + + "sigs.k8s.io/cluster-api/test/framework/clusterctl" + "sigs.k8s.io/cluster-api/util" +) + +// CksK8SSpec implements a spec that creates a cluster and checks whether an entry is created in ACS. +func CksK8SSpec(ctx context.Context, inputGetter func() CommonSpecInput) { + var ( + specName = "k8s-cks" + input CommonSpecInput + namespace *corev1.Namespace + cancelWatches context.CancelFunc + clusterResources *clusterctl.ApplyClusterTemplateAndWaitResult + ) + + BeforeEach(func() { + Expect(ctx).NotTo(BeNil(), "ctx is required for %s spec", specName) + input = inputGetter() + + csClient := CreateCloudStackClient(ctx, input.BootstrapClusterProxy.GetKubeconfigPath()) + version, err := GetACSVersion(csClient) + + if err != nil || version == "" { + Skip("Failed to get CloudStack's version") + } + + v, err := semver.ParseTolerant(strings.Join(strings.Split(version, ".")[0:3], ".")) + + if err != nil { + Skip("Failed to parse CloudStack version " + version) + } + + expectedRange, _ := semver.ParseRange(">=4.19.0") + + if !expectedRange(v) { + Skip("Cloudstack version " + version + " is less than 4.19.") + } + + Expect(input.E2EConfig).ToNot(BeNil(), "Invalid argument. input.E2EConfig can't be nil when calling %s spec", specName) + Expect(input.ClusterctlConfigPath).To(BeAnExistingFile(), "Invalid argument. input.ClusterctlConfigPath must be an existing file when calling %s spec", specName) + Expect(input.BootstrapClusterProxy).ToNot(BeNil(), "Invalid argument. input.BootstrapClusterProxy can't be nil when calling %s spec", specName) + Expect(os.MkdirAll(input.ArtifactFolder, 0750)).To(Succeed(), "Invalid argument. input.ArtifactFolder can't be created for %s spec", specName) + + Expect(input.E2EConfig.Variables).To(HaveKey(KubernetesVersion)) + + // Setup a Namespace where to host objects for this spec and create a watcher for the namespace events. + namespace, cancelWatches = setupSpecNamespace(ctx, specName, input.BootstrapClusterProxy, input.ArtifactFolder) + clusterResources = new(clusterctl.ApplyClusterTemplateAndWaitResult) + }) + + It("Should create a workload cluster", func() { + By("Creating a workload cluster") + + clusterName := fmt.Sprintf("%s-%s", specName, util.RandomString(6)) + + clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{ + ClusterProxy: input.BootstrapClusterProxy, + CNIManifestPath: input.E2EConfig.GetVariable(CNIPath), + ConfigCluster: clusterctl.ConfigClusterInput{ + LogFolder: filepath.Join(input.ArtifactFolder, "clusters", input.BootstrapClusterProxy.GetName()), + ClusterctlConfigPath: input.ClusterctlConfigPath, + KubeconfigPath: input.BootstrapClusterProxy.GetKubeconfigPath(), + InfrastructureProvider: clusterctl.DefaultInfrastructureProvider, + Flavor: specName, + Namespace: namespace.Name, + ClusterName: clusterName, + KubernetesVersion: input.E2EConfig.GetVariable(KubernetesVersion), + ControlPlaneMachineCount: pointer.Int64Ptr(1), + WorkerMachineCount: pointer.Int64Ptr(1), + }, + WaitForClusterIntervals: input.E2EConfig.GetIntervals(specName, "wait-cluster"), + WaitForControlPlaneIntervals: input.E2EConfig.GetIntervals(specName, "wait-control-plane"), + WaitForMachineDeployments: input.E2EConfig.GetIntervals(specName, "wait-worker-nodes"), + }, clusterResources) + + By("checking cks resource is created on ACS") + // Get details from ACS + csClient := CreateCloudStackClient(ctx, input.BootstrapClusterProxy.GetKubeconfigPath()) + lkcp := csClient.Kubernetes.NewListKubernetesClustersParams() + lkcp.SetListall(true) + + clusters, err := csClient.Kubernetes.ListKubernetesClusters(lkcp) + + if err != nil { + Fail("Failed to get Kubernetes clusters from ACS") + } + + var cluster *cloudstack.KubernetesCluster + + for _, d := range clusters.KubernetesClusters { + if strings.HasPrefix(d.Name, fmt.Sprintf("%s - %s", clusterName, clusterName)) { + cluster = d + } + } + + Expect(cluster).ShouldNot(BeNil(), "Couldn't find the external managed kubernetes in ACS") + Expect(len(cluster.Virtualmachines)).Should(Equal(2), "Expected 2 VMs in the cluster, found %d", len(cluster.Virtualmachines)) + By("PASSED!") + }) + + AfterEach(func() { + // Dumps all the resources in the spec namespace, then cleanups the cluster object and the spec namespace itself. + dumpSpecResourcesAndCleanup(ctx, specName, input.BootstrapClusterProxy, input.ArtifactFolder, namespace, cancelWatches, clusterResources.Cluster, input.E2EConfig.GetIntervals, input.SkipCleanup) + }) +} diff --git a/test/e2e/kubernetes_cks_cluster_test.go b/test/e2e/kubernetes_cks_cluster_test.go new file mode 100644 index 00000000..4be4d8c7 --- /dev/null +++ b/test/e2e/kubernetes_cks_cluster_test.go @@ -0,0 +1,36 @@ +//go:build e2e +// +build e2e + +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + . "github.com/onsi/ginkgo/v2" +) + +var _ = Describe("When testing creation of CKS cluster in ACS", func() { + CksK8SSpec(ctx, func() CommonSpecInput { + return CommonSpecInput{ + E2EConfig: e2eConfig, + ClusterctlConfigPath: clusterctlConfigPath, + BootstrapClusterProxy: bootstrapClusterProxy, + ArtifactFolder: artifactFolder, + SkipCleanup: skipCleanup, + } + }) +})