Skip to content

Commit

Permalink
Tests: apply embedded capi ff after rancher install (rancher#190)
Browse files Browse the repository at this point in the history
Signed-off-by: Danil Grigorev <[email protected]>
  • Loading branch information
Danil-Grigorev committed Oct 9, 2023
1 parent 8b8ecf3 commit 9378211
Show file tree
Hide file tree
Showing 11 changed files with 330 additions and 71 deletions.
3 changes: 1 addition & 2 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -190,7 +190,6 @@ manifests: vendor controller-gen ## Generate WebhookConfiguration, ClusterRole a
vendor:
go mod tidy
go mod vendor
go mod verify

.PHONY: vendor-clean
vendor-clean:
Expand Down Expand Up @@ -242,7 +241,7 @@ ARTIFACTS ?= ${ROOT_DIR}/_artifacts
KUBEBUILDER_ASSETS ?= $(shell $(SETUP_ENVTEST) use --use-env -p path $(KUBEBUILDER_ENVTEST_KUBERNETES_VERSION))

.PHONY: test
test: $(SETUP_ENVTEST) manifests ## Run tests.
test: $(SETUP_ENVTEST) manifests kubectl ## Run tests.
KUBEBUILDER_ASSETS="$(KUBEBUILDER_ASSETS)" go test ./... $(TEST_ARGS)

##@ Build
Expand Down
163 changes: 163 additions & 0 deletions internal/controllers/cross_cluster_import_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,163 @@
/*
Copyright 2023 SUSE.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package controllers

import (
"context"
"fmt"
"os"
"os/exec"
"path/filepath"

. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
provisioningv1 "github.com/rancher-sandbox/rancher-turtles/internal/rancher/provisioning/v1"
"github.com/rancher-sandbox/rancher-turtles/internal/rancher/setup"
"github.com/rancher-sandbox/rancher-turtles/internal/test"
turtlesnaming "github.com/rancher-sandbox/rancher-turtles/util/naming"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/rest"
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
"sigs.k8s.io/cluster-api/util/kubeconfig"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/controller"
"sigs.k8s.io/controller-runtime/pkg/envtest"

ctrl "sigs.k8s.io/controller-runtime"
)

var (
rancherEnv *envtest.Environment
otherEnv *envtest.Environment
rancherCfg *rest.Config
otherCfg *rest.Config
rancherCl client.Client
otherCl client.Client
clusterCtx context.Context
cancel context.CancelFunc
capiCluster *clusterv1.Cluster
rancherCluster *provisioningv1.Cluster
r *CAPIImportReconciler
)

var _ = Describe("In separate clusters", func() {

BeforeEach(func() {
By("bootstrapping rancher environment")
var err error
rancherEnv = &envtest.Environment{
CRDDirectoryPaths: []string{
filepath.Join("..", "..", "hack", "crd", "bases"),
},
ErrorIfCRDPathMissing: true,
Scheme: test.FullScheme,
}
rancherCfg, rancherCl, err = test.StartEnvTest(rancherEnv)
Expect(err).NotTo(HaveOccurred())
Expect(rancherCfg).NotTo(BeNil())
Expect(rancherCl).NotTo(BeNil())

By("Bootstrapping other cluster environment")
otherEnv = &envtest.Environment{
CRDDirectoryPaths: []string{
filepath.Join("..", "..", "hack", "crd", "bases"),
},
ErrorIfCRDPathMissing: true,
Scheme: test.FullScheme,
}
otherCfg, otherCl, err = test.StartEnvTest(otherEnv)
Expect(err).NotTo(HaveOccurred())
Expect(otherCfg).NotTo(BeNil())
Expect(otherCl).NotTo(BeNil())

Expect(otherCl.Create(ctx, &corev1.Namespace{
ObjectMeta: metav1.ObjectMeta{
Name: testNamespace,
Labels: map[string]string{
importLabelName: "true",
},
},
})).To(Succeed())

Expect(rancherCl.Create(ctx, &corev1.Namespace{
ObjectMeta: metav1.ObjectMeta{
Name: testNamespace,
},
})).To(Succeed())

clusterCtx, cancel = context.WithCancel(ctx)
capiCluster = &clusterv1.Cluster{
ObjectMeta: metav1.ObjectMeta{
Name: "test-cluster",
Namespace: testNamespace,
},
}

rancherCluster = &provisioningv1.Cluster{
ObjectMeta: metav1.ObjectMeta{
Name: turtlesnaming.Name(capiCluster.Name).ToRancherName(),
Namespace: testNamespace,
},
}
})

AfterEach(func() {
cancel()
By("tearing down the 2 clsuter environment")
Expect(test.StopEnvTest(rancherEnv)).To(Succeed())
Expect(test.StopEnvTest(otherEnv)).To(Succeed())
})

It("minimal controller setup should create a Rancher cluster object from a CAPI cluster object located in a different cluster", func() {
mgr, err := ctrl.NewManager(otherCfg, ctrl.Options{
Scheme: otherEnv.Scheme,
MetricsBindAddress: "0",
HealthProbeBindAddress: "0",
})
Expect(err).ToNot(HaveOccurred())

config := kubeconfig.FromEnvTestConfig(rancherCfg, &clusterv1.Cluster{
ObjectMeta: metav1.ObjectMeta{Name: "test"},
})
kubeconfigFile, err := os.CreateTemp("", "kubeconfig")
Expect(err).ToNot(HaveOccurred())
defer os.Remove(kubeconfigFile.Name())
Expect(os.WriteFile(kubeconfigFile.Name(), config, 0600)).To(Succeed())

rancher, err := setup.RancherCluster(mgr, kubeconfigFile.Name())
Expect(err).ToNot(HaveOccurred())

reconciler := &CAPIImportReconciler{
Client: mgr.GetClient(),
RancherCluster: rancher,
}
Expect(reconciler.SetupWithManager(ctx, mgr, controller.Options{})).To(Succeed())

go func() {
Expect(mgr.Start(clusterCtx)).To(Succeed())
}()

Expect(otherCl.Create(ctx, capiCluster)).To(Succeed())
capiCluster.Status.ControlPlaneReady = true
Expect(otherCl.Status().Update(ctx, capiCluster)).To(Succeed())

args := []string{"get", fmt.Sprintf("clusters.%s", provisioningv1.GroupVersion.Group), rancherCluster.Name, "-n", rancherCluster.Namespace, "--kubeconfig", kubeconfigFile.Name()}
Eventually(exec.Command("kubectl", args...).Run()).Should(Succeed())
})

})
13 changes: 7 additions & 6 deletions internal/controllers/import_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ import (
"k8s.io/client-go/tools/record"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/cluster"
"sigs.k8s.io/controller-runtime/pkg/controller"
"sigs.k8s.io/controller-runtime/pkg/event"
"sigs.k8s.io/controller-runtime/pkg/handler"
Expand Down Expand Up @@ -68,7 +69,7 @@ const (
// CAPIImportReconciler represents a reconciler for importing CAPI clusters in Rancher.
type CAPIImportReconciler struct {
Client client.Client
RancherClient client.Client
RancherCluster cluster.Cluster
recorder record.EventRecorder
WatchFilterValue string
Scheme *runtime.Scheme
Expand Down Expand Up @@ -106,7 +107,7 @@ func (r *CAPIImportReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Ma
// Watch Rancher provisioningv2 clusters
// NOTE: we will import the types from rancher in the future
err = c.Watch(
source.Kind(mgr.GetCache(), &provisioningv1.Cluster{}),
source.Kind(r.RancherCluster.GetCache(), &provisioningv1.Cluster{}),
handler.EnqueueRequestsFromMapFunc(r.rancherClusterToCapiCluster(ctx, capiPredicates)),
//&handler.EnqueueRequestForOwner{OwnerType: &clusterv1.Cluster{}},
)
Expand Down Expand Up @@ -200,7 +201,7 @@ func (r *CAPIImportReconciler) reconcile(ctx context.Context, capiCluster *clust
Name: turtlesnaming.Name(capiCluster.Name).ToRancherName(),
}}

err := r.RancherClient.Get(ctx, client.ObjectKeyFromObject(rancherCluster), rancherCluster)
err := r.RancherCluster.GetClient().Get(ctx, client.ObjectKeyFromObject(rancherCluster), rancherCluster)
if client.IgnoreNotFound(err) != nil {
log.Error(err, fmt.Sprintf("Unable to fetch rancher cluster %s", client.ObjectKeyFromObject(rancherCluster)))
return ctrl.Result{Requeue: true}, err
Expand All @@ -218,7 +219,7 @@ func (r *CAPIImportReconciler) reconcileNormal(ctx context.Context, capiCluster
) (ctrl.Result, error) {
log := log.FromContext(ctx)

err := r.RancherClient.Get(ctx, client.ObjectKeyFromObject(rancherCluster), rancherCluster)
err := r.RancherCluster.GetClient().Get(ctx, client.ObjectKeyFromObject(rancherCluster), rancherCluster)
if apierrors.IsNotFound(err) {
shouldImport, err := util.ShouldAutoImport(ctx, log, r.Client, capiCluster, importLabelName)
if err != nil {
Expand All @@ -230,7 +231,7 @@ func (r *CAPIImportReconciler) reconcileNormal(ctx context.Context, capiCluster
return ctrl.Result{}, nil
}

if err := r.RancherClient.Create(ctx, &provisioningv1.Cluster{
if err := r.RancherCluster.GetClient().Create(ctx, &provisioningv1.Cluster{
ObjectMeta: metav1.ObjectMeta{
Name: turtlesnaming.Name(capiCluster.Name).ToRancherName(),
Namespace: capiCluster.Namespace,
Expand Down Expand Up @@ -323,7 +324,7 @@ func (r *CAPIImportReconciler) getClusterRegistrationManifest(ctx context.Contex
Name: clusterRegistrationTokenName,
Namespace: clusterName,
}}
err := r.RancherClient.Get(ctx, client.ObjectKeyFromObject(token), token)
err := r.RancherCluster.GetClient().Get(ctx, client.ObjectKeyFromObject(token), token)

if client.IgnoreNotFound(err) != nil {
return "", fmt.Errorf("error getting registration token for cluster %s: %w", clusterName, err)
Expand Down
17 changes: 16 additions & 1 deletion internal/controllers/import_controller_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ package controllers

import (
"bufio"
"context"
"encoding/json"
"errors"
"fmt"
Expand All @@ -43,6 +44,7 @@ import (
"sigs.k8s.io/cluster-api/controllers/remote"
"sigs.k8s.io/cluster-api/util/secret"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/cluster"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
)

Expand All @@ -53,12 +55,24 @@ var _ = Describe("reconcile CAPI Cluster", func() {
rancherCluster *provisioningv1.Cluster
clusterRegistrationToken *managementv3.ClusterRegistrationToken
capiKubeconfigSecret *corev1.Secret
cancel context.CancelFunc
clusterCtx context.Context
)

BeforeEach(func() {
// rancher and rancher-turtles deployed in the same cluster
clusterCtx, cancel = context.WithCancel(ctx)
cluster, err := cluster.New(testEnv.Config, func(clusterOptions *cluster.Options) {
clusterOptions.Scheme = testEnv.Scheme
})
Expect(err).ToNot(HaveOccurred())
go func() {
Expect(cluster.Start(clusterCtx)).To(Succeed())
}()

r = &CAPIImportReconciler{
Client: cl,
RancherClient: cl, // rancher and rancher-turtles deployed in the same cluster
RancherCluster: cluster,
remoteClientGetter: remote.NewClusterClient,
}

Expand Down Expand Up @@ -95,6 +109,7 @@ var _ = Describe("reconcile CAPI Cluster", func() {
})

AfterEach(func() {
defer cancel()
objs, err := manifestToObjects(strings.NewReader(testdata.ImportManifest))
clientObjs := []client.Object{
capiCluster,
Expand Down
1 change: 1 addition & 0 deletions internal/controllers/suite_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ var _ = BeforeSuite(func() {
filepath.Join("..", "..", "hack", "crd", "bases"),
},
ErrorIfCRDPathMissing: true,
Scheme: test.FullScheme,
}
cfg, cl, err = test.StartEnvTest(testEnv)
Expect(err).NotTo(HaveOccurred())
Expand Down
5 changes: 3 additions & 2 deletions internal/rancher/doc.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,8 @@ See the License for the specific language governing permissions and
limitations under the License.
*/

// Package rancher contains the rancher provisioning.cattle.io/v1 and
// management.cattle.io/v3 API proxy implementations.
// Package rancher contains rancher provisioning.cattle.io/v1 and
// management.cattle.io/v3 API proxy implementations and cluster connectivity
// setup procedures.
// +kubebuilder:object:generate=true
package rancher
Loading

0 comments on commit 9378211

Please sign in to comment.