Skip to content

Commit

Permalink
Tests: apply embedded capi ff after rancher install (rancher#190)
Browse files Browse the repository at this point in the history
Signed-off-by: Danil Grigorev <[email protected]>
  • Loading branch information
Danil-Grigorev committed Oct 9, 2023
1 parent f9cf357 commit f8f3a81
Show file tree
Hide file tree
Showing 17 changed files with 460 additions and 152 deletions.
10 changes: 4 additions & 6 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -180,7 +180,7 @@ help: ## Display this help.
##@ Development
.PHONY: manifests
manifests: vendor controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects.
$(CONTROLLER_GEN) rbac:roleName=manager-role crd paths="./..." output:crd:artifacts:config=hack/crd/bases
$(CONTROLLER_GEN) rbac:roleName=manager-role crd paths="./internal/rancher/..." output:crd:artifacts:config=hack/crd/bases
$(CONTROLLER_GEN) rbac:roleName=manager-role crd paths="./vendor/sigs.k8s.io/cluster-api/..." output:crd:artifacts:config=hack/crd/bases
# Vendor is only required for pulling latest CRDs from the dependencies
$(MAKE) vendor-clean
Expand All @@ -190,15 +190,13 @@ manifests: vendor controller-gen ## Generate WebhookConfiguration, ClusterRole a
vendor:
go mod tidy
go mod vendor
go mod verify

.PHONY: vendor-clean
vendor-clean:
rm -rf vendor

.PHONY: generate
generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations.
$(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..."

.PHONY: generate-modules
generate-modules: ## Run go mod tidy to ensure modules are up to date
Expand Down Expand Up @@ -243,17 +241,17 @@ ARTIFACTS ?= ${ROOT_DIR}/_artifacts
KUBEBUILDER_ASSETS ?= $(shell $(SETUP_ENVTEST) use --use-env -p path $(KUBEBUILDER_ENVTEST_KUBERNETES_VERSION))

.PHONY: test
test: $(SETUP_ENVTEST) manifests ## Run tests.
test: $(SETUP_ENVTEST) manifests kubectl ## Run tests.
KUBEBUILDER_ASSETS="$(KUBEBUILDER_ASSETS)" go test ./... $(TEST_ARGS)

##@ Build

.PHONY: build
build: manifests generate fmt vet ## Build manager binary.
build: generate fmt vet ## Build manager binary.
go build -o bin/manager main.go

.PHONY: run
run: manifests generate fmt vet ## Run a controller from your host.
run: generate fmt vet ## Run a controller from your host.
go run ./main.go

## --------------------------------------
Expand Down
4 changes: 2 additions & 2 deletions charts/rancher-turtles/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ rancherTurtles:
tag: v0.0.0
imagePullPolicy: Never
namespace: rancher-turtles-system
managerArguments: {}
managerArguments: []
imagePullSecrets: []
features:
embedded-capi:
Expand All @@ -14,7 +14,7 @@ rancherTurtles:
cluster-api-operator:
enabled: true
cert-manager:
enabled: true
enabled: false
cluster-api:
enabled: true
version: v1.4.6
Expand Down
163 changes: 163 additions & 0 deletions internal/controllers/cross_cluster_import_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,163 @@
/*
Copyright 2023 SUSE.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package controllers

import (
"context"
"fmt"
"os"
"os/exec"
"path/filepath"

. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
provisioningv1 "github.com/rancher-sandbox/rancher-turtles/internal/rancher/provisioning/v1"
"github.com/rancher-sandbox/rancher-turtles/internal/rancher/setup"
"github.com/rancher-sandbox/rancher-turtles/internal/test"
turtlesnaming "github.com/rancher-sandbox/rancher-turtles/util/naming"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/rest"
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
"sigs.k8s.io/cluster-api/util/kubeconfig"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/controller"
"sigs.k8s.io/controller-runtime/pkg/envtest"

ctrl "sigs.k8s.io/controller-runtime"
)

var (
rancherEnv *envtest.Environment
otherEnv *envtest.Environment
rancherCfg *rest.Config
otherCfg *rest.Config
rancherCl client.Client
otherCl client.Client
clusterCtx context.Context
cancel context.CancelFunc
capiCluster *clusterv1.Cluster
rancherCluster *provisioningv1.Cluster
r *CAPIImportReconciler
)

var _ = Describe("In separate clusters", func() {

BeforeEach(func() {
By("bootstrapping rancher environment")
var err error
rancherEnv = &envtest.Environment{
CRDDirectoryPaths: []string{
filepath.Join("..", "..", "hack", "crd", "bases"),
},
ErrorIfCRDPathMissing: true,
Scheme: test.FullScheme,
}
rancherCfg, rancherCl, err = test.StartEnvTest(rancherEnv)
Expect(err).NotTo(HaveOccurred())
Expect(rancherCfg).NotTo(BeNil())
Expect(rancherCl).NotTo(BeNil())

By("Bootstrapping other cluster environment")
otherEnv = &envtest.Environment{
CRDDirectoryPaths: []string{
filepath.Join("..", "..", "hack", "crd", "bases"),
},
ErrorIfCRDPathMissing: true,
Scheme: test.FullScheme,
}
otherCfg, otherCl, err = test.StartEnvTest(otherEnv)
Expect(err).NotTo(HaveOccurred())
Expect(otherCfg).NotTo(BeNil())
Expect(otherCl).NotTo(BeNil())

Expect(otherCl.Create(ctx, &corev1.Namespace{
ObjectMeta: metav1.ObjectMeta{
Name: testNamespace,
Labels: map[string]string{
importLabelName: "true",
},
},
})).To(Succeed())

Expect(rancherCl.Create(ctx, &corev1.Namespace{
ObjectMeta: metav1.ObjectMeta{
Name: testNamespace,
},
})).To(Succeed())

clusterCtx, cancel = context.WithCancel(ctx)
capiCluster = &clusterv1.Cluster{
ObjectMeta: metav1.ObjectMeta{
Name: "test-cluster",
Namespace: testNamespace,
},
}

rancherCluster = &provisioningv1.Cluster{
ObjectMeta: metav1.ObjectMeta{
Name: turtlesnaming.Name(capiCluster.Name).ToRancherName(),
Namespace: testNamespace,
},
}
})

AfterEach(func() {
cancel()
By("tearing down the 2 clsuter environment")
Expect(test.StopEnvTest(rancherEnv)).To(Succeed())
Expect(test.StopEnvTest(otherEnv)).To(Succeed())
})

It("minimal controller setup should create a Rancher cluster object from a CAPI cluster object located in a different cluster", func() {
mgr, err := ctrl.NewManager(otherCfg, ctrl.Options{
Scheme: otherEnv.Scheme,
MetricsBindAddress: "0",
HealthProbeBindAddress: "0",
})
Expect(err).ToNot(HaveOccurred())

config := kubeconfig.FromEnvTestConfig(rancherCfg, &clusterv1.Cluster{
ObjectMeta: metav1.ObjectMeta{Name: "test"},
})
kubeconfigFile, err := os.CreateTemp("", "kubeconfig")
Expect(err).ToNot(HaveOccurred())
defer os.Remove(kubeconfigFile.Name())
Expect(os.WriteFile(kubeconfigFile.Name(), config, 0600)).To(Succeed())

rancher, err := setup.RancherCluster(mgr, kubeconfigFile.Name())
Expect(err).ToNot(HaveOccurred())

reconciler := &CAPIImportReconciler{
Client: mgr.GetClient(),
RancherCluster: rancher,
}
Expect(reconciler.SetupWithManager(ctx, mgr, controller.Options{})).To(Succeed())

go func() {
Expect(mgr.Start(clusterCtx)).To(Succeed())
}()

Expect(otherCl.Create(ctx, capiCluster)).To(Succeed())
capiCluster.Status.ControlPlaneReady = true
Expect(otherCl.Status().Update(ctx, capiCluster)).To(Succeed())

args := []string{"get", fmt.Sprintf("clusters.%s", provisioningv1.GroupVersion.Group), rancherCluster.Name, "-n", rancherCluster.Namespace, "--kubeconfig", kubeconfigFile.Name()}
Eventually(exec.Command("kubectl", args...).Run()).Should(Succeed())
})

})
13 changes: 7 additions & 6 deletions internal/controllers/import_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ import (
"k8s.io/client-go/tools/record"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/cluster"
"sigs.k8s.io/controller-runtime/pkg/controller"
"sigs.k8s.io/controller-runtime/pkg/event"
"sigs.k8s.io/controller-runtime/pkg/handler"
Expand Down Expand Up @@ -68,7 +69,7 @@ const (
// CAPIImportReconciler represents a reconciler for importing CAPI clusters in Rancher.
type CAPIImportReconciler struct {
Client client.Client
RancherClient client.Client
RancherCluster cluster.Cluster
recorder record.EventRecorder
WatchFilterValue string
Scheme *runtime.Scheme
Expand Down Expand Up @@ -106,7 +107,7 @@ func (r *CAPIImportReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Ma
// Watch Rancher provisioningv2 clusters
// NOTE: we will import the types from rancher in the future
err = c.Watch(
source.Kind(mgr.GetCache(), &provisioningv1.Cluster{}),
source.Kind(r.RancherCluster.GetCache(), &provisioningv1.Cluster{}),
handler.EnqueueRequestsFromMapFunc(r.rancherClusterToCapiCluster(ctx, capiPredicates)),
//&handler.EnqueueRequestForOwner{OwnerType: &clusterv1.Cluster{}},
)
Expand Down Expand Up @@ -200,7 +201,7 @@ func (r *CAPIImportReconciler) reconcile(ctx context.Context, capiCluster *clust
Name: turtlesnaming.Name(capiCluster.Name).ToRancherName(),
}}

err := r.RancherClient.Get(ctx, client.ObjectKeyFromObject(rancherCluster), rancherCluster)
err := r.RancherCluster.GetClient().Get(ctx, client.ObjectKeyFromObject(rancherCluster), rancherCluster)
if client.IgnoreNotFound(err) != nil {
log.Error(err, fmt.Sprintf("Unable to fetch rancher cluster %s", client.ObjectKeyFromObject(rancherCluster)))
return ctrl.Result{Requeue: true}, err
Expand All @@ -218,7 +219,7 @@ func (r *CAPIImportReconciler) reconcileNormal(ctx context.Context, capiCluster
) (ctrl.Result, error) {
log := log.FromContext(ctx)

err := r.RancherClient.Get(ctx, client.ObjectKeyFromObject(rancherCluster), rancherCluster)
err := r.RancherCluster.GetClient().Get(ctx, client.ObjectKeyFromObject(rancherCluster), rancherCluster)
if apierrors.IsNotFound(err) {
shouldImport, err := util.ShouldAutoImport(ctx, log, r.Client, capiCluster, importLabelName)
if err != nil {
Expand All @@ -230,7 +231,7 @@ func (r *CAPIImportReconciler) reconcileNormal(ctx context.Context, capiCluster
return ctrl.Result{}, nil
}

if err := r.RancherClient.Create(ctx, &provisioningv1.Cluster{
if err := r.RancherCluster.GetClient().Create(ctx, &provisioningv1.Cluster{
ObjectMeta: metav1.ObjectMeta{
Name: turtlesnaming.Name(capiCluster.Name).ToRancherName(),
Namespace: capiCluster.Namespace,
Expand Down Expand Up @@ -323,7 +324,7 @@ func (r *CAPIImportReconciler) getClusterRegistrationManifest(ctx context.Contex
Name: clusterRegistrationTokenName,
Namespace: clusterName,
}}
err := r.RancherClient.Get(ctx, client.ObjectKeyFromObject(token), token)
err := r.RancherCluster.GetClient().Get(ctx, client.ObjectKeyFromObject(token), token)

if client.IgnoreNotFound(err) != nil {
return "", fmt.Errorf("error getting registration token for cluster %s: %w", clusterName, err)
Expand Down
17 changes: 16 additions & 1 deletion internal/controllers/import_controller_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ package controllers

import (
"bufio"
"context"
"encoding/json"
"errors"
"fmt"
Expand All @@ -43,6 +44,7 @@ import (
"sigs.k8s.io/cluster-api/controllers/remote"
"sigs.k8s.io/cluster-api/util/secret"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/cluster"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
)

Expand All @@ -53,12 +55,24 @@ var _ = Describe("reconcile CAPI Cluster", func() {
rancherCluster *provisioningv1.Cluster
clusterRegistrationToken *managementv3.ClusterRegistrationToken
capiKubeconfigSecret *corev1.Secret
cancel context.CancelFunc
clusterCtx context.Context
)

BeforeEach(func() {
// rancher and rancher-turtles deployed in the same cluster
clusterCtx, cancel = context.WithCancel(ctx)
cluster, err := cluster.New(testEnv.Config, func(clusterOptions *cluster.Options) {
clusterOptions.Scheme = testEnv.Scheme
})
Expect(err).ToNot(HaveOccurred())
go func() {
Expect(cluster.Start(clusterCtx)).To(Succeed())
}()

r = &CAPIImportReconciler{
Client: cl,
RancherClient: cl, // rancher and rancher-turtles deployed in the same cluster
RancherCluster: cluster,
remoteClientGetter: remote.NewClusterClient,
}

Expand Down Expand Up @@ -95,6 +109,7 @@ var _ = Describe("reconcile CAPI Cluster", func() {
})

AfterEach(func() {
defer cancel()
objs, err := manifestToObjects(strings.NewReader(testdata.ImportManifest))
clientObjs := []client.Object{
capiCluster,
Expand Down
1 change: 1 addition & 0 deletions internal/controllers/suite_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ var _ = BeforeSuite(func() {
filepath.Join("..", "..", "hack", "crd", "bases"),
},
ErrorIfCRDPathMissing: true,
Scheme: test.FullScheme,
}
cfg, cl, err = test.StartEnvTest(testEnv)
Expect(err).NotTo(HaveOccurred())
Expand Down
5 changes: 3 additions & 2 deletions internal/rancher/doc.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,8 @@ See the License for the specific language governing permissions and
limitations under the License.
*/

// Package rancher contains the rancher provisioning.cattle.io/v1 and
// management.cattle.io/v3 API proxy implementations.
// Package rancher contains rancher provisioning.cattle.io/v1 and
// management.cattle.io/v3 API proxy implementations and cluster connectivity
// setup procedures.
// +kubebuilder:object:generate=true
package rancher
Loading

0 comments on commit f8f3a81

Please sign in to comment.