Skip to content

Commit

Permalink
Tests: apply embedded capi ff after rancher install (#190)
Browse files Browse the repository at this point in the history
Signed-off-by: Danil Grigorev <[email protected]>
  • Loading branch information
Danil-Grigorev authored Oct 9, 2023
1 parent f9cf357 commit 8b8ecf3
Show file tree
Hide file tree
Showing 7 changed files with 130 additions and 81 deletions.
7 changes: 3 additions & 4 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -180,7 +180,7 @@ help: ## Display this help.
##@ Development
.PHONY: manifests
manifests: vendor controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects.
$(CONTROLLER_GEN) rbac:roleName=manager-role crd paths="./..." output:crd:artifacts:config=hack/crd/bases
$(CONTROLLER_GEN) rbac:roleName=manager-role crd paths="./internal/rancher/..." output:crd:artifacts:config=hack/crd/bases
$(CONTROLLER_GEN) rbac:roleName=manager-role crd paths="./vendor/sigs.k8s.io/cluster-api/..." output:crd:artifacts:config=hack/crd/bases
# Vendor is only required for pulling latest CRDs from the dependencies
$(MAKE) vendor-clean
Expand All @@ -198,7 +198,6 @@ vendor-clean:

.PHONY: generate
generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations.
$(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..."

.PHONY: generate-modules
generate-modules: ## Run go mod tidy to ensure modules are up to date
Expand Down Expand Up @@ -249,11 +248,11 @@ test: $(SETUP_ENVTEST) manifests ## Run tests.
##@ Build

.PHONY: build
build: manifests generate fmt vet ## Build manager binary.
build: generate fmt vet ## Build manager binary.
go build -o bin/manager main.go

.PHONY: run
run: manifests generate fmt vet ## Run a controller from your host.
run: generate fmt vet ## Run a controller from your host.
go run ./main.go

## --------------------------------------
Expand Down
4 changes: 2 additions & 2 deletions charts/rancher-turtles/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ rancherTurtles:
tag: v0.0.0
imagePullPolicy: Never
namespace: rancher-turtles-system
managerArguments: {}
managerArguments: []
imagePullSecrets: []
features:
embedded-capi:
Expand All @@ -14,7 +14,7 @@ rancherTurtles:
cluster-api-operator:
enabled: true
cert-manager:
enabled: true
enabled: false
cluster-api:
enabled: true
version: v1.4.6
Expand Down
5 changes: 4 additions & 1 deletion test/e2e/config/operator.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -21,13 +21,16 @@ intervals:
variables:
RANCHER_VERSION: "v2.7.6"
RANCHER_HOSTNAME: "localhost"
RANCHER_FEATURES: "embedded-cluster-api=false"
RANCHER_FEATURES: ""
RANCHER_PATH: "rancher-stable/rancher"
KUBERNETES_VERSION: "v1.26.3"
RKE2_VERSION: "v1.26.8+rke2r1"
CAPI_CORE: "cluster-api:v1.4.6"
RANCHER_REPO_NAME: "rancher-stable"
RANCHER_URL: "https://releases.rancher.com/server-charts/stable"
CERT_MANAGER_URL: "https://charts.jetstack.io"
CERT_MANAGER_REPO_NAME: "jetstack"
CERT_MANAGER_PATH: "jetstack/cert-manager"
RANCHER_PASSWORD: "rancheradmin"
# Additional setup for establishing rancher ingress
NGROK_REPO_NAME: "ngrok"
Expand Down
23 changes: 13 additions & 10 deletions test/e2e/const.go
Original file line number Diff line number Diff line change
Expand Up @@ -71,16 +71,19 @@ const (
)

const (
KubernetesVersionVar = "KUBERNETES_VERSION"
RancherFeaturesVar = "RANCHER_FEATURES"
RancherHostnameVar = "RANCHER_HOSTNAME"
RancherVersionVar = "RANCHER_VERSION"
RancherPathVar = "RANCHER_PATH"
RancherUrlVar = "RANCHER_URL"
RancherRepoNameVar = "RANCHER_REPO_NAME"
RancherPasswordVar = "RANCHER_PASSWORD"
CapiInfrastructureVar = "CAPI_INFRASTRUCTURE"
CapiCoreVar = "CAPI_CORE"
KubernetesVersionVar = "KUBERNETES_VERSION"
RancherFeaturesVar = "RANCHER_FEATURES"
RancherHostnameVar = "RANCHER_HOSTNAME"
RancherVersionVar = "RANCHER_VERSION"
RancherPathVar = "RANCHER_PATH"
RancherUrlVar = "RANCHER_URL"
RancherRepoNameVar = "RANCHER_REPO_NAME"
RancherPasswordVar = "RANCHER_PASSWORD"
CertManagerUrlVar = "CERT_MANAGER_URL"
CertManagerRepoNameVar = "CERT_MANAGER_REPO_NAME"
CertManagerPathVar = "CERT_MANAGER_PATH"
CapiInfrastructureVar = "CAPI_INFRASTRUCTURE"
CapiCoreVar = "CAPI_CORE"

NgrokRepoNameVar = "NGROK_REPO_NAME"
NgrokUrlVar = "NGROK_URL"
Expand Down
70 changes: 36 additions & 34 deletions test/e2e/suites/import-gitops/suite_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,42 @@ var _ = BeforeSuite(func() {
hostName = setupClusterResult.IsolatedHostName
}

testenv.RancherDeployIngress(ctx, testenv.RancherDeployIngressInput{
BootstrapClusterProxy: setupClusterResult.BootstrapClusterProxy,
HelmBinaryPath: flagVals.HelmBinaryPath,
IsolatedMode: flagVals.IsolatedMode,
NginxIngress: e2e.NginxIngress,
NginxIngressNamespace: e2e.NginxIngressNamespace,
IngressWaitInterval: e2eConfig.GetIntervals(setupClusterResult.BootstrapClusterProxy.GetName(), "wait-rancher"),
NgrokApiKey: e2eConfig.GetVariable(e2e.NgrokApiKeyVar),
NgrokAuthToken: e2eConfig.GetVariable(e2e.NgrokAuthTokenVar),
NgrokPath: e2eConfig.GetVariable(e2e.NgrokPathVar),
NgrokRepoName: e2eConfig.GetVariable(e2e.NgrokRepoNameVar),
NgrokRepoURL: e2eConfig.GetVariable(e2e.NgrokUrlVar),
DefaultIngressClassPatch: e2e.IngressClassPatch,
})

testenv.DeployRancher(ctx, testenv.DeployRancherInput{
BootstrapClusterProxy: setupClusterResult.BootstrapClusterProxy,
HelmBinaryPath: flagVals.HelmBinaryPath,
CertManagerChartPath: e2eConfig.GetVariable(e2e.CertManagerPathVar),
CertManagerUrl: e2eConfig.GetVariable(e2e.CertManagerUrlVar),
CertManagerRepoName: e2eConfig.GetVariable(e2e.CertManagerRepoNameVar),
RancherChartRepoName: e2eConfig.GetVariable(e2e.RancherRepoNameVar),
RancherChartURL: e2eConfig.GetVariable(e2e.RancherUrlVar),
RancherChartPath: e2eConfig.GetVariable(e2e.RancherPathVar),
RancherVersion: e2eConfig.GetVariable(e2e.RancherVersionVar),
RancherHost: hostName,
RancherNamespace: e2e.RancherNamespace,
RancherPassword: e2eConfig.GetVariable(e2e.RancherPasswordVar),
RancherPatches: [][]byte{e2e.RancherSettingPatch},
RancherWaitInterval: e2eConfig.GetIntervals(setupClusterResult.BootstrapClusterProxy.GetName(), "wait-rancher"),
ControllerWaitInterval: e2eConfig.GetIntervals(setupClusterResult.BootstrapClusterProxy.GetName(), "wait-controllers"),
IsolatedMode: flagVals.IsolatedMode,
RancherIngressConfig: e2e.IngressConfig,
RancherServicePatch: e2e.RancherServicePatch,
})

testenv.DeployRancherTurtles(ctx, testenv.DeployRancherTurtlesInput{
BootstrapClusterProxy: setupClusterResult.BootstrapClusterProxy,
HelmBinaryPath: flagVals.HelmBinaryPath,
Expand Down Expand Up @@ -146,40 +182,6 @@ var _ = BeforeSuite(func() {
})
}

testenv.RancherDeployIngress(ctx, testenv.RancherDeployIngressInput{
BootstrapClusterProxy: setupClusterResult.BootstrapClusterProxy,
HelmBinaryPath: flagVals.HelmBinaryPath,
IsolatedMode: flagVals.IsolatedMode,
NginxIngress: e2e.NginxIngress,
NginxIngressNamespace: e2e.NginxIngressNamespace,
IngressWaitInterval: e2eConfig.GetIntervals(setupClusterResult.BootstrapClusterProxy.GetName(), "wait-rancher"),
NgrokApiKey: e2eConfig.GetVariable(e2e.NgrokApiKeyVar),
NgrokAuthToken: e2eConfig.GetVariable(e2e.NgrokAuthTokenVar),
NgrokPath: e2eConfig.GetVariable(e2e.NgrokPathVar),
NgrokRepoName: e2eConfig.GetVariable(e2e.NgrokRepoNameVar),
NgrokRepoURL: e2eConfig.GetVariable(e2e.NgrokUrlVar),
DefaultIngressClassPatch: e2e.IngressClassPatch,
})

testenv.DeployRancher(ctx, testenv.DeployRancherInput{
BootstrapClusterProxy: setupClusterResult.BootstrapClusterProxy,
HelmBinaryPath: flagVals.HelmBinaryPath,
RancherChartRepoName: e2eConfig.GetVariable(e2e.RancherRepoNameVar),
RancherChartURL: e2eConfig.GetVariable(e2e.RancherUrlVar),
RancherChartPath: e2eConfig.GetVariable(e2e.RancherPathVar),
RancherVersion: e2eConfig.GetVariable(e2e.RancherVersionVar),
RancherHost: hostName,
RancherNamespace: e2e.RancherNamespace,
RancherPassword: e2eConfig.GetVariable(e2e.RancherPasswordVar),
RancherFeatures: e2eConfig.GetVariable(e2e.RancherFeaturesVar),
RancherSettingsPatch: e2e.RancherSettingPatch,
RancherWaitInterval: e2eConfig.GetIntervals(setupClusterResult.BootstrapClusterProxy.GetName(), "wait-rancher"),
ControllerWaitInterval: e2eConfig.GetIntervals(setupClusterResult.BootstrapClusterProxy.GetName(), "wait-controllers"),
IsolatedMode: flagVals.IsolatedMode,
RancherIngressConfig: e2e.IngressConfig,
RancherServicePatch: e2e.RancherServicePatch,
})

giteaResult = testenv.DeployGitea(ctx, testenv.DeployGiteaInput{
BootstrapClusterProxy: setupClusterResult.BootstrapClusterProxy,
HelmBinaryPath: flagVals.HelmBinaryPath,
Expand Down
30 changes: 17 additions & 13 deletions test/e2e/suites/v2prov/suite_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -103,18 +103,6 @@ var _ = BeforeSuite(func() {
hostName = setupClusterResult.IsolatedHostName
}

testenv.DeployRancherTurtles(ctx, testenv.DeployRancherTurtlesInput{
BootstrapClusterProxy: setupClusterResult.BootstrapClusterProxy,
HelmBinaryPath: flagVals.HelmBinaryPath,
ChartPath: flagVals.ChartPath,
CAPIProvidersSecretYAML: e2e.CapiProvidersSecret,
CAPIProvidersYAML: e2e.CapiProviders,
Namespace: turtlesframework.DefaultRancherTurtlesNamespace,
Image: "ghcr.io/rancher-sandbox/rancher-turtles-amd64",
Tag: "v0.0.1",
WaitDeploymentsReadyInterval: e2eConfig.GetIntervals(setupClusterResult.BootstrapClusterProxy.GetName(), "wait-controllers"),
})

testenv.RancherDeployIngress(ctx, testenv.RancherDeployIngressInput{
BootstrapClusterProxy: setupClusterResult.BootstrapClusterProxy,
HelmBinaryPath: flagVals.HelmBinaryPath,
Expand All @@ -133,6 +121,9 @@ var _ = BeforeSuite(func() {
testenv.DeployRancher(ctx, testenv.DeployRancherInput{
BootstrapClusterProxy: setupClusterResult.BootstrapClusterProxy,
HelmBinaryPath: flagVals.HelmBinaryPath,
CertManagerChartPath: e2eConfig.GetVariable(e2e.CertManagerPathVar),
CertManagerUrl: e2eConfig.GetVariable(e2e.CertManagerUrlVar),
CertManagerRepoName: e2eConfig.GetVariable(e2e.CertManagerRepoNameVar),
RancherChartRepoName: "rancher-latest",
RancherChartURL: "https://releases.rancher.com/server-charts/latest",
RancherChartPath: "rancher-latest/rancher",
Expand All @@ -143,13 +134,26 @@ var _ = BeforeSuite(func() {
RancherNamespace: e2e.RancherNamespace,
RancherPassword: e2eConfig.GetVariable(e2e.RancherPasswordVar),
RancherFeatures: e2eConfig.GetVariable(e2e.RancherFeaturesVar),
RancherSettingsPatch: e2e.RancherSettingPatch,
RancherPatches: [][]byte{e2e.RancherSettingPatch},
RancherWaitInterval: e2eConfig.GetIntervals(setupClusterResult.BootstrapClusterProxy.GetName(), "wait-rancher"),
ControllerWaitInterval: e2eConfig.GetIntervals(setupClusterResult.BootstrapClusterProxy.GetName(), "wait-controllers"),
IsolatedMode: flagVals.IsolatedMode,
RancherIngressConfig: e2e.IngressConfig,
RancherServicePatch: e2e.RancherServicePatch,
})

testenv.DeployRancherTurtles(ctx, testenv.DeployRancherTurtlesInput{
BootstrapClusterProxy: setupClusterResult.BootstrapClusterProxy,
HelmBinaryPath: flagVals.HelmBinaryPath,
ChartPath: flagVals.ChartPath,
CAPIProvidersSecretYAML: e2e.CapiProvidersSecret,
CAPIProvidersYAML: e2e.CapiProviders,
Namespace: turtlesframework.DefaultRancherTurtlesNamespace,
Image: "ghcr.io/rancher-sandbox/rancher-turtles-amd64",
Tag: "v0.0.1",
WaitDeploymentsReadyInterval: e2eConfig.GetIntervals(setupClusterResult.BootstrapClusterProxy.GetName(), "wait-controllers"),
})

})

var _ = AfterSuite(func() {
Expand Down
72 changes: 55 additions & 17 deletions test/testenv/rancher.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,20 +22,27 @@ import (

. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
turtlesframework "github.com/rancher-sandbox/rancher-turtles/test/framework"

"github.com/drone/envsubst/v2"
"github.com/rancher-sandbox/rancher-turtles/test/e2e"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
opframework "sigs.k8s.io/cluster-api-operator/test/framework"
"sigs.k8s.io/cluster-api/test/framework"
"sigs.k8s.io/cluster-api/test/framework/clusterctl"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/envtest/komega"
)

type DeployRancherInput struct {
BootstrapClusterProxy framework.ClusterProxy
E2EConfig *clusterctl.E2EConfig
HelmBinaryPath string
CertManagerChartPath string
CertManagerUrl string
CertManagerRepoName string
RancherChartRepoName string
RancherChartURL string
RancherChartPath string
Expand All @@ -45,7 +52,7 @@ type DeployRancherInput struct {
RancherHost string
RancherPassword string
RancherFeatures string
RancherSettingsPatch []byte
RancherPatches [][]byte
RancherWaitInterval []interface{}
ControllerWaitInterval []interface{}
IsolatedMode bool
Expand All @@ -59,6 +66,9 @@ func DeployRancher(ctx context.Context, input DeployRancherInput) {
Expect(ctx).NotTo(BeNil(), "ctx is required for DeployRancher")
Expect(input.BootstrapClusterProxy).ToNot(BeNil(), "BootstrapClusterProxy is required for DeployRancher")
Expect(input.HelmBinaryPath).ToNot(BeEmpty(), "HelmBinaryPath is required for DeployRancher")
Expect(input.CertManagerRepoName).ToNot(BeEmpty(), "CertManagerRepoName is required for DeployRancher")
Expect(input.CertManagerUrl).ToNot(BeEmpty(), "CertManagerUrl is required for DeployRancher")
Expect(input.CertManagerChartPath).ToNot(BeEmpty(), "CertManagerChartPath is required for DeployRancher")
Expect(input.RancherChartRepoName).ToNot(BeEmpty(), "RancherChartRepoName is required for DeployRancher")
Expect(input.RancherChartURL).ToNot(BeEmpty(), "RancherChartURL is required for DeployRancher")
Expect(input.RancherChartPath).ToNot(BeEmpty(), "RancherChartPath is required for DeployRancher")
Expand All @@ -75,16 +85,28 @@ func DeployRancher(ctx context.Context, input DeployRancherInput) {
Fail("Only one of RancherVersion or RancherImageTag cen be used")
}

By("Adding Rancher chart repo")
By("Add cert manager chart repo")
addChart := &opframework.HelmChart{
BinaryPath: input.HelmBinaryPath,
Name: input.CertManagerRepoName,
Path: input.CertManagerUrl,
Commands: opframework.Commands(opframework.Repo, opframework.Add),
AdditionalFlags: opframework.Flags("--force-update"),
Kubeconfig: input.BootstrapClusterProxy.GetKubeconfigPath(),
}
_, err := addChart.Run(nil)
Expect(err).ToNot(HaveOccurred())

By("Adding Rancher chart repo")
addChart = &opframework.HelmChart{
BinaryPath: input.HelmBinaryPath,
Name: input.RancherChartRepoName,
Path: input.RancherChartURL,
Commands: opframework.Commands(opframework.Repo, opframework.Add),
AdditionalFlags: opframework.Flags("--force-update"),
Kubeconfig: input.BootstrapClusterProxy.GetKubeconfigPath(),
}
_, err := addChart.Run(nil)
_, err = addChart.Run(nil)
Expect(err).ToNot(HaveOccurred())

updateChart := &opframework.HelmChart{
Expand All @@ -95,11 +117,28 @@ func DeployRancher(ctx context.Context, input DeployRancherInput) {
_, err = updateChart.Run(nil)
Expect(err).ToNot(HaveOccurred())

By("Installing cert-manager")
chart := &opframework.HelmChart{
BinaryPath: input.HelmBinaryPath,
Path: input.CertManagerChartPath,
Name: "cert-manager",
Kubeconfig: input.BootstrapClusterProxy.GetKubeconfigPath(),
AdditionalFlags: opframework.Flags(
"--namespace", "cert-manager",
"--version", "v1.12.0",
"--create-namespace",
),
Wait: true,
}
_, err = chart.Run(map[string]string{
"installCRDs": "true",
})
Expect(err).ToNot(HaveOccurred())

By("Installing Rancher")
installFlags := opframework.Flags(
"--namespace", input.RancherNamespace,
"--create-namespace",
"--wait",
)
if input.RancherVersion != "" {
installFlags = append(installFlags, "--version", input.RancherVersion)
Expand All @@ -108,12 +147,13 @@ func DeployRancher(ctx context.Context, input DeployRancherInput) {
installFlags = append(installFlags, "--devel")
}

chart := &opframework.HelmChart{
chart = &opframework.HelmChart{
BinaryPath: input.HelmBinaryPath,
Path: input.RancherChartPath,
Name: "rancher",
Kubeconfig: input.BootstrapClusterProxy.GetKubeconfigPath(),
AdditionalFlags: installFlags,
Wait: true,
}
values := map[string]string{
"bootstrapPassword": input.RancherPassword,
Expand All @@ -131,18 +171,16 @@ func DeployRancher(ctx context.Context, input DeployRancherInput) {
_, err = chart.Run(values)
Expect(err).ToNot(HaveOccurred())

if len(input.RancherSettingsPatch) > 0 {
By("Updating rancher settings")
settingPatch, err := envsubst.Eval(string(input.RancherSettingsPatch), func(s string) string {
switch s {
case "RANCHER_HOSTNAME":
return input.RancherHost
default:
return os.Getenv(s)
}
})
Expect(err).ToNot(HaveOccurred())
Expect(input.BootstrapClusterProxy.Apply(ctx, []byte(settingPatch))).To(Succeed())
By("Updating rancher configuration")
for _, patch := range input.RancherPatches {
Expect(turtlesframework.ApplyFromTemplate(ctx, turtlesframework.ApplyFromTemplateInput{
Proxy: input.BootstrapClusterProxy,
Template: patch,
Getter: input.E2EConfig.GetVariable,
AddtionalEnvironmentVariables: map[string]string{
e2e.RancherHostnameVar: input.RancherHost,
},
})).To(Succeed())
}

if !input.IsolatedMode {
Expand Down

0 comments on commit 8b8ecf3

Please sign in to comment.