diff --git a/.github/workflows/run-e2e-suite.yaml b/.github/workflows/run-e2e-suite.yaml index 22f89ddc..ce25e603 100644 --- a/.github/workflows/run-e2e-suite.yaml +++ b/.github/workflows/run-e2e-suite.yaml @@ -99,7 +99,7 @@ jobs: ${{ runner.os }}-buildx- - name: Build and push e2e image if: ${{ inputs.MANAGEMENT_CLUSTER_ENVIRONMENT == 'eks' }} - run: make e2e-image-push + run: CACHE_DIR=/tmp/.buildx-cache make e2e-image-push - name: Run e2e tests run: CACHE_DIR=/tmp/.buildx-cache make test-e2e - name: Collect run artifacts diff --git a/.github/workflows/test_chart.yaml b/.github/workflows/test_chart.yaml index ba405ae8..5ed42fea 100644 --- a/.github/workflows/test_chart.yaml +++ b/.github/workflows/test_chart.yaml @@ -84,13 +84,13 @@ jobs: run: helm install rancher-turtles out/charts/rancher-turtles/ -n rancher-turtles-system --create-namespace --wait --debug - name: Wait for core provider rollout - run: sleep 30 && kubectl rollout status deployment capi-controller-manager -n capi-system --timeout=180s + run: sleep 30 && kubectl rollout status deployment capi-controller-manager -n capi-system --timeout=10m - name: Wait for RKE2 bootstrap provider rollout - run: sleep 30 && kubectl rollout status deployment rke2-bootstrap-controller-manager -n rke2-bootstrap-system --timeout=180s + run: sleep 30 && kubectl rollout status deployment rke2-bootstrap-controller-manager -n rke2-bootstrap-system --timeout=10m - name: Wait for RKE2 control plane provider rollout - run: sleep 30 && kubectl rollout status deployment rke2-control-plane-controller-manager -n rke2-control-plane-system --timeout=180s + run: sleep 30 && kubectl rollout status deployment rke2-control-plane-controller-manager -n rke2-control-plane-system --timeout=10m - name: Run chart-testing (un-install) run: helm uninstall rancher-turtles -n rancher-turtles-system --cascade foreground --wait --debug --timeout=10m diff --git a/test/e2e/config/operator.yaml b/test/e2e/config/operator.yaml index 0ac5c843..ad5041e3 100644 --- a/test/e2e/config/operator.yaml +++ b/test/e2e/config/operator.yaml @@ -40,8 +40,8 @@ variables: ARTIFACTS_FOLDER: "_artifacts" HELM_BINARY_PATH: "helm" HELM_EXTRA_VALUES_FOLDER: "/tmp" - KUBERNETES_VERSION: "v1.31.0" - KUBERNETES_MANAGEMENT_VERSION: "v1.30.0" + KUBERNETES_VERSION: "v1.30.3" + KUBERNETES_MANAGEMENT_VERSION: "v1.30.3" RKE2_VERSION: "v1.31.0+rke2r1" KUBERNETES_MANAGEMENT_AWS_REGION: "eu-west-2" RANCHER_HOSTNAME: "localhost" diff --git a/test/e2e/const.go b/test/e2e/const.go index a35ed41b..fbbc2d02 100644 --- a/test/e2e/const.go +++ b/test/e2e/const.go @@ -66,6 +66,9 @@ var ( //go:embed data/chartmuseum/deployment.yaml ChartMuseum []byte + //go:embed data/chartmuseum/ingress.yaml + ChartMuseumIngress []byte + //go:embed data/rancher/azure-rke-config.yaml V2ProvAzureRkeConfig []byte diff --git a/test/e2e/data/chartmuseum/ingress.yaml b/test/e2e/data/chartmuseum/ingress.yaml new file mode 100644 index 00000000..e62229f1 --- /dev/null +++ b/test/e2e/data/chartmuseum/ingress.yaml @@ -0,0 +1,18 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: chart-museum-http + namespace: default +spec: + ingressClassName: ngrok # This ingress in intended to be used only with ngrok + rules: + - host: chart-museum.${RANCHER_HOSTNAME} + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: chartmuseum-service + port: + number: 8080 diff --git a/test/e2e/suites/chart-upgrade/chart_upgrade_test.go b/test/e2e/suites/chart-upgrade/chart_upgrade_test.go index c9450eb3..49121251 100644 --- a/test/e2e/suites/chart-upgrade/chart_upgrade_test.go +++ b/test/e2e/suites/chart-upgrade/chart_upgrade_test.go @@ -41,13 +41,6 @@ var _ = Describe("Chart upgrade functionality should work", Label(e2e.ShortTestL SetClient(setupClusterResult.BootstrapClusterProxy.GetClient()) SetContext(ctx) - testenv.DeployChartMuseum(ctx, testenv.DeployChartMuseumInput{ - HelmBinaryPath: e2eConfig.GetVariable(e2e.HelmBinaryPathVar), - ChartsPath: e2eConfig.GetVariable(e2e.TurtlesPathVar), - ChartVersion: e2eConfig.GetVariable(e2e.TurtlesVersionVar), - BootstrapClusterProxy: setupClusterResult.BootstrapClusterProxy, - WaitInterval: e2eConfig.GetIntervals(setupClusterResult.BootstrapClusterProxy.GetName(), "wait-controllers"), - }) }) It("Should perform upgrade from GA version to latest", func() { @@ -63,13 +56,18 @@ var _ = Describe("Chart upgrade functionality should work", Label(e2e.ShortTestL } testenv.DeployRancherTurtles(ctx, rtInput) - testenv.DeployChartMuseum(ctx, testenv.DeployChartMuseumInput{ + chartMuseumDeployInput := testenv.DeployChartMuseumInput{ HelmBinaryPath: e2eConfig.GetVariable(e2e.HelmBinaryPathVar), ChartsPath: e2eConfig.GetVariable(e2e.TurtlesPathVar), ChartVersion: e2eConfig.GetVariable(e2e.TurtlesVersionVar), BootstrapClusterProxy: setupClusterResult.BootstrapClusterProxy, WaitInterval: e2eConfig.GetIntervals(setupClusterResult.BootstrapClusterProxy.GetName(), "wait-controllers"), - }) + Variables: e2eConfig.Variables, + } + + testenv.PreChartMuseumInstallHook(&chartMuseumDeployInput, e2eConfig) + + testenv.DeployChartMuseum(ctx, chartMuseumDeployInput) upgradeInput := testenv.UpgradeRancherTurtlesInput{ BootstrapClusterProxy: setupClusterResult.BootstrapClusterProxy, @@ -122,7 +120,7 @@ var _ = Describe("Chart upgrade functionality should work", Label(e2e.ShortTestL }, e2eConfig.GetIntervals(setupClusterResult.BootstrapClusterProxy.GetName(), "wait-controllers")...) }, func() { framework.WaitForCAPIProviderRollout(ctx, framework.WaitForCAPIProviderRolloutInput{ - Getter: setupClusterResult.BootstrapClusterProxy.GetClient(), + Getter: setupClusterResult.BootstrapClusterProxy.GetClient(), Deployment: &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{ Name: "rke2-bootstrap-controller-manager", Namespace: "rke2-bootstrap-system", @@ -133,7 +131,7 @@ var _ = Describe("Chart upgrade functionality should work", Label(e2e.ShortTestL }, e2eConfig.GetIntervals(setupClusterResult.BootstrapClusterProxy.GetName(), "wait-controllers")...) }, func() { framework.WaitForCAPIProviderRollout(ctx, framework.WaitForCAPIProviderRolloutInput{ - Getter: setupClusterResult.BootstrapClusterProxy.GetClient(), + Getter: setupClusterResult.BootstrapClusterProxy.GetClient(), Deployment: &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{ Name: "rke2-control-plane-controller-manager", Namespace: "rke2-control-plane-system", diff --git a/test/e2e/suites/import-gitops/suite_test.go b/test/e2e/suites/import-gitops/suite_test.go index a552c967..9be53f5f 100644 --- a/test/e2e/suites/import-gitops/suite_test.go +++ b/test/e2e/suites/import-gitops/suite_test.go @@ -155,13 +155,18 @@ var _ = BeforeSuite(func() { testenv.DeployRancher(ctx, rancherInput) if shortTestOnly() { - testenv.DeployChartMuseum(ctx, testenv.DeployChartMuseumInput{ + chartMuseumDeployInput := testenv.DeployChartMuseumInput{ HelmBinaryPath: e2eConfig.GetVariable(e2e.HelmBinaryPathVar), ChartsPath: e2eConfig.GetVariable(e2e.TurtlesPathVar), ChartVersion: e2eConfig.GetVariable(e2e.TurtlesVersionVar), BootstrapClusterProxy: setupClusterResult.BootstrapClusterProxy, WaitInterval: e2eConfig.GetIntervals(setupClusterResult.BootstrapClusterProxy.GetName(), "wait-controllers"), - }) + Variables: e2eConfig.Variables, + } + + testenv.PreChartMuseumInstallHook(&chartMuseumDeployInput, e2eConfig) + + testenv.DeployChartMuseum(ctx, chartMuseumDeployInput) rtInput := testenv.DeployRancherTurtlesInput{ BootstrapClusterProxy: setupClusterResult.BootstrapClusterProxy, diff --git a/test/e2e/suites/migrate-gitops/suite_test.go b/test/e2e/suites/migrate-gitops/suite_test.go index b994e9c7..774da87b 100644 --- a/test/e2e/suites/migrate-gitops/suite_test.go +++ b/test/e2e/suites/migrate-gitops/suite_test.go @@ -156,13 +156,18 @@ var _ = BeforeSuite(func() { testenv.DeployRancher(ctx, rancherInput) - testenv.DeployChartMuseum(ctx, testenv.DeployChartMuseumInput{ + chartMuseumDeployInput := testenv.DeployChartMuseumInput{ HelmBinaryPath: e2eConfig.GetVariable(e2e.HelmBinaryPathVar), ChartsPath: e2eConfig.GetVariable(e2e.TurtlesPathVar), ChartVersion: e2eConfig.GetVariable(e2e.TurtlesVersionVar), BootstrapClusterProxy: setupClusterResult.BootstrapClusterProxy, WaitInterval: e2eConfig.GetIntervals(setupClusterResult.BootstrapClusterProxy.GetName(), "wait-controllers"), - }) + Variables: e2eConfig.Variables, + } + + testenv.PreChartMuseumInstallHook(&chartMuseumDeployInput, e2eConfig) + + testenv.DeployChartMuseum(ctx, chartMuseumDeployInput) rtInput := testenv.DeployRancherTurtlesInput{ BootstrapClusterProxy: setupClusterResult.BootstrapClusterProxy, diff --git a/test/framework/chartmuseum_helper.go b/test/framework/chartmuseum_helper.go index c4bf47d0..3d3395cf 100644 --- a/test/framework/chartmuseum_helper.go +++ b/test/framework/chartmuseum_helper.go @@ -21,6 +21,7 @@ import ( "fmt" "os/exec" + "github.com/drone/envsubst/v2" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -58,6 +59,12 @@ type ChartMuseumInput struct { // WaitInterval is the wait interval. WaitInterval []interface{} + + // CustomIngressConfig is the custom ingress configuration. + CustomIngressConfig []byte + + // Variables is the collection of variables. + Variables VariableCollection } // DeployChartMuseum will create a new repo in the Gitea server. @@ -103,17 +110,41 @@ func DeployChartMuseum(ctx context.Context, input ChartMuseumInput) string { AddressIndex: 0, }) + path := fmt.Sprintf("http://%s:%d", addr, port.NodePort) + + if input.CustomIngressConfig != nil { + By("Creating custom ingress for chartmuseum") + variableGetter := GetVariable(input.Variables) + ingress, err := envsubst.Eval(string(input.CustomIngressConfig), variableGetter) + Expect(err).ToNot(HaveOccurred()) + Expect(input.Proxy.Apply(ctx, []byte(ingress))).To(Succeed()) + + By("Getting git server ingress address") + host := GetIngressHost(ctx, GetIngressHostInput{ + GetLister: input.Proxy.GetClient(), + IngressRuleIndex: 0, + IngressName: "chart-museum-http", + IngressNamespace: "default", + }) + + path = fmt.Sprintf("http://%s", host) + } + By("Adding local rancher turtles chart repo") addChart := &opframework.HelmChart{ BinaryPath: input.HelmBinaryPath, Name: "rancher-turtles-local", - Path: fmt.Sprintf("http://%s:%d", addr, port.NodePort), + Path: path, Commands: opframework.Commands(opframework.Repo, opframework.Add), AdditionalFlags: opframework.Flags("--force-update"), Kubeconfig: input.Proxy.GetKubeconfigPath(), } - _, err := addChart.Run(nil) - Expect(err).ToNot(HaveOccurred()) + + Eventually(func() error { + _, err := addChart.Run(nil) + + return err + }, input.WaitInterval...).Should(Succeed(), "Failed to connect to workload cluster using CAPI kubeconfig") By("Pushing local chart to chartmuseum") exec.Command( diff --git a/test/testenv/charmuseum.go b/test/testenv/charmuseum.go index fdd82239..2e97e7a6 100644 --- a/test/testenv/charmuseum.go +++ b/test/testenv/charmuseum.go @@ -24,6 +24,7 @@ import ( "github.com/rancher/turtles/test/e2e" "sigs.k8s.io/cluster-api/test/framework" + "sigs.k8s.io/cluster-api/test/framework/clusterctl" turtlesframework "github.com/rancher/turtles/test/framework" ) @@ -44,6 +45,12 @@ type DeployChartMuseumInput struct { // WaitInterval is the interval to wait for. WaitInterval []interface{} + + // CustomIngressConfig is the custom ingress configuration. + CustomIngressConfig []byte + + // Variables is the collection of variables. + Variables turtlesframework.VariableCollection } // DeployChartMuseum installs ChartMuseum to the Kubernetes cluster using the provided input parameters. @@ -66,5 +73,17 @@ func DeployChartMuseum(ctx context.Context, input DeployChartMuseumInput) { DeploymentName: "chartmuseum", ServiceName: "chartmuseum-service", PortName: "http", + CustomIngressConfig: input.CustomIngressConfig, + Variables: input.Variables, }) } + +// PreChartMuseumInstallHook is a pre-install hook for ChartMuseum. +func PreChartMuseumInstallHook(chartMuseumInput *DeployChartMuseumInput, e2eConfig *clusterctl.E2EConfig) { + infrastructureType := e2e.ManagementClusterEnvironmentType(e2eConfig.GetVariable(e2e.ManagementClusterEnvironmentVar)) + + switch infrastructureType { + case e2e.ManagementClusterEnvironmentKind: + chartMuseumInput.CustomIngressConfig = e2e.ChartMuseumIngress + } +} diff --git a/test/testenv/turtles.go b/test/testenv/turtles.go index fc323e49..cd1918d2 100644 --- a/test/testenv/turtles.go +++ b/test/testenv/turtles.go @@ -157,7 +157,7 @@ func DeployRancherTurtles(ctx context.Context, input DeployRancherTurtlesInput) AdditionalFlags: opframework.Flags( "--dependency-update", "-n", namespace, - "--create-namespace", "--wait"), + "--create-namespace", "--wait", "--timeout", "10m"), } values := map[string]string{ @@ -325,6 +325,7 @@ func UpgradeRancherTurtles(ctx context.Context, input UpgradeRancherTurtlesInput "upgrade", "rancher-turtles", "rancher-turtles-local/rancher-turtles", "-n", input.Namespace, "--wait", + "--timeout", "10m", "--kubeconfig", input.BootstrapClusterProxy.GetKubeconfigPath(), "--set", "rancherTurtles.managerArguments[0]=--insecure-skip-verify=true", "--set", fmt.Sprintf("rancherTurtles.image=%s", input.Image),