Use ngrok ingress for chart museum when running locally

.github/workflows/run-e2e-suite.yaml

@@ -99,7 +99,7 @@ jobs:
             ${{ runner.os }}-buildx-
       - name: Build and push e2e image
         if: ${{ inputs.MANAGEMENT_CLUSTER_ENVIRONMENT == 'eks' }}
-        run: make e2e-image-push
+        run: CACHE_DIR=/tmp/.buildx-cache make e2e-image-push
       - name: Run e2e tests
         run: CACHE_DIR=/tmp/.buildx-cache make test-e2e
       - name: Collect run artifacts

.github/workflows/test_chart.yaml

@@ -93,7 +93,7 @@ jobs:
         run: sleep 30 && kubectl rollout status deployment rke2-control-plane-controller-manager -n rke2-control-plane-system --timeout=180s
 
       - name: Run chart-testing (un-install)
-        run: helm uninstall rancher-turtles -n rancher-turtles-system --cascade foreground --wait --debug --timeout=10m
+        run: helm uninstall rancher-turtles -n rancher-turtles-system --cascade foreground --wait --debug --timeout=180s
 
       - name: Run chart re-install
         run: helm install rancher-turtles out/charts/rancher-turtles/ -n rancher-turtles-system --create-namespace --wait --debug

test/e2e/config/operator.yaml

@@ -40,8 +40,8 @@ variables:
   ARTIFACTS_FOLDER: "_artifacts"
   HELM_BINARY_PATH: "helm"
   HELM_EXTRA_VALUES_FOLDER: "/tmp"
-  KUBERNETES_VERSION: "v1.31.0"
-  KUBERNETES_MANAGEMENT_VERSION: "v1.30.0"
+  KUBERNETES_VERSION: "v1.30.3"
+  KUBERNETES_MANAGEMENT_VERSION: "v1.30.3"
   RKE2_VERSION: "v1.31.0+rke2r1"
   KUBERNETES_MANAGEMENT_AWS_REGION: "eu-west-2"
   RANCHER_HOSTNAME: "localhost"

test/e2e/const.go

@@ -66,6 +66,9 @@ var (
 	//go:embed data/chartmuseum/deployment.yaml
 	ChartMuseum []byte
 
+	//go:embed data/chartmuseum/ingress.yaml
+	ChartMuseumIngress []byte
+
 	//go:embed data/rancher/azure-rke-config.yaml
 	V2ProvAzureRkeConfig []byte
 

test/e2e/data/chartmuseum/ingress.yaml

@@ -0,0 +1,18 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: chart-museum-http
+  namespace: default
+spec:
+  ingressClassName: ngrok # This ingress in intended to be used only with ngrok
+  rules:
+  - host: chart-museum.${RANCHER_HOSTNAME}
+    http:
+      paths:
+      - path: /
+        pathType: Prefix
+        backend:
+          service:
+            name: chartmuseum-service
+            port:
+              number: 8080

test/e2e/suites/chart-upgrade/chart_upgrade_test.go

@@ -41,13 +41,6 @@ var _ = Describe("Chart upgrade functionality should work", Label(e2e.ShortTestL
 		SetClient(setupClusterResult.BootstrapClusterProxy.GetClient())
 		SetContext(ctx)
 
-		testenv.DeployChartMuseum(ctx, testenv.DeployChartMuseumInput{
-			HelmBinaryPath:        e2eConfig.GetVariable(e2e.HelmBinaryPathVar),
-			ChartsPath:            e2eConfig.GetVariable(e2e.TurtlesPathVar),
-			ChartVersion:          e2eConfig.GetVariable(e2e.TurtlesVersionVar),
-			BootstrapClusterProxy: setupClusterResult.BootstrapClusterProxy,
-			WaitInterval:          e2eConfig.GetIntervals(setupClusterResult.BootstrapClusterProxy.GetName(), "wait-controllers"),
-		})
 	})
 
 	It("Should perform upgrade from GA version to latest", func() {
@@ -63,13 +56,18 @@ var _ = Describe("Chart upgrade functionality should work", Label(e2e.ShortTestL
 		}
 		testenv.DeployRancherTurtles(ctx, rtInput)
 
-		testenv.DeployChartMuseum(ctx, testenv.DeployChartMuseumInput{
+		chartMuseumDeployInput := testenv.DeployChartMuseumInput{
 			HelmBinaryPath:        e2eConfig.GetVariable(e2e.HelmBinaryPathVar),
 			ChartsPath:            e2eConfig.GetVariable(e2e.TurtlesPathVar),
 			ChartVersion:          e2eConfig.GetVariable(e2e.TurtlesVersionVar),
 			BootstrapClusterProxy: setupClusterResult.BootstrapClusterProxy,
 			WaitInterval:          e2eConfig.GetIntervals(setupClusterResult.BootstrapClusterProxy.GetName(), "wait-controllers"),
-		})
+			Variables:             e2eConfig.Variables,
+		}
+
+		testenv.PreChartMuseumInstallHook(&chartMuseumDeployInput, e2eConfig)
+
+		testenv.DeployChartMuseum(ctx, chartMuseumDeployInput)
 
 		upgradeInput := testenv.UpgradeRancherTurtlesInput{
 			BootstrapClusterProxy:        setupClusterResult.BootstrapClusterProxy,
@@ -122,7 +120,7 @@ var _ = Describe("Chart upgrade functionality should work", Label(e2e.ShortTestL
 			}, e2eConfig.GetIntervals(setupClusterResult.BootstrapClusterProxy.GetName(), "wait-controllers")...)
 		}, func() {
 			framework.WaitForCAPIProviderRollout(ctx, framework.WaitForCAPIProviderRolloutInput{
-				Getter:  setupClusterResult.BootstrapClusterProxy.GetClient(),
+				Getter: setupClusterResult.BootstrapClusterProxy.GetClient(),
 				Deployment: &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{
 					Name:      "rke2-bootstrap-controller-manager",
 					Namespace: "rke2-bootstrap-system",
@@ -133,7 +131,7 @@ var _ = Describe("Chart upgrade functionality should work", Label(e2e.ShortTestL
 			}, e2eConfig.GetIntervals(setupClusterResult.BootstrapClusterProxy.GetName(), "wait-controllers")...)
 		}, func() {
 			framework.WaitForCAPIProviderRollout(ctx, framework.WaitForCAPIProviderRolloutInput{
-				Getter:  setupClusterResult.BootstrapClusterProxy.GetClient(),
+				Getter: setupClusterResult.BootstrapClusterProxy.GetClient(),
 				Deployment: &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{
 					Name:      "rke2-control-plane-controller-manager",
 					Namespace: "rke2-control-plane-system",

test/e2e/suites/import-gitops/suite_test.go

@@ -155,13 +155,18 @@ var _ = BeforeSuite(func() {
 	testenv.DeployRancher(ctx, rancherInput)
 
 	if shortTestOnly() {
-		testenv.DeployChartMuseum(ctx, testenv.DeployChartMuseumInput{
+		chartMuseumDeployInput := testenv.DeployChartMuseumInput{
 			HelmBinaryPath:        e2eConfig.GetVariable(e2e.HelmBinaryPathVar),
 			ChartsPath:            e2eConfig.GetVariable(e2e.TurtlesPathVar),
 			ChartVersion:          e2eConfig.GetVariable(e2e.TurtlesVersionVar),
 			BootstrapClusterProxy: setupClusterResult.BootstrapClusterProxy,
 			WaitInterval:          e2eConfig.GetIntervals(setupClusterResult.BootstrapClusterProxy.GetName(), "wait-controllers"),
-		})
+			Variables:             e2eConfig.Variables,
+		}
+
+		testenv.PreChartMuseumInstallHook(&chartMuseumDeployInput, e2eConfig)
+
+		testenv.DeployChartMuseum(ctx, chartMuseumDeployInput)
 
 		rtInput := testenv.DeployRancherTurtlesInput{
 			BootstrapClusterProxy:        setupClusterResult.BootstrapClusterProxy,

test/e2e/suites/migrate-gitops/suite_test.go

@@ -156,13 +156,18 @@ var _ = BeforeSuite(func() {
 
 	testenv.DeployRancher(ctx, rancherInput)
 
-	testenv.DeployChartMuseum(ctx, testenv.DeployChartMuseumInput{
+	chartMuseumDeployInput := testenv.DeployChartMuseumInput{
 		HelmBinaryPath:        e2eConfig.GetVariable(e2e.HelmBinaryPathVar),
 		ChartsPath:            e2eConfig.GetVariable(e2e.TurtlesPathVar),
 		ChartVersion:          e2eConfig.GetVariable(e2e.TurtlesVersionVar),
 		BootstrapClusterProxy: setupClusterResult.BootstrapClusterProxy,
 		WaitInterval:          e2eConfig.GetIntervals(setupClusterResult.BootstrapClusterProxy.GetName(), "wait-controllers"),
-	})
+		Variables:             e2eConfig.Variables,
+	}
+
+	testenv.PreChartMuseumInstallHook(&chartMuseumDeployInput, e2eConfig)
+
+	testenv.DeployChartMuseum(ctx, chartMuseumDeployInput)
 
 	rtInput := testenv.DeployRancherTurtlesInput{
 		BootstrapClusterProxy:        setupClusterResult.BootstrapClusterProxy,

test/framework/chartmuseum_helper.go

@@ -21,6 +21,7 @@ import (
 	"fmt"
 	"os/exec"
 
+	"github.com/drone/envsubst/v2"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 
@@ -58,6 +59,12 @@ type ChartMuseumInput struct {
 
 	// WaitInterval is the wait interval.
 	WaitInterval []interface{}
+
+	// CustomIngressConfig is the custom ingress configuration.
+	CustomIngressConfig []byte
+
+	// Variables is the collection of variables.
+	Variables VariableCollection
 }
 
 // DeployChartMuseum will create a new repo in the Gitea server.
@@ -103,17 +110,41 @@ func DeployChartMuseum(ctx context.Context, input ChartMuseumInput) string {
 		AddressIndex: 0,
 	})
 
+	path := fmt.Sprintf("http://%s:%d", addr, port.NodePort)
+
+	if input.CustomIngressConfig != nil {
+		By("Creating custom ingress for chartmuseum")
+		variableGetter := GetVariable(input.Variables)
+		ingress, err := envsubst.Eval(string(input.CustomIngressConfig), variableGetter)
+		Expect(err).ToNot(HaveOccurred())
+		Expect(input.Proxy.Apply(ctx, []byte(ingress))).To(Succeed())
+
+		By("Getting git server ingress address")
+		host := GetIngressHost(ctx, GetIngressHostInput{
+			GetLister:        input.Proxy.GetClient(),
+			IngressRuleIndex: 0,
+			IngressName:      "chart-museum-http",
+			IngressNamespace: "default",
+		})
+
+		path = fmt.Sprintf("http://%s", host)
+	}
+
 	By("Adding local rancher turtles chart repo")
 	addChart := &opframework.HelmChart{
 		BinaryPath:      input.HelmBinaryPath,
 		Name:            "rancher-turtles-local",
-		Path:            fmt.Sprintf("http://%s:%d", addr, port.NodePort),
+		Path:            path,
 		Commands:        opframework.Commands(opframework.Repo, opframework.Add),
 		AdditionalFlags: opframework.Flags("--force-update"),
 		Kubeconfig:      input.Proxy.GetKubeconfigPath(),
 	}
-	_, err := addChart.Run(nil)
-	Expect(err).ToNot(HaveOccurred())
+
+	Eventually(func() error {
+		_, err := addChart.Run(nil)
+
+		return err
+	}, input.WaitInterval...).Should(Succeed(), "Failed to connect to workload cluster using CAPI kubeconfig")
 
 	By("Pushing local chart to chartmuseum")
 	exec.Command(

test/testenv/charmuseum.go

@@ -24,6 +24,7 @@ import (
 
 	"github.com/rancher/turtles/test/e2e"
 	"sigs.k8s.io/cluster-api/test/framework"
+	"sigs.k8s.io/cluster-api/test/framework/clusterctl"
 
 	turtlesframework "github.com/rancher/turtles/test/framework"
 )
@@ -44,6 +45,12 @@ type DeployChartMuseumInput struct {
 
 	// WaitInterval is the interval to wait for.
 	WaitInterval []interface{}
+
+	// CustomIngressConfig is the custom ingress configuration.
+	CustomIngressConfig []byte
+
+	// Variables is the collection of variables.
+	Variables turtlesframework.VariableCollection
 }
 
 // DeployChartMuseum installs ChartMuseum to the Kubernetes cluster using the provided input parameters.
@@ -66,5 +73,17 @@ func DeployChartMuseum(ctx context.Context, input DeployChartMuseumInput) {
 		DeploymentName:       "chartmuseum",
 		ServiceName:          "chartmuseum-service",
 		PortName:             "http",
+		CustomIngressConfig:  input.CustomIngressConfig,
+		Variables:            input.Variables,
 	})
 }
+
+// PreChartMuseumInstallHook is a pre-install hook for ChartMuseum.
+func PreChartMuseumInstallHook(chartMuseumInput *DeployChartMuseumInput, e2eConfig *clusterctl.E2EConfig) {
+	infrastructureType := e2e.ManagementClusterEnvironmentType(e2eConfig.GetVariable(e2e.ManagementClusterEnvironmentVar))
+
+	switch infrastructureType {
+	case e2e.ManagementClusterEnvironmentKind:
+		chartMuseumInput.CustomIngressConfig = e2e.ChartMuseumIngress
+	}
+}