Add etcd snapshot restore e2e setup (#854)

* Configure helm chart for etcd snapshot restore manifests Signed-off-by: Alexandr Demicev <[email protected]> * Add etcd snapshot restore e2e setup Signed-off-by: Alexandr Demicev <[email protected]> --------- Signed-off-by: Alexandr Demicev <[email protected]>
rancher · Nov 27, 2024 · e9ba820 · e9ba820
1 parent aae4302
commit e9ba820
Show file tree

Hide file tree

Showing 8 changed files with 347 additions and 23 deletions.
diff --git a/.github/workflows/e2e-long.yaml b/.github/workflows/e2e-long.yaml
@@ -62,3 +62,12 @@ jobs:
       artifact_name: artifacts_embedded_capi_v3
       MANAGEMENT_CLUSTER_ENVIRONMENT: eks
     secrets: inherit
+  e2e_etcd_snapshot_restore:
+    uses: ./.github/workflows/run-e2e-suite.yaml
+    with:
+      test_suite: test/e2e/suites/etcd-snapshot-restore
+      test_name: Etcd snapshot restore
+      run_azure_janitor: false
+      artifact_name: artifacts_etcd_snapshot_restore
+      MANAGEMENT_CLUSTER_ENVIRONMENT: eks
+    secrets: inherit
diff --git a/Makefile b/Makefile
@@ -585,9 +585,15 @@ build-chart: $(HELM) $(KUSTOMIZE) $(RELEASE_DIR) $(CHART_RELEASE_DIR) $(CHART_PA
 	$(KUSTOMIZE) build ./exp/etcdrestore/config/default > $(CHART_DIR)/templates/rancher-turtles-exp-etcdrestore-components.yaml
 	./scripts/process-exp-etcdrestore-manifests.sh $(CHART_DIR)/templates/rancher-turtles-exp-etcdrestore-components.yaml
 	cp -rf $(CHART_DIR)/* $(CHART_RELEASE_DIR)
+
 	sed -i'' -e 's@image: .*@image: '"$(CONTROLLER_IMG)"'@' $(CHART_RELEASE_DIR)/values.yaml
 	sed -i'' -e 's@imageVersion: .*@imageVersion: '"$(RELEASE_TAG)"'@' $(CHART_RELEASE_DIR)/values.yaml
 	sed -i'' -e 's@imagePullPolicy: .*@imagePullPolicy: '"$(PULL_POLICY)"'@' $(CHART_RELEASE_DIR)/values.yaml
+
+	sed -i'' -e '/etcd-snapshot-restore:/,/image:/ s@image: .*@image: '"$(ETCDRESTORE_IMG)"'@' $(CHART_RELEASE_DIR)/values.yaml
+	sed -i'' -e '/etcd-snapshot-restore:/,/imageVersion:/ s@imageVersion: .*@imageVersion: '"$(RELEASE_TAG)"'@' $(CHART_RELEASE_DIR)/values.yaml
+	sed -i'' -e '/etcd-snapshot-restore:/,/imagePullPolicy:/ s@imagePullPolicy: .*@imagePullPolicy: '"$(PULL_POLICY)"'@' $(CHART_RELEASE_DIR)/values.yaml
+
 	cd $(CHART_RELEASE_DIR) && $(HELM) dependency update
 	$(HELM) package $(CHART_RELEASE_DIR) --app-version=$(HELM_CHART_TAG) --version=$(HELM_CHART_TAG) --destination=$(CHART_PACKAGE_DIR)
 
@@ -621,12 +627,17 @@ test-e2e-push-image: $(GINKGO) $(HELM) $(CLUSTERCTL) kubectl e2e-image-push
 .PHONY: e2e-image
 e2e-image: $(CACHE_DIR) ## Build the image for e2e tests
 	ADDITIONAL_COMMANDS=$(CACHE_COMMANDS) TAG=v0.0.1 CONTROLLER_IMAGE_NAME=turtles-e2e $(MAKE) docker-build
-	RELEASE_TAG=v0.0.1 CONTROLLER_IMG=$(REGISTRY)/$(ORG)/turtles-e2e CONTROLLER_IMAGE_VERSION=v0.0.1 $(MAKE) build-chart
+	ADDITIONAL_COMMANDS=$(CACHE_COMMANDS) TAG=v0.0.1 ETCDRESTORE_IMAGE_NAME=turtles-etcd-snapshot-restore-e2e $(MAKE) docker-build-etcdrestore
+	RELEASE_TAG=v0.0.1 CONTROLLER_IMG=$(REGISTRY)/$(ORG)/turtles-e2e ETCDRESTORE_IMG=$(REGISTRY)/$(ORG)/turtles-etcd-snapshot-restore-e2e \
+	CONTROLLER_IMAGE_VERSION=v0.0.1 \
+	$(MAKE) build-chart
 
 .PHONY: e2e-image-push
 e2e-image-push: $(CACHE_DIR) ## Push the image for e2e tests
 	TARGET_PLATFORMS=$(ARCH) TAG=v0.0.1 CONTROLLER_IMAGE_NAME=turtles-e2e \
 	ADDITIONAL_COMMANDS=$(CACHE_COMMANDS) $(MAKE) docker-build-and-push
+	TARGET_PLATFORMS=$(ARCH) TAG=v0.0.1 ETCDRESTORE_IMAGE_NAME=turtles-etcd-snapshot-restore-e2e \
+	ADDITIONAL_COMMANDS=$(CACHE_COMMANDS) $(MAKE) docker-build-and-push-etcdrestore
 
 .PHONY: compile-e2e
 e2e-compile: ## Test e2e compilation

diff --git a/charts/rancher-turtles/templates/rancher-turtles-exp-etcdrestore-components.yaml b/charts/rancher-turtles/templates/rancher-turtles-exp-etcdrestore-components.yaml
@@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    cert-manager.io/inject-ca-from: rancher-turtles-system/rancher-turtles-etcdsnapshotrestore-serving-cert
+    cert-manager.io/inject-ca-from: {{ index .Values "rancherTurtles" "namespace" }}/rancher-turtles-etcdsnapshotrestore-serving-cert
     controller-gen.kubebuilder.io/version: v0.14.0
   labels:
     turtles-capi.cattle.io: etcd-restore
@@ -101,7 +101,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    cert-manager.io/inject-ca-from: rancher-turtles-system/rancher-turtles-etcdsnapshotrestore-serving-cert
+    cert-manager.io/inject-ca-from: {{ index .Values "rancherTurtles" "namespace" }}/rancher-turtles-etcdsnapshotrestore-serving-cert
     controller-gen.kubebuilder.io/version: v0.14.0
   labels:
     turtles-capi.cattle.io: etcd-restore
@@ -220,7 +220,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    cert-manager.io/inject-ca-from: rancher-turtles-system/rancher-turtles-etcdsnapshotrestore-serving-cert
+    cert-manager.io/inject-ca-from: {{ index .Values "rancherTurtles" "namespace" }}/rancher-turtles-etcdsnapshotrestore-serving-cert
     controller-gen.kubebuilder.io/version: v0.14.0
   labels:
     turtles-capi.cattle.io: etcd-restore
@@ -309,7 +309,7 @@ metadata:
     app.kubernetes.io/part-of: rancher-turtles
     turtles-capi.cattle.io: etcd-restore
   name: rancher-turtles-etcdsnapshotrestore-manager
-  namespace: rancher-turtles-system
+  namespace: {{ index .Values "rancherTurtles" "namespace" }}
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
@@ -323,7 +323,7 @@ metadata:
     app.kubernetes.io/part-of: rancher-turtles
     turtles-capi.cattle.io: etcd-restore
   name: rancher-turtles-etcdsnapshotrestore-leader-election-role
-  namespace: rancher-turtles-system
+  namespace: {{ index .Values "rancherTurtles" "namespace" }}
 rules:
 - apiGroups:
   - ""
@@ -572,15 +572,15 @@ metadata:
     app.kubernetes.io/part-of: rancher-turtles
     turtles-capi.cattle.io: etcd-restore
   name: rancher-turtles-etcdsnapshotrestore-leader-election-rolebinding
-  namespace: rancher-turtles-system
+  namespace: {{ index .Values "rancherTurtles" "namespace" }}
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: Role
   name: rancher-turtles-etcdsnapshotrestore-leader-election-role
 subjects:
 - kind: ServiceAccount
   name: rancher-turtles-etcdsnapshotrestore-manager
-  namespace: rancher-turtles-system
+  namespace: {{ index .Values "rancherTurtles" "namespace" }}
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
@@ -601,15 +601,15 @@ roleRef:
 subjects:
 - kind: ServiceAccount
   name: rancher-turtles-etcdsnapshotrestore-manager
-  namespace: rancher-turtles-system
+  namespace: {{ index .Values "rancherTurtles" "namespace" }}
 ---
 apiVersion: v1
 kind: Service
 metadata:
   labels:
     turtles-capi.cattle.io: etcd-restore
   name: rancher-turtles-etcdsnapshotrestore-webhook-service
-  namespace: rancher-turtles-system
+  namespace: {{ index .Values "rancherTurtles" "namespace" }}
 spec:
   ports:
   - port: 443
@@ -624,7 +624,7 @@ metadata:
     control-plane: controller-manager
     turtles-capi.cattle.io: etcd-restore
   name: rancher-turtles-etcdsnapshotrestore-controller-manager
-  namespace: rancher-turtles-system
+  namespace: {{ index .Values "rancherTurtles" "namespace" }}
 spec:
   replicas: 1
   selector:
@@ -657,8 +657,13 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: metadata.uid
-        image: ghcr.io/rancher/turtles-etcd-snapshot-restore:dev
-        imagePullPolicy: IfNotPresent
+        {{- $imageVersion := index .Values "rancherTurtles" "features" "etcd-snapshot-restore" "imageVersion" -}}
+        {{- if contains "sha256:" $imageVersion }}
+        image: {{ index .Values "rancherTurtles" "features" "etcd-snapshot-restore" "image" }}@{{ index .Values "rancherTurtles" "features" "etcd-snapshot-restore" "imageVersion" }}
+        {{- else }}
+        image: {{ index .Values "rancherTurtles" "features" "etcd-snapshot-restore" "image" }}:{{ index .Values "rancherTurtles" "features" "etcd-snapshot-restore" "imageVersion" }}
+        {{- end }}
+        imagePullPolicy: '{{ index .Values "rancherTurtles" "features" "etcd-snapshot-restore" "imagePullPolicy" }}'
         livenessProbe:
           httpGet:
             path: /healthz
@@ -705,11 +710,11 @@ metadata:
   labels:
     turtles-capi.cattle.io: etcd-restore
   name: rancher-turtles-etcdsnapshotrestore-serving-cert
-  namespace: rancher-turtles-system
+  namespace: {{ index .Values "rancherTurtles" "namespace" }}
 spec:
   dnsNames:
-  - rancher-turtles-etcdsnapshotrestore-webhook-service.rancher-turtles-system.svc
-  - rancher-turtles-etcdsnapshotrestore-webhook-service.rancher-turtles-system.svc.cluster.local
+  - rancher-turtles-etcdsnapshotrestore-webhook-service.{{ index .Values "rancherTurtles" "namespace" }}.svc
+  - rancher-turtles-etcdsnapshotrestore-webhook-service.{{ index .Values "rancherTurtles" "namespace" }}.svc.cluster.local
   issuerRef:
     kind: Issuer
     name: rancher-turtles-etcdsnapshotrestore-selfsigned-issuer
@@ -721,15 +726,15 @@ metadata:
   labels:
     turtles-capi.cattle.io: etcd-restore
   name: rancher-turtles-etcdsnapshotrestore-selfsigned-issuer
-  namespace: rancher-turtles-system
+  namespace: {{ index .Values "rancherTurtles" "namespace" }}
 spec:
   selfSigned: {}
 ---
 apiVersion: admissionregistration.k8s.io/v1
 kind: MutatingWebhookConfiguration
 metadata:
   annotations:
-    cert-manager.io/inject-ca-from: rancher-turtles-system/rancher-turtles-etcdsnapshotrestore-serving-cert
+    cert-manager.io/inject-ca-from: {{ index .Values "rancherTurtles" "namespace" }}/rancher-turtles-etcdsnapshotrestore-serving-cert
   labels:
     turtles-capi.cattle.io: etcd-restore
   name: rancher-turtles-etcdsnapshotrestore-mutating-webhook-configuration
@@ -739,7 +744,7 @@ webhooks:
   clientConfig:
     service:
       name: rancher-turtles-etcdsnapshotrestore-webhook-service
-      namespace: rancher-turtles-system
+      namespace: {{ index .Values "rancherTurtles" "namespace" }}
       path: /mutate-bootstrap-cluster-x-k8s-io-v1beta1-rke2config
   failurePolicy: Fail
   name: systemagentrke2config.kb.io
@@ -759,7 +764,7 @@ apiVersion: admissionregistration.k8s.io/v1
 kind: ValidatingWebhookConfiguration
 metadata:
   annotations:
-    cert-manager.io/inject-ca-from: rancher-turtles-system/rancher-turtles-etcdsnapshotrestore-serving-cert
+    cert-manager.io/inject-ca-from: {{ index .Values "rancherTurtles" "namespace" }}/rancher-turtles-etcdsnapshotrestore-serving-cert
   labels:
     turtles-capi.cattle.io: etcd-restore
   name: rancher-turtles-etcdsnapshotrestore-validating-webhook-configuration
@@ -769,7 +774,7 @@ webhooks:
   clientConfig:
     service:
       name: rancher-turtles-etcdsnapshotrestore-webhook-service
-      namespace: rancher-turtles-system
+      namespace: {{ index .Values "rancherTurtles" "namespace" }}
       path: /validate-turtles-capi-cattle-io-v1alpha1-etcdmachinesnapshot
   failurePolicy: Fail
   matchPolicy: Equivalent
@@ -790,7 +795,7 @@ webhooks:
   clientConfig:
     service:
       name: rancher-turtles-etcdsnapshotrestore-webhook-service
-      namespace: rancher-turtles-system
+      namespace: {{ index .Values "rancherTurtles" "namespace" }}
       path: /validate-turtles-capi-cattle-io-v1alpha1-etcdsnapshotrestore
   failurePolicy: Fail
   matchPolicy: Equivalent

diff --git a/charts/rancher-turtles/values.yaml b/charts/rancher-turtles/values.yaml
@@ -25,6 +25,9 @@ rancherTurtles:
       enabled: false
     etcd-snapshot-restore:
       enabled: false
+      image: controller
+      imageVersion: v0.0.0
+      imagePullPolicy: IfNotPresent
     addon-provider-fleet:
       enabled: false
 cluster-api-operator:

diff --git a/scripts/process-exp-etcdrestore-manifests.sh b/scripts/process-exp-etcdrestore-manifests.sh
@@ -8,6 +8,13 @@ fi
 
 filename=$1
 
+# Determine the appropriate sed command
+if [[ "$OSTYPE" == "darwin"* ]]; then
+    sed_cmd="gsed"
+else
+    sed_cmd="sed"
+fi
+
 # Define the content to append at the beginning and end of the file
 start_content="{{- if index .Values \"rancherTurtles\" \"features\" \"etcd-snapshot-restore\" \"enabled\" }}"
 end_content="{{- end }}"
@@ -18,3 +25,23 @@ echo "$start_content" | cat - "$filename" > temp && mv temp "$filename"
 # Append content at the end of the file
 echo "$end_content" >> "$filename"
 
+# 1. Replace "rancher-turtles-system" with the templated namespace
+$sed_cmd -i "s|rancher-turtles-system|{{ index .Values \"rancherTurtles\" \"namespace\" }}|g" "$filename"
+
+# 2. Replace "rancher-turtles-system" with the templated image
+$sed_cmd -i "s|rancher-turtles-system|{{ index .Values \"rancherTurtles\" \"features\" \"etcd-snapshot-restore\" \"image\" }}|g" "$filename"
+
+# 3. Update the "image:" section dynamically based on conditions
+$sed_cmd -i '/image: ghcr.io\/rancher\/turtles-etcd-snapshot-restore:dev/c\
+        {{- $imageVersion := index .Values "rancherTurtles" "features" "etcd-snapshot-restore" "imageVersion" -}}\
+        {{- if contains "sha256:" $imageVersion }}\
+        image: {{ index .Values "rancherTurtles" "features" "etcd-snapshot-restore" "image" }}@{{ index .Values "rancherTurtles" "features" "etcd-snapshot-restore" "imageVersion" }}\
+        {{- else }}\
+        image: {{ index .Values "rancherTurtles" "features" "etcd-snapshot-restore" "image" }}:{{ index .Values "rancherTurtles" "features" "etcd-snapshot-restore" "imageVersion" }}\
+        {{- end }}' "$filename"
+
+# 4. Replace the "imagePullPolicy" dynamically
+$sed_cmd -i "s|imagePullPolicy: IfNotPresent|imagePullPolicy: '{{ index .Values \"rancherTurtles\" \"features\" \"etcd-snapshot-restore\" \"imagePullPolicy\" }}'|g" "$filename"
+
+# Confirmation message
+echo "All replacements applied successfully to $filename"
diff --git a/test/e2e/config/operator.yaml b/test/e2e/config/operator.yaml
@@ -2,7 +2,9 @@ managementClusterName: rancher-turtles-e2e
 
 images:
 # Use local dev images built source tree;
-- name: ghcr.io/rancher/turtles-e2e:v0.0.1 # This should be substituted with operator image
+- name: ghcr.io/rancher/turtles-e2e:v0.0.1
+  loadBehavior: tryLoad
+- name: ghcr.io/rancher/turtles-etcd-snapshot-restore-e2e:v0.0.1
   loadBehavior: tryLoad
 
 intervals:

diff --git a/test/e2e/suites/etcd-snapshot-restore/etcd_snapshot_restore_test.go b/test/e2e/suites/etcd-snapshot-restore/etcd_snapshot_restore_test.go
@@ -0,0 +1,39 @@
+//go:build e2e
+// +build e2e
+
+/*
+Copyright © 2023 - 2024 SUSE LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package etcd_snapshot_restore
+
+import (
+	_ "embed"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "sigs.k8s.io/controller-runtime/pkg/envtest/komega"
+
+	"github.com/rancher/turtles/test/e2e"
+)
+
+var _ = Describe("[Docker] [RKE2] Perform an ETCD backup and restore of the cluster", Label(e2e.ShortTestLabel), func() {
+	BeforeEach(func() {
+		SetClient(setupClusterResult.BootstrapClusterProxy.GetClient())
+		SetContext(ctx)
+	})
+
+	It("should perform an ETCD backup and restore of the cluster", func() {
+	})
+})