Add md_remediation test

Signed-off-by: Mohammed Boukhalfa <[email protected]>

Makefile

@@ -171,6 +171,8 @@ cluster-templates: $(KUSTOMIZE) ## Generate cluster templates
 	$(KUSTOMIZE) build $(E2E_TEMPLATES_DIR)/cluster-template-ubuntu > $(E2E_OUT_DIR)/cluster-template-ubuntu.yaml
 	$(KUSTOMIZE) build $(E2E_TEMPLATES_DIR)/cluster-template-centos > $(E2E_OUT_DIR)/cluster-template-centos.yaml
 	$(KUSTOMIZE) build $(E2E_TEMPLATES_DIR)/cluster-template-upgrade-workload > $(E2E_OUT_DIR)/cluster-template-upgrade-workload.yaml
+	$(KUSTOMIZE) build $(E2E_TEMPLATES_DIR)/cluster-template-centos-md-remediation > $(E2E_OUT_DIR)/cluster-template-centos-md-remediation.yaml
+	$(KUSTOMIZE) build $(E2E_TEMPLATES_DIR)/cluster-template-ubuntu-md-remediation > $(E2E_OUT_DIR)/cluster-template-ubuntu-md-remediation.yaml
 	touch $(E2E_OUT_DIR)/clusterclass.yaml
 
 .PHONY: clusterclass-templates

test/e2e/config/e2e_conf.yaml

@@ -117,6 +117,10 @@ providers:
       targetName: "cluster-template-centos.yaml"
     - sourcePath: "../_out/clusterclass.yaml"
       targetName: "clusterclass-test-clusterclass.yaml"
+    - sourcePath: "../_out/cluster-template-centos-md-remediation.yaml"
+      targetName: "cluster-template-centos-md-remediation.yaml"
+    - sourcePath: "../_out/cluster-template-ubuntu-md-remediation.yaml"
+      targetName: "cluster-template-ubuntu-md-remediation.yaml"
 
 variables:
   CNI: "/tmp/calico.yaml"

test/e2e/data/infrastructure-metal3/cluster-template-centos-md-remediation/kustomization.yaml

@@ -0,0 +1,7 @@
+resources:
+- ../bases/ippool
+- ../bases/centos-kubeadm-config
+
+- mhc.yaml
+patchesStrategicMerge:
+- md.yaml

test/e2e/data/infrastructure-metal3/cluster-template-centos-md-remediation/md.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: cluster.x-k8s.io/${CAPI_VERSION}
+kind: MachineDeployment
+metadata:
+  labels:
+    cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME}
+    nodepool: nodepool-0
+  name: ${CLUSTER_NAME}
+  namespace: ${NAMESPACE}
+spec:
+  template:
+    metadata:
+      labels:
+        "e2e.remediation.label": ""

test/e2e/data/infrastructure-metal3/cluster-template-centos-md-remediation/mhc.yaml

@@ -0,0 +1,19 @@
+---
+# MachineHealthCheck object with
+# - a selector that targets all the machines with label e2e.remediation.label=""
+# - unhealthyConditions triggering remediation after 10s the condition is set
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: MachineHealthCheck
+metadata:
+  name: "${CLUSTER_NAME}-mhc-0"
+  namespace: ${NAMESPACE}
+spec:
+  clusterName: "${CLUSTER_NAME}"
+  maxUnhealthy: 100%
+  selector:
+    matchLabels:
+      e2e.remediation.label: ""
+  unhealthyConditions:
+  - type: e2e.remediation.condition
+    status: "False"
+    timeout: 10s

test/e2e/data/infrastructure-metal3/cluster-template-ubuntu-md-remediation/kustomization.yaml

@@ -0,0 +1,7 @@
+resources:
+- ../bases/ippool
+- ../bases/ubuntu-kubeadm-config
+
+- mhc.yaml
+patchesStrategicMerge:
+- md.yaml

test/e2e/data/infrastructure-metal3/cluster-template-ubuntu-md-remediation/md.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: cluster.x-k8s.io/${CAPI_VERSION}
+kind: MachineDeployment
+metadata:
+  labels:
+    cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME}
+    nodepool: nodepool-0
+  name: ${CLUSTER_NAME}
+  namespace: ${NAMESPACE}
+spec:
+  template:
+    metadata:
+      labels:
+        "e2e.remediation.label": ""

test/e2e/data/infrastructure-metal3/cluster-template-ubuntu-md-remediation/mhc.yaml

@@ -0,0 +1,19 @@
+---
+# MachineHealthCheck object with
+# - a selector that targets all the machines with label e2e.remediation.label=""
+# - unhealthyConditions triggering remediation after 10s the condition is set
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: MachineHealthCheck
+metadata:
+  name: "${CLUSTER_NAME}-mhc-0"
+  namespace: ${NAMESPACE}
+spec:
+  clusterName: "${CLUSTER_NAME}"
+  maxUnhealthy: 100%
+  selector:
+    matchLabels:
+      e2e.remediation.label: ""
+  unhealthyConditions:
+  - type: e2e.remediation.condition
+    status: "False"
+    timeout: 10s

test/e2e/md_remediations_test.go

@@ -0,0 +1,56 @@
+package e2e
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	"k8s.io/utils/ptr"
+	bmov1alpha1 "github.com/metal3-io/baremetal-operator/apis/metal3.io/v1alpha1"
+	capi_e2e "sigs.k8s.io/cluster-api/test/e2e"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+var _ = Describe("When testing MachineDeployment remediation [healthcheck] [remediation] [features]", Label("healthcheck", "remediation", "features"), func() {
+	BeforeEach(func() {
+		osType := strings.ToLower(os.Getenv("OS"))
+		Expect(osType).ToNot(Equal(""))
+		validateGlobals(specName)
+		// We need to override clusterctl apply log folder to avoid getting our credentials exposed.
+		clusterctlLogFolder = filepath.Join(os.TempDir(), "clusters", bootstrapClusterProxy.GetName())
+	})
+	capi_e2e.MachineDeploymentRemediationSpec(ctx, func() capi_e2e.MachineDeploymentRemediationSpecInput {
+		return capi_e2e.MachineDeploymentRemediationSpecInput{
+			E2EConfig:             e2eConfig,
+			ClusterctlConfigPath:  clusterctlConfigPath,
+			BootstrapClusterProxy: bootstrapClusterProxy,
+			ArtifactFolder:        artifactFolder,
+			SkipCleanup:           skipCleanup,
+			PostNamespaceCreated:  postNamespaceCreated,
+			Flavor:                ptr.To(fmt.Sprintf("%s-md-remediation", osType)),
+		}
+	})
+	AfterEach(func() {
+		// Recreate bmh that was used in capi namespace in metal3
+		//#nosec G204 -- We need to pass in the file name here.
+		cmd := exec.Command("bash", "-c", "kubectl apply -f bmhosts_crs.yaml  -n metal3")
+		cmd.Dir = workDir
+		output, err := cmd.CombinedOutput()
+		Logf("Applying bmh to metal3 namespace : \n %v", string(output))
+		Expect(err).ToNot(HaveOccurred())
+		// wait for all bmh to become available
+		bootstrapClient := bootstrapClusterProxy.GetClient()
+		ListBareMetalHosts(ctx, bootstrapClient, client.InNamespace(namespace))
+		WaitForNumBmhInState(ctx, bmov1alpha1.StateAvailable, WaitForNumInput{
+			Client:    bootstrapClient,
+			Options:   []client.ListOption{client.InNamespace(namespace)},
+			Replicas:  4,
+			Intervals: e2eConfig.GetIntervals(specName, "wait-bmh-available"),
+		})
+		ListBareMetalHosts(ctx, bootstrapClient, client.InNamespace(namespace))
+	})
+})