Merge pull request #200 from Kuadrant/unify_e2e_test_suite

Unify e2e test suite
Kuadrant · Jul 31, 2024 · d11aea1 · d11aea1
2 parents 0635362 + 2ba6aca
commit d11aea1
Show file tree

Hide file tree

Showing 14 changed files with 829 additions and 1,657 deletions.
diff --git a/.github/workflows/ci-e2e.yaml b/.github/workflows/ci-e2e.yaml
@@ -57,23 +57,17 @@ jobs:
       - name: Run suite AWS
         run: |
           export TEST_DNS_MANAGED_ZONE_NAME=dev-mz-aws
-          export TEST_DNS_ZONE_DOMAIN_NAME=e2e.hcpapps.net
-          export TEST_DNS_NAMESPACE=${{ env.TEST_NAMESPACE }}
-          export TEST_DNS_PROVIDER=aws
+          export TEST_DNS_NAMESPACES=${{ env.TEST_NAMESPACE }}
           make test-e2e
       - name: Run suite GCP
         run: |
           export TEST_DNS_MANAGED_ZONE_NAME=dev-mz-gcp
-          export TEST_DNS_ZONE_DOMAIN_NAME=e2e.google.hcpapps.net
-          export TEST_DNS_NAMESPACE=${{ env.TEST_NAMESPACE }}
-          export TEST_DNS_PROVIDER=gcp
+          export TEST_DNS_NAMESPACES=${{ env.TEST_NAMESPACE }}
           make test-e2e
       - name: Run suite Azure
         run: |
           export TEST_DNS_MANAGED_ZONE_NAME=dev-mz-azure
-          export TEST_DNS_ZONE_DOMAIN_NAME=e2e.azure.hcpapps.net
-          export TEST_DNS_NAMESPACE=${{ env.TEST_NAMESPACE }}
-          export TEST_DNS_PROVIDER=azure
+          export TEST_DNS_NAMESPACES=${{ env.TEST_NAMESPACE }}
           make test-e2e
       - name: Dump Controller logs
         if: ${{ failure() }}

diff --git a/Makefile b/Makefile
@@ -157,13 +157,13 @@ test-e2e: ginkgo
 
 .PHONY: test-e2e-multi
 test-e2e-multi: ginkgo
-	$(GINKGO) -tags=e2e_multi_instance -v ./test/e2e/multi_instance
+	$(GINKGO) -tags=e2e -v --label-filter=multi_record -v ./test/e2e
 
-.PHONY: local-setup
-local-setup: DEPLOY=false
-local-setup: TEST_NAMESPACE=dnstest
-local-setup: DEPLOYMENT_SCOPE=cluster
-local-setup: $(KIND) ## Setup local development kind cluster, dependencies and optionally deploy the dns operator DEPLOY=false|true
+.PHONY: local-setup-cluster
+local-setup-cluster: DEPLOY=false
+local-setup-cluster: TEST_NAMESPACE=dnstest
+local-setup-cluster: DEPLOYMENT_SCOPE=cluster
+local-setup-cluster: $(KIND) ## Setup local development kind cluster, dependencies and optionally deploy the dns operator DEPLOY=false|true
 	@echo "local-setup: KIND_CLUSTER_NAME=${KIND_CLUSTER_NAME} DEPLOY=${DEPLOY} DEPLOYMENT_SCOPE=${DEPLOYMENT_SCOPE} TEST_NAMESPACE=${TEST_NAMESPACE}"
 	@$(MAKE) -s kind-delete-cluster
 	@$(MAKE) -s kind-create-cluster
@@ -188,18 +188,18 @@ local-setup: $(KIND) ## Setup local development kind cluster, dependencies and o
 	$(KUBECTL) get managedzones -A
 	@echo "local-setup: Complete!!"
 
-.PHONY: local-setup-multi
-local-setup-multi: CLUSTER_COUNT=1
-local-setup-multi: ## Setup multiple local development kind clusters
+.PHONY: local-setup
+local-setup: CLUSTER_COUNT=1
+local-setup: ## Setup local development kind cluster(s)
 	@n=1 ; while [[ $$n -le $(CLUSTER_COUNT) ]] ; do \
-		$(MAKE) -s local-setup KIND_CLUSTER_NAME=${KIND_CLUSTER_NAME_PREFIX}-$$n;\
+		$(MAKE) -s local-setup-cluster KIND_CLUSTER_NAME=${KIND_CLUSTER_NAME_PREFIX}-$$n;\
 		((n = n + 1)) ;\
 	done ;\
 
 .PHONY: local-cleanup
-local-cleanup: ## Delete local cluster
-	$(MAKE) kind-delete-cluster
-	$(MAKE) remove-cluster-overlay
+local-cleanup: ## Delete local clusters
+	$(MAKE) kind-delete-all-clusters
+	$(MAKE) remove-all-cluster-overlays
 
 .PHONY: local-deploy
 local-deploy: docker-build kind-load-image ## Deploy the dns operator into local kind cluster from the current code

diff --git a/README.md b/README.md
@@ -84,15 +84,13 @@ kubectl logs -f deployments/dns-operator-controller-manager -n dns-operator-syst
 The e2e test suite can be executed against any cluster running the DNS Operator with configuration added for any supported provider.
 
 ```
-make test-e2e TEST_DNS_MANAGED_ZONE_NAME=<My managed zone name> TEST_DNS_ZONE_DOMAIN_NAME=<My domain name> TEST_DNS_NAMESPACE=<My test namesapace> TEST_DNS_PROVIDER=<aws|gcp>
+make test-e2e TEST_DNS_MANAGED_ZONE_NAME=<My managed zone name> TEST_DNS_NAMESPACES=<My test namespace(s)>
 ```
 
-| Environment Variable       | Description                                                                                                                                            |
-|----------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------|
-| TEST_DNS_MANAGED_ZONE_NAME | Name of the managed zone relevant for the test domain (TEST_DNS_ZONE_DOMAIN_NAME). If using local-setup Managed zones, one of [dev-mz-aws; dev-mz-gcp] | 
-| TEST_DNS_ZONE_DOMAIN_NAME  | Domain name being used for the test, must match the domain of the managed zone (TEST_DNS_MANAGED_ZONE_NAME)                                            | 
-| TEST_DNS_NAMESPACE         | The namespace to run the test in, must be the same namespace as the TEST_DNS_MANAGED_ZONE_NAME                                                         | 
-| TEST_DNS_PROVIDER          | DNS Provider currently being tested, one of [aws; gcp]                                                                                                 | 
+| Environment Variable       | Description                                                                                          |
+|----------------------------|------------------------------------------------------------------------------------------------------|
+| TEST_DNS_MANAGED_ZONE_NAME | Name of the managed zone to use. If using local-setup Managed zones, one of [dev-mz-aws; dev-mz-gcp] | 
+| TEST_DNS_NAMESPACES        | The namespace(s) where the managed zone with the name (TEST_DNS_MANAGED_ZONE_NAME) can be found      | 
 
 ### Modifying the API definitions
 If you are editing the API definitions, generate the manifests such as CRs or CRDs using:

diff --git a/make/kind.mk b/make/kind.mk
@@ -14,6 +14,10 @@ kind-create-cluster: kind ## Create the "kuadrant-dns-local" kind cluster.
 kind-delete-cluster: kind ## Delete the "kuadrant-dns-local" kind cluster.
 	- $(KIND) delete cluster --name $(KIND_CLUSTER_NAME)
 
+.PHONY: kind-delete-all-clusters
+kind-delete-all-clusters: kind ## Delete the all "kuadrant-dns-local*" kind clusters.
+	- $(KIND) get clusters | grep $(KIND_CLUSTER_NAME_PREFIX) | xargs -I % sh -c "kind delete cluster --name %"
+
 .PHONY: kind-load-image
 kind-load-image: kind ## Load image to "kuadrant-dns-local" kind cluster.
 	$(KIND) load docker-image $(IMG) --name $(KIND_CLUSTER_NAME)
diff --git a/make/kustomize_overlays.mk b/make/kustomize_overlays.mk
@@ -26,6 +26,10 @@ generate-cluster-overlay: remove-cluster-overlay ## Generate a cluster overlay w
 remove-cluster-overlay: ## Remove an existing cluster overlay for the current cluster (CLUSTER_NAME)
 	rm -rf $(CLUSTER_OVERLAY_DIR)/$(CLUSTER_NAME)
 
+.PHONY: remove-all-cluster-overlays
+remove-all-cluster-overlays: ## Remove all existing cluster overlays (kuadrant-dns-local*)
+	rm -rf $(CLUSTER_OVERLAY_DIR)/kuadrant-dns-local*
+
 .PHONY: generate-operator-deployment-overlay
 generate-operator-deployment-overlay: ## Generate a DNS Operator deployment overlay for the current cluster (CLUSTER_NAME)
 	# Generate dns-operator deployment overlay

diff --git a/test/e2e/multi_instance/README.md → test/e2e/README.md b/test/e2e/multi_instance/README.md → test/e2e/README.md
@@ -1,11 +1,20 @@
-# Multi Instance Test Suite
+# E2E Test Suite
 
-The multi instance test suite intended to test scenarios where multiple instances of the dns operator are running, each reconciling DNSRecord resources contributing to the same shared dns zone. 
+The e2e test suite is used to test common scenarios in each supported dns provider. The suite contains tests for single instance tests (single_record) where only a single running controller are expected as well as multi instance tests (multi_record) that are intended to test scenarios where multiple instances of the dns operator are running, each reconciling DNSRecord resources contributing to the same shared dns zone. 
 The suite allows runtime configuration to alter the number of instances that are under test allowing stress testing scenarios to be executed using more extreme numbers of instances and records.
 
 ## Local Setup
 
-Deploy the operator on a local kind cluster with X operator instances (DEPLOYMENT_COUNT):
+### Cluster scoped on single cluster
+
+Deploy the operator on a single kind cluster with one operator instance watching all namespaces:
+```shell
+make local-setup DEPLOY=true
+```
+
+### Namespace scoped on single cluster
+
+Deploy the operator on a single kind cluster with two operator instances in two namespaces watching their own namespace only:
 ```shell
 make local-setup DEPLOY=true DEPLOYMENT_SCOPE=namespace DEPLOYMENT_COUNT=2
 ```
@@ -32,9 +41,40 @@ dnstest          dev-mz-aws   mn.hcpapps.net
 dnstest          dev-mz-gcp   mn.google.hcpapps.net
 ```
 
+### Cluster scoped on multiple clusters
+
+Deploy the operator on two kind clusters each with one operator instance watching all namespaces:
+```shell
+make local-setup DEPLOY=true CLUSTER_COUNT=2
+```
+
+### Namespace scoped on multiple clusters
+
+Deploy the operator on two local kind clusters with two operator instances in two namespaces watching their own namespace only:
+```shell
+make local-setup DEPLOY=true DEPLOYMENT_SCOPE=namespace DEPLOYMENT_COUNT=2 CLUSTER_COUNT=2
+```
+
 ## Run the test suite
+
+### Cluster scoped on single cluster
+```shell
+make test-e2e TEST_DNS_MANAGED_ZONE_NAME=dev-mz-aws TEST_DNS_NAMESPACES=dnstest
+```
+
+### Namespace scoped on single cluster
+```shell
+make test-e2e TEST_DNS_MANAGED_ZONE_NAME=dev-mz-aws TEST_DNS_NAMESPACES=dns-operator DEPLOYMENT_COUNT=2
+```
+
+### Cluster scoped on multiple clusters
+```shell
+make test-e2e TEST_DNS_MANAGED_ZONE_NAME=dev-mz-aws TEST_DNS_NAMESPACES=dnstest TEST_DNS_CLUSTER_CONTEXTS=kind-kuadrant-dns-local CLUSTER_COUNT=2
+```
+
+### Namespace scoped on multiple clusters
 ```shell
-make test-e2e-multi TEST_DNS_MANAGED_ZONE_NAME=dev-mz-aws TEST_DNS_NAMESPACES=dns-operator DEPLOYMENT_COUNT=2
+make test-e2e TEST_DNS_MANAGED_ZONE_NAME=dev-mz-aws TEST_DNS_NAMESPACES=dns-operator DEPLOYMENT_COUNT=2 TEST_DNS_CLUSTER_CONTEXTS=kind-kuadrant-dns-local CLUSTER_COUNT=2
 ```
 
 ## Tailing operator pod logs

diff --git a/test/e2e/healthcheck_test.go b/test/e2e/healthcheck_test.go
@@ -9,6 +9,7 @@ import (
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
+	. "github.com/onsi/gomega/gstruct"
 
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -28,15 +29,20 @@ var _ = Describe("Health Check Test", Serial, Labels{"health_checks"}, func() {
 	// testHostname generated hostname for this test e.g. t-gw-mgc-12345.t-e2e-12345.e2e.hcpapps.net
 	var testHostname string
 
+	var k8sClient client.Client
+	var testManagedZone *v1alpha1.ManagedZone
+
 	var dnsRecord *v1alpha1.DNSRecord
 
 	BeforeEach(func() {
 		testID = "t-health-" + GenerateName()
 		testDomainName = strings.Join([]string{testSuiteID, testZoneDomainName}, ".")
 		testHostname = strings.Join([]string{testID, testDomainName}, ".")
+		k8sClient = testClusters[0].k8sClient
+		testManagedZone = testClusters[0].testManagedZones[0]
 		SetTestEnv("testID", testID)
 		SetTestEnv("testHostname", testHostname)
-		SetTestEnv("testNamespace", testNamespace)
+		SetTestEnv("testNamespace", testManagedZone.Namespace)
 	})
 
 	AfterEach(func(ctx SpecContext) {
@@ -61,14 +67,26 @@ var _ = Describe("Health Check Test", Serial, Labels{"health_checks"}, func() {
 			provider, err := ProviderForManagedZone(ctx, testManagedZone, k8sClient)
 			Expect(err).To(BeNil())
 
-			By("creating a DNS Record")
 			dnsRecord = &v1alpha1.DNSRecord{}
 			err = ResourceFromFile("./fixtures/healthcheck_test/geo-dnsrecord-healthchecks.yaml", dnsRecord, GetTestEnv)
 			Expect(err).ToNot(HaveOccurred())
 
+			By("creating dnsrecord " + dnsRecord.Name)
 			err = k8sClient.Create(ctx, dnsRecord)
 			Expect(err).ToNot(HaveOccurred())
 
+			By("checking " + dnsRecord.Name + " becomes ready")
+			Eventually(func(g Gomega) {
+				err := k8sClient.Get(ctx, client.ObjectKeyFromObject(dnsRecord), dnsRecord)
+				g.Expect(err).NotTo(HaveOccurred())
+				g.Expect(dnsRecord.Status.Conditions).To(
+					ContainElement(MatchFields(IgnoreExtras, Fields{
+						"Type":   Equal(string(v1alpha1.ConditionTypeReady)),
+						"Status": Equal(metav1.ConditionTrue),
+					})),
+				)
+			}, time.Minute, 10*time.Second, ctx).Should(Succeed())
+
 			By("Confirming the DNS Record status")
 			Eventually(func(g Gomega) {
 				err := k8sClient.Get(ctx, client.ObjectKeyFromObject(dnsRecord), dnsRecord)
@@ -203,7 +221,7 @@ var _ = Describe("Health Check Test", Serial, Labels{"health_checks"}, func() {
 
 				err = k8sClient.Get(ctx, client.ObjectKeyFromObject(dnsRecord), dnsRecord)
 				g.Expect(errors.IsNotFound(err)).Should(BeTrue())
-			}, TestTimeoutMedium, time.Second).Should(Succeed())
+			}, TestTimeoutLong, time.Second).Should(Succeed())
 
 			By("confirming the health checks were removed in the provider")
 			Eventually(func(g Gomega) {

diff --git a/test/e2e/helpers/common.go b/test/e2e/helpers/common.go
@@ -22,10 +22,6 @@ import (
 	"github.com/kuadrant/dns-operator/internal/provider"
 )
 
-var (
-	SupportedProviders = []string{"aws", "gcp", "azure"}
-)
-
 const (
 	TestTimeoutMedium = 10 * time.Second
 	TestTimeoutLong   = 60 * time.Second