From 2dc19df43e67db0adfca54ae056a15ae6b8df06b Mon Sep 17 00:00:00 2001 From: Michael Nairn Date: Tue, 23 Jul 2024 10:41:21 +0100 Subject: [PATCH 1/3] tests: Improve dev/test setup and cleanup * Remove all kuadrant clusters and overlays on cleanup * Doc updates, add more examples for cluster vs namespaced scoped deployments and tests Signed-off-by: Michael Nairn --- Makefile | 6 +++--- make/kind.mk | 4 ++++ make/kustomize_overlays.mk | 4 ++++ test/e2e/multi_instance/README.md | 18 +++++++++++++++++- 4 files changed, 28 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 07767ecb..8a17a318 100644 --- a/Makefile +++ b/Makefile @@ -197,9 +197,9 @@ local-setup-multi: ## Setup multiple local development kind clusters done ;\ .PHONY: local-cleanup -local-cleanup: ## Delete local cluster - $(MAKE) kind-delete-cluster - $(MAKE) remove-cluster-overlay +local-cleanup: ## Delete local clusters + $(MAKE) kind-delete-all-clusters + $(MAKE) remove-all-cluster-overlays .PHONY: local-deploy local-deploy: docker-build kind-load-image ## Deploy the dns operator into local kind cluster from the current code diff --git a/make/kind.mk b/make/kind.mk index a7bfaa87..424623ee 100644 --- a/make/kind.mk +++ b/make/kind.mk @@ -14,6 +14,10 @@ kind-create-cluster: kind ## Create the "kuadrant-dns-local" kind cluster. kind-delete-cluster: kind ## Delete the "kuadrant-dns-local" kind cluster. - $(KIND) delete cluster --name $(KIND_CLUSTER_NAME) +.PHONY: kind-delete-all-clusters +kind-delete-all-clusters: kind ## Delete the all "kuadrant-dns-local*" kind clusters. + - $(KIND) get clusters | grep $(KIND_CLUSTER_NAME_PREFIX) | xargs -I % sh -c "kind delete cluster --name %" + .PHONY: kind-load-image kind-load-image: kind ## Load image to "kuadrant-dns-local" kind cluster. $(KIND) load docker-image $(IMG) --name $(KIND_CLUSTER_NAME) diff --git a/make/kustomize_overlays.mk b/make/kustomize_overlays.mk index 571a6a0a..07c67415 100644 --- a/make/kustomize_overlays.mk +++ b/make/kustomize_overlays.mk @@ -26,6 +26,10 @@ generate-cluster-overlay: remove-cluster-overlay ## Generate a cluster overlay w remove-cluster-overlay: ## Remove an existing cluster overlay for the current cluster (CLUSTER_NAME) rm -rf $(CLUSTER_OVERLAY_DIR)/$(CLUSTER_NAME) +.PHONY: remove-all-cluster-overlays +remove-all-cluster-overlays: ## Remove all existing cluster overlays (kuadrant-dns-local*) + rm -rf $(CLUSTER_OVERLAY_DIR)/kuadrant-dns-local* + .PHONY: generate-operator-deployment-overlay generate-operator-deployment-overlay: ## Generate a DNS Operator deployment overlay for the current cluster (CLUSTER_NAME) # Generate dns-operator deployment overlay diff --git a/test/e2e/multi_instance/README.md b/test/e2e/multi_instance/README.md index 3c35295f..3d59f8d0 100644 --- a/test/e2e/multi_instance/README.md +++ b/test/e2e/multi_instance/README.md @@ -5,6 +5,8 @@ The suite allows runtime configuration to alter the number of instances that are ## Local Setup +### Namespaced on single cluster + Deploy the operator on a local kind cluster with X operator instances (DEPLOYMENT_COUNT): ```shell make local-setup DEPLOY=true DEPLOYMENT_SCOPE=namespace DEPLOYMENT_COUNT=2 @@ -32,9 +34,23 @@ dnstest dev-mz-aws mn.hcpapps.net dnstest dev-mz-gcp mn.google.hcpapps.net ``` +### Namespaced on multiple clusters + +Deploy the operator on N local kind cluster (CLUSTER_COUNT) with X operator instances (DEPLOYMENT_COUNT): +```shell +make local-setup-multi DEPLOY=true DEPLOYMENT_SCOPE=namespace DEPLOYMENT_COUNT=2 CLUSTER_COUNT=2 +``` + ## Run the test suite + +### Namespaced on single cluster +```shell +make test-e2e-multi TEST_DNS_MANAGED_ZONE_NAME=dev-mz-aws TEST_DNS_NAMESPACES=dns-operator DEPLOYMENT_COUNT=2 CLUSTER_COUNT=2 +``` + +### Namespaced on multiple clusters ```shell -make test-e2e-multi TEST_DNS_MANAGED_ZONE_NAME=dev-mz-aws TEST_DNS_NAMESPACES=dns-operator DEPLOYMENT_COUNT=2 +make test-e2e-multi TEST_DNS_MANAGED_ZONE_NAME=dev-mz-aws TEST_DNS_NAMESPACES=dns-operator DEPLOYMENT_COUNT=2 CLUSTER_COUNT=2 ``` ## Tailing operator pod logs From d9bd07b87ee5ec00b7ba8229c037969afd9b35af Mon Sep 17 00:00:00 2001 From: Michael Nairn Date: Thu, 25 Jul 2024 11:11:17 +0100 Subject: [PATCH 2/3] tests: Unify single and multi record e2e tests Combines the e2e and e2e-multi test suites into one unified test suite. Tests for single and multi instance scenarios can still be executed individually if required, but all tests should work in both cases. `make test-e2e` - Runs all e2e tests `make test-e2e-multi` - Runs only e2e tests for multiple record scenarios Signed-off-by: Michael Nairn --- .github/workflows/ci-e2e.yaml | 12 +- Makefile | 2 +- README.md | 12 +- test/e2e/{multi_instance => }/README.md | 44 +- test/e2e/healthcheck_test.go | 24 +- test/e2e/helpers/common.go | 4 - test/e2e/multi_instance/multi_record_test.go | 682 ----------- test/e2e/multi_instance/suite_test.go | 265 ----- test/e2e/multi_record_test.go | 1125 ++++++++---------- test/e2e/provider_errors_test.go | 17 +- test/e2e/single_record_test.go | 24 +- test/e2e/suite_test.go | 238 +++- 12 files changed, 798 insertions(+), 1651 deletions(-) rename test/e2e/{multi_instance => }/README.md (57%) delete mode 100644 test/e2e/multi_instance/multi_record_test.go delete mode 100644 test/e2e/multi_instance/suite_test.go diff --git a/.github/workflows/ci-e2e.yaml b/.github/workflows/ci-e2e.yaml index 3dd4750d..017de05b 100644 --- a/.github/workflows/ci-e2e.yaml +++ b/.github/workflows/ci-e2e.yaml @@ -57,23 +57,17 @@ jobs: - name: Run suite AWS run: | export TEST_DNS_MANAGED_ZONE_NAME=dev-mz-aws - export TEST_DNS_ZONE_DOMAIN_NAME=e2e.hcpapps.net - export TEST_DNS_NAMESPACE=${{ env.TEST_NAMESPACE }} - export TEST_DNS_PROVIDER=aws + export TEST_DNS_NAMESPACES=${{ env.TEST_NAMESPACE }} make test-e2e - name: Run suite GCP run: | export TEST_DNS_MANAGED_ZONE_NAME=dev-mz-gcp - export TEST_DNS_ZONE_DOMAIN_NAME=e2e.google.hcpapps.net - export TEST_DNS_NAMESPACE=${{ env.TEST_NAMESPACE }} - export TEST_DNS_PROVIDER=gcp + export TEST_DNS_NAMESPACES=${{ env.TEST_NAMESPACE }} make test-e2e - name: Run suite Azure run: | export TEST_DNS_MANAGED_ZONE_NAME=dev-mz-azure - export TEST_DNS_ZONE_DOMAIN_NAME=e2e.azure.hcpapps.net - export TEST_DNS_NAMESPACE=${{ env.TEST_NAMESPACE }} - export TEST_DNS_PROVIDER=azure + export TEST_DNS_NAMESPACES=${{ env.TEST_NAMESPACE }} make test-e2e - name: Dump Controller logs if: ${{ failure() }} diff --git a/Makefile b/Makefile index 8a17a318..e9ca4fa2 100644 --- a/Makefile +++ b/Makefile @@ -157,7 +157,7 @@ test-e2e: ginkgo .PHONY: test-e2e-multi test-e2e-multi: ginkgo - $(GINKGO) -tags=e2e_multi_instance -v ./test/e2e/multi_instance + $(GINKGO) -tags=e2e -v --label-filter=multi_record -v ./test/e2e .PHONY: local-setup local-setup: DEPLOY=false diff --git a/README.md b/README.md index 6b24ab3e..7dbe161b 100644 --- a/README.md +++ b/README.md @@ -84,15 +84,13 @@ kubectl logs -f deployments/dns-operator-controller-manager -n dns-operator-syst The e2e test suite can be executed against any cluster running the DNS Operator with configuration added for any supported provider. ``` -make test-e2e TEST_DNS_MANAGED_ZONE_NAME= TEST_DNS_ZONE_DOMAIN_NAME= TEST_DNS_NAMESPACE= TEST_DNS_PROVIDER= +make test-e2e TEST_DNS_MANAGED_ZONE_NAME= TEST_DNS_NAMESPACES= ``` -| Environment Variable | Description | -|----------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------| -| TEST_DNS_MANAGED_ZONE_NAME | Name of the managed zone relevant for the test domain (TEST_DNS_ZONE_DOMAIN_NAME). If using local-setup Managed zones, one of [dev-mz-aws; dev-mz-gcp] | -| TEST_DNS_ZONE_DOMAIN_NAME | Domain name being used for the test, must match the domain of the managed zone (TEST_DNS_MANAGED_ZONE_NAME) | -| TEST_DNS_NAMESPACE | The namespace to run the test in, must be the same namespace as the TEST_DNS_MANAGED_ZONE_NAME | -| TEST_DNS_PROVIDER | DNS Provider currently being tested, one of [aws; gcp] | +| Environment Variable | Description | +|----------------------------|------------------------------------------------------------------------------------------------------| +| TEST_DNS_MANAGED_ZONE_NAME | Name of the managed zone to use. If using local-setup Managed zones, one of [dev-mz-aws; dev-mz-gcp] | +| TEST_DNS_NAMESPACES | The namespace(s) where the managed zone with the name (TEST_DNS_MANAGED_ZONE_NAME) can be found | ### Modifying the API definitions If you are editing the API definitions, generate the manifests such as CRs or CRDs using: diff --git a/test/e2e/multi_instance/README.md b/test/e2e/README.md similarity index 57% rename from test/e2e/multi_instance/README.md rename to test/e2e/README.md index 3d59f8d0..90b4f2b7 100644 --- a/test/e2e/multi_instance/README.md +++ b/test/e2e/README.md @@ -1,13 +1,20 @@ -# Multi Instance Test Suite +# E2E Test Suite -The multi instance test suite intended to test scenarios where multiple instances of the dns operator are running, each reconciling DNSRecord resources contributing to the same shared dns zone. +The e2e test suite is used to test common scenarios in each supported dns provider. The suite contains tests for single instance tests (single_record) where only a single running controller are expected as well as multi instance tests (multi_record) that are intended to test scenarios where multiple instances of the dns operator are running, each reconciling DNSRecord resources contributing to the same shared dns zone. The suite allows runtime configuration to alter the number of instances that are under test allowing stress testing scenarios to be executed using more extreme numbers of instances and records. ## Local Setup -### Namespaced on single cluster +### Cluster scoped on single cluster -Deploy the operator on a local kind cluster with X operator instances (DEPLOYMENT_COUNT): +Deploy the operator on a single kind cluster with one operator instance watching all namespaces: +```shell +make local-setup DEPLOY=true +``` + +### Namespace scoped on single cluster + +Deploy the operator on a single kind cluster with two operator instances in two namespaces watching their own namespace only: ```shell make local-setup DEPLOY=true DEPLOYMENT_SCOPE=namespace DEPLOYMENT_COUNT=2 ``` @@ -34,23 +41,40 @@ dnstest dev-mz-aws mn.hcpapps.net dnstest dev-mz-gcp mn.google.hcpapps.net ``` -### Namespaced on multiple clusters +### Cluster scoped on multiple clusters + +Deploy the operator on two kind cluster each with one operator instance watching all namespaces: +```shell +make local-setup-multi DEPLOY=true CLUSTER_COUNT=2 +``` + +### Namespace scoped on multiple clusters -Deploy the operator on N local kind cluster (CLUSTER_COUNT) with X operator instances (DEPLOYMENT_COUNT): +Deploy the operator on two local kind cluster with two operator instances in two namespaces watching their own namespace only: ```shell make local-setup-multi DEPLOY=true DEPLOYMENT_SCOPE=namespace DEPLOYMENT_COUNT=2 CLUSTER_COUNT=2 ``` ## Run the test suite -### Namespaced on single cluster +### Cluster scoped on single cluster +```shell +make test-e2e TEST_DNS_MANAGED_ZONE_NAME=dev-mz-aws TEST_DNS_NAMESPACES=dnstest +``` + +### Namespace scoped on single cluster +```shell +make test-e2e TEST_DNS_MANAGED_ZONE_NAME=dev-mz-aws TEST_DNS_NAMESPACES=dns-operator DEPLOYMENT_COUNT=2 +``` + +### Cluster scoped on multiple clusters ```shell -make test-e2e-multi TEST_DNS_MANAGED_ZONE_NAME=dev-mz-aws TEST_DNS_NAMESPACES=dns-operator DEPLOYMENT_COUNT=2 CLUSTER_COUNT=2 +make test-e2e TEST_DNS_MANAGED_ZONE_NAME=dev-mz-aws TEST_DNS_NAMESPACES=dnstest TEST_DNS_CLUSTER_CONTEXTS=kind-kuadrant-dns-local CLUSTER_COUNT=2 ``` -### Namespaced on multiple clusters +### Namespace scoped on multiple clusters ```shell -make test-e2e-multi TEST_DNS_MANAGED_ZONE_NAME=dev-mz-aws TEST_DNS_NAMESPACES=dns-operator DEPLOYMENT_COUNT=2 CLUSTER_COUNT=2 +make test-e2e TEST_DNS_MANAGED_ZONE_NAME=dev-mz-aws TEST_DNS_NAMESPACES=dns-operator DEPLOYMENT_COUNT=2 TEST_DNS_CLUSTER_CONTEXTS=kind-kuadrant-dns-local CLUSTER_COUNT=2 ``` ## Tailing operator pod logs diff --git a/test/e2e/healthcheck_test.go b/test/e2e/healthcheck_test.go index 98b21451..14e13b9f 100644 --- a/test/e2e/healthcheck_test.go +++ b/test/e2e/healthcheck_test.go @@ -9,6 +9,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + . "github.com/onsi/gomega/gstruct" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -28,15 +29,20 @@ var _ = Describe("Health Check Test", Serial, Labels{"health_checks"}, func() { // testHostname generated hostname for this test e.g. t-gw-mgc-12345.t-e2e-12345.e2e.hcpapps.net var testHostname string + var k8sClient client.Client + var testManagedZone *v1alpha1.ManagedZone + var dnsRecord *v1alpha1.DNSRecord BeforeEach(func() { testID = "t-health-" + GenerateName() testDomainName = strings.Join([]string{testSuiteID, testZoneDomainName}, ".") testHostname = strings.Join([]string{testID, testDomainName}, ".") + k8sClient = testClusters[0].k8sClient + testManagedZone = testClusters[0].testManagedZones[0] SetTestEnv("testID", testID) SetTestEnv("testHostname", testHostname) - SetTestEnv("testNamespace", testNamespace) + SetTestEnv("testNamespace", testManagedZone.Namespace) }) AfterEach(func(ctx SpecContext) { @@ -61,14 +67,26 @@ var _ = Describe("Health Check Test", Serial, Labels{"health_checks"}, func() { provider, err := ProviderForManagedZone(ctx, testManagedZone, k8sClient) Expect(err).To(BeNil()) - By("creating a DNS Record") dnsRecord = &v1alpha1.DNSRecord{} err = ResourceFromFile("./fixtures/healthcheck_test/geo-dnsrecord-healthchecks.yaml", dnsRecord, GetTestEnv) Expect(err).ToNot(HaveOccurred()) + By("creating dnsrecord " + dnsRecord.Name) err = k8sClient.Create(ctx, dnsRecord) Expect(err).ToNot(HaveOccurred()) + By("checking " + dnsRecord.Name + " becomes ready") + Eventually(func(g Gomega) { + err := k8sClient.Get(ctx, client.ObjectKeyFromObject(dnsRecord), dnsRecord) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(dnsRecord.Status.Conditions).To( + ContainElement(MatchFields(IgnoreExtras, Fields{ + "Type": Equal(string(v1alpha1.ConditionTypeReady)), + "Status": Equal(metav1.ConditionTrue), + })), + ) + }, time.Minute, 10*time.Second, ctx).Should(Succeed()) + By("Confirming the DNS Record status") Eventually(func(g Gomega) { err := k8sClient.Get(ctx, client.ObjectKeyFromObject(dnsRecord), dnsRecord) @@ -203,7 +221,7 @@ var _ = Describe("Health Check Test", Serial, Labels{"health_checks"}, func() { err = k8sClient.Get(ctx, client.ObjectKeyFromObject(dnsRecord), dnsRecord) g.Expect(errors.IsNotFound(err)).Should(BeTrue()) - }, TestTimeoutMedium, time.Second).Should(Succeed()) + }, TestTimeoutLong, time.Second).Should(Succeed()) By("confirming the health checks were removed in the provider") Eventually(func(g Gomega) { diff --git a/test/e2e/helpers/common.go b/test/e2e/helpers/common.go index 9dd2d13b..e9b3ff8b 100644 --- a/test/e2e/helpers/common.go +++ b/test/e2e/helpers/common.go @@ -22,10 +22,6 @@ import ( "github.com/kuadrant/dns-operator/internal/provider" ) -var ( - SupportedProviders = []string{"aws", "gcp", "azure"} -) - const ( TestTimeoutMedium = 10 * time.Second TestTimeoutLong = 60 * time.Second diff --git a/test/e2e/multi_instance/multi_record_test.go b/test/e2e/multi_instance/multi_record_test.go deleted file mode 100644 index 34b0f7c7..00000000 --- a/test/e2e/multi_instance/multi_record_test.go +++ /dev/null @@ -1,682 +0,0 @@ -//go:build e2e_multi_instance - -package multi_instance - -import ( - "context" - "fmt" - "strings" - "time" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - . "github.com/onsi/gomega/gstruct" - "github.com/onsi/gomega/types" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "sigs.k8s.io/controller-runtime/pkg/client" - externaldnsendpoint "sigs.k8s.io/external-dns/endpoint" - - "github.com/kuadrant/dns-operator/api/v1alpha1" - "github.com/kuadrant/dns-operator/internal/provider" - . "github.com/kuadrant/dns-operator/test/e2e/helpers" -) - -// Test Cases covering multiple DNSRecords updating a set of records in a zone -var _ = Describe("Multi Record Test", func() { - // testID is a randomly generated identifier for the test - // it is used to name resources and/or namespaces so different - // tests can be run in parallel in the same cluster - var testID string - // testDomainName generated domain for this test e.g. t-e2e-12345.e2e.hcpapps.net - var testDomainName string - // testHostname generated hostname for this test e.g. t-gw-mgc-12345.t-e2e-12345.e2e.hcpapps.net - var testHostname string - - var geoCode1 string - var geoCode2 string - - var testRecords []*testDNSRecord - - recordsReadyMaxDuration := time.Minute - recordsRemovedMaxDuration := time.Minute - - BeforeEach(func(ctx SpecContext) { - testID = "t-multi-" + GenerateName() - testDomainName = strings.Join([]string{testSuiteID, testZoneDomainName}, ".") - testHostname = strings.Join([]string{testID, testDomainName}, ".") - testRecords = []*testDNSRecord{} - - if testDNSProvider == "google" { - geoCode1 = "us-east1" - geoCode2 = "europe-west1" - } else { - geoCode1 = "US" - geoCode2 = "EU" - } - }) - - AfterEach(func(ctx SpecContext) { - By("ensuring all dns records are deleted") - for _, tr := range testRecords { - err := tr.cluster.k8sClient.Delete(ctx, tr.record, - client.PropagationPolicy(metav1.DeletePropagationForeground)) - Expect(client.IgnoreNotFound(err)).ToNot(HaveOccurred()) - } - - By("checking all dns records are removed") - Eventually(func(g Gomega, ctx context.Context) { - for _, tr := range testRecords { - err := tr.cluster.k8sClient.Get(ctx, client.ObjectKeyFromObject(tr.record), tr.record) - g.Expect(err).To(MatchError(ContainSubstring("not found"))) - } - }, time.Minute, 10*time.Second, ctx).Should(Succeed()) - }) - - Context("simple", func() { - It("creates and deletes distributed dns records", func(ctx SpecContext) { - By(fmt.Sprintf("creating %d simple dnsrecords accross %d clusters", len(testNamespaces)*len(testClusters), len(testClusters))) - for ci, tc := range testClusters { - for mi, mz := range tc.testManagedZones { - config := testConfig{ - testTargetIP: fmt.Sprintf("127.0.%d.%d", ci+1, mi+1), - } - record := &v1alpha1.DNSRecord{ - ObjectMeta: metav1.ObjectMeta{ - Name: testID, - Namespace: mz.Namespace, - }, - Spec: v1alpha1.DNSRecordSpec{ - RootHost: testHostname, - ManagedZoneRef: &v1alpha1.ManagedZoneReference{ - Name: mz.Name, - }, - Endpoints: []*externaldnsendpoint.Endpoint{ - { - DNSName: testHostname, - Targets: []string{ - config.testTargetIP, - }, - RecordType: "A", - RecordTTL: 60, - }, - }, - HealthCheck: nil, - }, - } - - By(fmt.Sprintf("creating dns record [name: `%s`, namespace: `%s`, mz: `%s`, endpoint: [dnsname: `%s`, target: `%s`]] on cluster [name: `%s`]", record.Name, record.Namespace, mz.Name, testHostname, config.testTargetIP, tc.name)) - err := tc.k8sClient.Create(ctx, record) - Expect(err).ToNot(HaveOccurred()) - - testRecords = append(testRecords, &testDNSRecord{ - cluster: &testClusters[ci], - managedZone: mz, - record: record, - config: config, - }) - } - } - - By(fmt.Sprintf("checking all dns records become ready within %s", recordsReadyMaxDuration)) - Eventually(func(g Gomega, ctx context.Context) { - for _, tr := range testRecords { - err := tr.cluster.k8sClient.Get(ctx, client.ObjectKeyFromObject(tr.record), tr.record) - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(tr.record.Status.Conditions).To( - ContainElement(MatchFields(IgnoreExtras, Fields{ - "Type": Equal(string(v1alpha1.ConditionTypeReady)), - "Status": Equal(metav1.ConditionTrue), - })), - ) - } - }, recordsReadyMaxDuration, 5*time.Second, ctx).Should(Succeed()) - - By("checking provider zone records are created as expected") - testProvider, err := ProviderForManagedZone(ctx, testClusters[0].testManagedZones[0], testClusters[0].k8sClient) - Expect(err).NotTo(HaveOccurred()) - - zoneEndpoints, err := EndpointsForHost(ctx, testProvider, testHostname) - Expect(err).NotTo(HaveOccurred()) - Expect(zoneEndpoints).To(HaveLen(2)) - - var allOwners = []string{} - var allTargetIps = []string{} - for i := range testRecords { - allOwners = append(allOwners, testRecords[i].record.Status.OwnerID) - allTargetIps = append(allTargetIps, testRecords[i].config.testTargetIP) - } - - By("checking all target ips are present") - Expect(zoneEndpoints).To(ContainElements( - PointTo(MatchFields(IgnoreExtras, Fields{ - "DNSName": Equal(testHostname), - "Targets": ConsistOf(allTargetIps), - "RecordType": Equal("A"), - "SetIdentifier": Equal(""), - "RecordTTL": Equal(externaldnsendpoint.TTL(60)), - })), - )) - - By("checking all owner references are present") - for _, owner := range allOwners { - Expect(zoneEndpoints).To(ContainElements( - PointTo(MatchFields(IgnoreExtras, Fields{ - "DNSName": Equal("kuadrant-a-" + testHostname), - "Targets": ContainElement(ContainSubstring(owner)), - "RecordType": Equal("TXT"), - "SetIdentifier": Equal(""), - "RecordTTL": Equal(externaldnsendpoint.TTL(300)), - })), - )) - } - - By("checking the authoritative nameserver resolves the hostname") - // speed up things by using the authoritative nameserver - authoritativeResolver := ResolverForDomainName(testZoneDomainName) - Eventually(func(g Gomega, ctx context.Context) { - ips, err := authoritativeResolver.LookupHost(ctx, testHostname) - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(ips).To(ContainElements(allTargetIps)) - }, 300*time.Second, 10*time.Second, ctx).Should(Succeed()) - - //Test Deletion of one of the records - recordToDelete := testRecords[0] - lastRecord := len(testRecords) == 1 - By(fmt.Sprintf("deleting dns record [name: `%s` namespace: `%s`]", recordToDelete.record.Name, recordToDelete.record.Namespace)) - err = recordToDelete.cluster.k8sClient.Delete(ctx, recordToDelete.record, - client.PropagationPolicy(metav1.DeletePropagationForeground)) - Expect(client.IgnoreNotFound(err)).ToNot(HaveOccurred()) - - By(fmt.Sprintf("checking dns record [name: `%s` namespace: `%s`] is removed within %s", recordToDelete.record.Name, recordToDelete.record.Namespace, recordsRemovedMaxDuration)) - Eventually(func(g Gomega, ctx context.Context) { - err := recordToDelete.cluster.k8sClient.Get(ctx, client.ObjectKeyFromObject(recordToDelete.record), recordToDelete.record) - g.Expect(err).To(HaveOccurred()) - g.Expect(err).To(MatchError(ContainSubstring("not found"))) - }, recordsRemovedMaxDuration, 5*time.Second, ctx).Should(Succeed()) - - By("checking provider zone records are updated as expected") - Eventually(func(g Gomega, ctx context.Context) { - zoneEndpoints, err := EndpointsForHost(ctx, testProvider, testHostname) - g.Expect(err).NotTo(HaveOccurred()) - if lastRecord { - g.Expect(zoneEndpoints).To(HaveLen(0)) - } else { - g.Expect(zoneEndpoints).To(HaveLen(2)) - By(fmt.Sprintf("checking ip `%s` and owner `%s` are removed", recordToDelete.config.testTargetIP, recordToDelete.record.Status.OwnerID)) - g.Expect(zoneEndpoints).To(ContainElements( - PointTo(MatchFields(IgnoreExtras, Fields{ - "DNSName": Equal(testHostname), - "Targets": Not(ContainElement(recordToDelete.config.testTargetIP)), - "RecordType": Equal("A"), - "SetIdentifier": Equal(""), - "RecordTTL": Equal(externaldnsendpoint.TTL(60)), - })), - PointTo(MatchFields(IgnoreExtras, Fields{ - "DNSName": Equal("kuadrant-a-" + testHostname), - "Targets": Not(ContainElement(ContainSubstring(recordToDelete.record.Status.OwnerID))), - "RecordType": Equal("TXT"), - "SetIdentifier": Equal(""), - "RecordTTL": Equal(externaldnsendpoint.TTL(300)), - })), - )) - } - }, 5*time.Second, time.Second, ctx).Should(Succeed()) - - By("deleting all remaining dns records") - for _, tr := range testRecords { - err := tr.cluster.k8sClient.Delete(ctx, tr.record, - client.PropagationPolicy(metav1.DeletePropagationForeground)) - Expect(client.IgnoreNotFound(err)).ToNot(HaveOccurred()) - } - - By(fmt.Sprintf("checking all dns records are removed within %s", recordsRemovedMaxDuration)) - Eventually(func(g Gomega, ctx context.Context) { - for _, tr := range testRecords { - err := tr.cluster.k8sClient.Get(ctx, client.ObjectKeyFromObject(tr.record), tr.record) - g.Expect(err).To(MatchError(ContainSubstring("not found"))) - } - }, recordsRemovedMaxDuration, 5*time.Second, ctx).Should(Succeed()) - - By("checking provider zone records are all removed") - Eventually(func(g Gomega, ctx context.Context) { - zoneEndpoints, err := EndpointsForHost(ctx, testProvider, testHostname) - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(zoneEndpoints).To(HaveLen(0)) - }, 5*time.Second, time.Second, ctx).Should(Succeed()) - - }) - - }) - - Context("loadbalanced", func() { - It("creates and deletes distributed dns records", func(ctx SpecContext) { - if testDNSProvider == "azure" { - Skip("not yet supported for azure") - } - testGeoRecords := map[string][]testDNSRecord{} - - By(fmt.Sprintf("creating %d loadbalanced dnsrecords accross %d clusters", len(testNamespaces)*len(testClusters), len(testClusters))) - for ci, tc := range testClusters { - for mi, mz := range tc.testManagedZones { - - var geoCode string - if (ci+mi)%2 == 0 { - geoCode = geoCode1 - } else { - geoCode = geoCode2 - } - - klbHostName := "klb." + testHostname - geoKlbHostName := strings.ToLower(geoCode) + "." + klbHostName - defaultGeoKlbHostName := strings.ToLower(geoCode1) + "." + klbHostName - clusterKlbHostName := fmt.Sprintf("cluster%d-%d.%s", ci+1, mi+1, klbHostName) - - config := testConfig{ - testTargetIP: fmt.Sprintf("127.0.%d.%d", ci+1, mi+1), - testGeoCode: geoCode, - testDefaultGeoCode: geoCode1, - hostnames: testHostnames{ - klb: klbHostName, - geoKlb: geoKlbHostName, - defaultGeoKlb: defaultGeoKlbHostName, - clusterKlb: clusterKlbHostName, - }, - } - - record := &v1alpha1.DNSRecord{ - ObjectMeta: metav1.ObjectMeta{ - Name: testID, - Namespace: mz.Namespace, - }, - Spec: v1alpha1.DNSRecordSpec{ - RootHost: testHostname, - ManagedZoneRef: &v1alpha1.ManagedZoneReference{ - Name: mz.Name, - }, - Endpoints: []*externaldnsendpoint.Endpoint{ - { - DNSName: clusterKlbHostName, - Targets: []string{ - config.testTargetIP, - }, - RecordType: "A", - RecordTTL: 60, - }, - { - DNSName: testHostname, - Targets: []string{ - klbHostName, - }, - RecordType: "CNAME", - RecordTTL: 300, - }, - { - DNSName: geoKlbHostName, - Targets: []string{ - clusterKlbHostName, - }, - RecordType: "CNAME", - RecordTTL: 60, - SetIdentifier: clusterKlbHostName, - ProviderSpecific: externaldnsendpoint.ProviderSpecific{ - { - Name: "weight", - Value: "200", - }, - }, - }, - { - DNSName: klbHostName, - Targets: []string{ - geoKlbHostName, - }, - RecordType: "CNAME", - RecordTTL: 300, - SetIdentifier: config.testGeoCode, - ProviderSpecific: externaldnsendpoint.ProviderSpecific{ - { - Name: "geo-code", - Value: config.testGeoCode, - }, - }, - }, - { - DNSName: klbHostName, - Targets: []string{ - defaultGeoKlbHostName, - }, - RecordType: "CNAME", - RecordTTL: 300, - SetIdentifier: "default", - ProviderSpecific: externaldnsendpoint.ProviderSpecific{ - { - Name: "geo-code", - Value: "*", - }, - }, - }, - }, - HealthCheck: nil, - }, - } - - By(fmt.Sprintf("creating dns record [name: `%s`, namespace: `%s`, managedZone: `%s`, endpoint: [dnsname: `%s`, target: `%s`, geoCode: `%s`]] on cluster [name: `%s`]", record.Name, record.Namespace, mz.Name, testHostname, config.testTargetIP, config.testGeoCode, tc.name)) - err := tc.k8sClient.Create(ctx, record) - Expect(err).ToNot(HaveOccurred()) - tr := &testDNSRecord{ - cluster: &testClusters[ci], - managedZone: mz, - record: record, - config: config, - } - testRecords = append(testRecords, tr) - testGeoRecords[config.testGeoCode] = append(testGeoRecords[config.testGeoCode], *tr) - } - } - - By(fmt.Sprintf("checking all dns records become ready within %s", recordsReadyMaxDuration)) - Eventually(func(g Gomega, ctx context.Context) { - for _, tr := range testRecords { - err := tr.cluster.k8sClient.Get(ctx, client.ObjectKeyFromObject(tr.record), tr.record) - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(tr.record.Status.Conditions).To( - ContainElement(MatchFields(IgnoreExtras, Fields{ - "Type": Equal(string(v1alpha1.ConditionTypeReady)), - "Status": Equal(metav1.ConditionTrue), - })), - ) - } - }, recordsReadyMaxDuration, 5*time.Second, ctx).Should(Succeed()) - - By("checking provider zone records are created as expected") - testProvider, err := ProviderForManagedZone(ctx, testClusters[0].testManagedZones[0], testClusters[0].k8sClient) - Expect(err).NotTo(HaveOccurred()) - - zoneEndpoints, err := EndpointsForHost(ctx, testProvider, testHostname) - Expect(err).NotTo(HaveOccurred()) - var expectedEndpointsLen int - if testDNSProvider == "google" { - expectedEndpointsLen = (2 + len(testGeoRecords) + len(testRecords)) * 2 - Expect(zoneEndpoints).To(HaveLen(expectedEndpointsLen)) - } else if testDNSProvider == "aws" { - expectedEndpointsLen = (2 + len(testGeoRecords) + (len(testRecords) * 2)) * 2 - Expect(zoneEndpoints).To(HaveLen(expectedEndpointsLen)) - } - - var totalEndpointsChecked = 0 - var allOwners = []string{} - var allOwnerMatcher = []types.GomegaMatcher{ - ContainSubstring("heritage=external-dns,external-dns/owner="), - } - var geoOwners = map[string][]string{} - var geoKlbHostname = map[string]string{} - var geoOwnerMatcher = map[string][]types.GomegaMatcher{} - for i := range testRecords { - ownerID := testRecords[i].record.Status.OwnerID - allOwners = append(allOwners, ownerID) - allOwnerMatcher = append(allOwnerMatcher, ContainSubstring(ownerID)) - - geoCode := testRecords[i].config.testGeoCode - geoOwners[geoCode] = append(geoOwners[geoCode], ownerID) - geoKlbHostname[geoCode] = testRecords[i].config.hostnames.geoKlb - if _, ok := geoOwnerMatcher[geoCode]; !ok { - geoOwnerMatcher[geoCode] = []types.GomegaMatcher{ - ContainSubstring("heritage=external-dns,external-dns/owner="), - } - } - geoOwnerMatcher[geoCode] = append(geoOwnerMatcher[geoCode], ContainSubstring(ownerID)) - } - - By("[Common] checking common endpoints") - // A CNAME record for testHostname should always exist and be owned by all endpoints - By("[Common] checking " + testHostname + " endpoint") - Expect(zoneEndpoints).To(ContainElement(PointTo(MatchFields(IgnoreExtras, Fields{ - "DNSName": Equal(testHostname), - "Targets": ConsistOf(testRecords[0].config.hostnames.klb), - "RecordType": Equal("CNAME"), - "SetIdentifier": Equal(""), - "RecordTTL": Equal(externaldnsendpoint.TTL(300)), - })))) - totalEndpointsChecked++ - By("[Common] checking " + testHostname + " TXT owner endpoint") - Expect(zoneEndpoints).To(ContainElement(PointTo(MatchFields(IgnoreExtras, Fields{ - "DNSName": Equal("kuadrant-cname-" + testHostname), - "Targets": ContainElement(And(allOwnerMatcher...)), - "RecordType": Equal("TXT"), - "SetIdentifier": Equal(""), - "RecordTTL": Equal(externaldnsendpoint.TTL(300)), - })))) - totalEndpointsChecked++ - - By("[Geo] checking geo endpoints") - if testDNSProvider == "google" { - // A CNAME record for klbHostName should always exist, be owned by all endpoints and target all geo hostnames - klbHostName := testRecords[0].config.hostnames.klb - - allKlbGeoHostnames := []string{} - gcpGeoProps := []externaldnsendpoint.ProviderSpecificProperty{ - {Name: "routingpolicy", Value: "geo"}, - } - for g, h := range geoKlbHostname { - allKlbGeoHostnames = append(allKlbGeoHostnames, h) - gcpGeoProps = append(gcpGeoProps, externaldnsendpoint.ProviderSpecificProperty{Name: h, Value: g}) - } - - By("[Geo] checking " + klbHostName + " endpoint") - Expect(zoneEndpoints).To(ContainElement(PointTo(MatchFields(IgnoreExtras, Fields{ - "DNSName": Equal(klbHostName), - "Targets": ConsistOf(allKlbGeoHostnames), - "RecordType": Equal("CNAME"), - "SetIdentifier": Equal(""), - "RecordTTL": Equal(externaldnsendpoint.TTL(300)), - "ProviderSpecific": ContainElements(gcpGeoProps), - })))) - totalEndpointsChecked++ - By("[Geo] checking " + klbHostName + " TXT owner endpoint") - Expect(zoneEndpoints).To(ContainElement(PointTo(MatchFields(IgnoreExtras, Fields{ - "DNSName": Equal("kuadrant-cname-" + klbHostName), - "Targets": ContainElement(And(allOwnerMatcher...)), - "RecordType": Equal("TXT"), - "SetIdentifier": Equal(""), - "RecordTTL": Equal(externaldnsendpoint.TTL(300)), - })))) - totalEndpointsChecked++ - } - if testDNSProvider == "aws" { - // A CNAME record for klbHostName should exist for each geo and be owned by all endpoints in that geo - klbHostName := testRecords[0].config.hostnames.klb - for geoCode, geoRecords := range testGeoRecords { - geoKlbHostName := geoRecords[0].config.hostnames.geoKlb - - By("[Geo] checking " + klbHostName + " -> " + geoCode + " -> " + geoKlbHostName + " - endpoint") - - awsGeoCodeKey := "aws/geolocation-country-code" - if !provider.IsISO3166Alpha2Code(geoCode) { - awsGeoCodeKey = "aws/geolocation-continent-code" - } - - Expect(zoneEndpoints).To(ContainElement(PointTo(MatchFields(IgnoreExtras, Fields{ - "DNSName": Equal(klbHostName), - "Targets": ConsistOf(geoKlbHostName), - "RecordType": Equal("CNAME"), - "SetIdentifier": Equal(geoCode), - "RecordTTL": Equal(externaldnsendpoint.TTL(300)), - "ProviderSpecific": Equal(externaldnsendpoint.ProviderSpecific{ - {Name: "alias", Value: "false"}, - {Name: awsGeoCodeKey, Value: geoCode}, - }), - })))) - totalEndpointsChecked++ - By("[Geo] checking " + klbHostName + " -> " + geoCode + " - TXT owner endpoint") - Expect(zoneEndpoints).To(ContainElement(PointTo(MatchFields(IgnoreExtras, Fields{ - "DNSName": Equal("kuadrant-cname-" + klbHostName), - "Targets": ContainElement(And(geoOwnerMatcher[geoCode]...)), - "RecordType": Equal("TXT"), - "SetIdentifier": Equal(geoCode), - "RecordTTL": Equal(externaldnsendpoint.TTL(300)), - "ProviderSpecific": Equal(externaldnsendpoint.ProviderSpecific{ - {Name: awsGeoCodeKey, Value: geoCode}, - }), - })))) - totalEndpointsChecked++ - } - - defaultGeoKlbHostName := testRecords[0].config.hostnames.defaultGeoKlb - defaultGeoCode := testRecords[0].config.testDefaultGeoCode - - By("[Geo] checking endpoint " + klbHostName + " -> default") - Expect(zoneEndpoints).To(ContainElement(PointTo(MatchFields(IgnoreExtras, Fields{ - "DNSName": Equal(klbHostName), - "Targets": ConsistOf(defaultGeoKlbHostName), - "RecordType": Equal("CNAME"), - "SetIdentifier": Equal("default"), - "RecordTTL": Equal(externaldnsendpoint.TTL(300)), - "ProviderSpecific": Equal(externaldnsendpoint.ProviderSpecific{ - {Name: "alias", Value: "false"}, - {Name: "aws/geolocation-country-code", Value: "*"}, - }), - })))) - totalEndpointsChecked++ - By("[Geo] checking " + klbHostName + " -> default - TXT owner endpoint") - Expect(zoneEndpoints).To(ContainElement(PointTo(MatchFields(IgnoreExtras, Fields{ - "DNSName": Equal("kuadrant-cname-" + klbHostName), - "Targets": ContainElement(And(geoOwnerMatcher[defaultGeoCode]...)), - "RecordType": Equal("TXT"), - "SetIdentifier": Equal("default"), - "RecordTTL": Equal(externaldnsendpoint.TTL(300)), - "ProviderSpecific": Equal(externaldnsendpoint.ProviderSpecific{ - {Name: "aws/geolocation-country-code", Value: "*"}, - }), - })))) - totalEndpointsChecked++ - } - - By("[Weight] checking weighted endpoints") - if testDNSProvider == "google" { - // A weighted CNAME record should exist for each geo, be owned by all endpoints in that geo, and target the hostname of all clusters in that geo - for geoCode, geoRecords := range testGeoRecords { - geoKlbHostName := geoRecords[0].config.hostnames.geoKlb - - allGeoClusterHostnames := []string{} - gcpWeightProps := []externaldnsendpoint.ProviderSpecificProperty{ - {Name: "routingpolicy", Value: "weighted"}, - } - for i := range geoRecords { - geoClusterHostname := geoRecords[i].config.hostnames.clusterKlb - allGeoClusterHostnames = append(allGeoClusterHostnames, geoClusterHostname) - gcpWeightProps = append(gcpWeightProps, externaldnsendpoint.ProviderSpecificProperty{Name: geoClusterHostname, Value: "200"}) - } - - By("[Weight] checking " + geoKlbHostName + " endpoint") - Expect(zoneEndpoints).To(ContainElement(PointTo(MatchFields(IgnoreExtras, Fields{ - "DNSName": Equal(geoKlbHostName), - "Targets": ConsistOf(allGeoClusterHostnames), - "RecordType": Equal("CNAME"), - "SetIdentifier": Equal(""), - "RecordTTL": Equal(externaldnsendpoint.TTL(60)), - "ProviderSpecific": ContainElements(gcpWeightProps), - })))) - totalEndpointsChecked++ - By("[Weight] checking " + geoKlbHostName + " TXT owner endpoint") - Expect(zoneEndpoints).To(ContainElement(PointTo(MatchFields(IgnoreExtras, Fields{ - "DNSName": Equal("kuadrant-cname-" + geoKlbHostName), - "Targets": ContainElement(And(geoOwnerMatcher[geoCode]...)), - "RecordType": Equal("TXT"), - "SetIdentifier": Equal(""), - "RecordTTL": Equal(externaldnsendpoint.TTL(300)), - })))) - totalEndpointsChecked++ - } - } - if testDNSProvider == "aws" { - // A weighted CNAME record should exist for each dns record in each geo and be owned only by that endpoint - for _, geoRecords := range testGeoRecords { - geoKlbHostName := geoRecords[0].config.hostnames.geoKlb - for i := range geoRecords { - clusterKlbHostName := geoRecords[i].config.hostnames.clusterKlb - ownerID := geoRecords[i].record.Status.OwnerID - By("[Weight] checking " + geoKlbHostName + " -> " + clusterKlbHostName + " - endpoint") - Expect(zoneEndpoints).To(ContainElement(PointTo(MatchFields(IgnoreExtras, Fields{ - "DNSName": Equal(geoKlbHostName), - "Targets": ConsistOf(clusterKlbHostName), - "RecordType": Equal("CNAME"), - "SetIdentifier": Equal(clusterKlbHostName), - "RecordTTL": Equal(externaldnsendpoint.TTL(60)), - "ProviderSpecific": Equal(externaldnsendpoint.ProviderSpecific{ - {Name: "alias", Value: "false"}, - {Name: "aws/weight", Value: "200"}, - }), - })))) - totalEndpointsChecked++ - By("[Weight] checking " + geoKlbHostName + " -> " + clusterKlbHostName + " -> " + ownerID + " TXT owner endpoint") - Expect(zoneEndpoints).To(ContainElement(PointTo(MatchFields(IgnoreExtras, Fields{ - "DNSName": Equal("kuadrant-cname-" + geoKlbHostName), - "Targets": ConsistOf("\"heritage=external-dns,external-dns/owner=" + ownerID + "\""), - "RecordType": Equal("TXT"), - "SetIdentifier": Equal(clusterKlbHostName), - "RecordTTL": Equal(externaldnsendpoint.TTL(300)), - "ProviderSpecific": Equal(externaldnsendpoint.ProviderSpecific{ - {Name: "aws/weight", Value: "200"}, - }), - })))) - totalEndpointsChecked++ - } - } - } - - By("[Cluster] checking cluster endpoints") - // An A record with the cluster target IP should exist for each dns record and owned only by that endpoint - for i := range testRecords { - clusterKlbHostName := testRecords[i].config.hostnames.clusterKlb - clusterTargetIP := testRecords[i].config.testTargetIP - ownerID := testRecords[i].record.Status.OwnerID - By("[Cluster] checking " + clusterKlbHostName + " endpoint") - Expect(zoneEndpoints).To(ContainElement(PointTo(MatchFields(IgnoreExtras, Fields{ - "DNSName": Equal(clusterKlbHostName), - "Targets": ConsistOf(clusterTargetIP), - "RecordType": Equal("A"), - "SetIdentifier": Equal(""), - "RecordTTL": Equal(externaldnsendpoint.TTL(60)), - })))) - totalEndpointsChecked++ - By("[Cluster] checking " + clusterKlbHostName + " TXT owner endpoint") - Expect(zoneEndpoints).To(ContainElement(PointTo(MatchFields(IgnoreExtras, Fields{ - "DNSName": Equal("kuadrant-a-" + clusterKlbHostName), - "Targets": ConsistOf("\"heritage=external-dns,external-dns/owner=" + ownerID + "\""), - "RecordType": Equal("TXT"), - "SetIdentifier": Equal(""), - "RecordTTL": Equal(externaldnsendpoint.TTL(300)), - })))) - totalEndpointsChecked++ - } - - By("checking all endpoints were validated") - Expect(totalEndpointsChecked).To(Equal(expectedEndpointsLen)) - - By("deleting all remaining dns records") - for _, tr := range testRecords { - err := tr.cluster.k8sClient.Delete(ctx, tr.record, - client.PropagationPolicy(metav1.DeletePropagationForeground)) - Expect(client.IgnoreNotFound(err)).ToNot(HaveOccurred()) - } - - By(fmt.Sprintf("checking all dns records are removed within %s", recordsRemovedMaxDuration)) - Eventually(func(g Gomega, ctx context.Context) { - for _, tr := range testRecords { - err := tr.cluster.k8sClient.Get(ctx, client.ObjectKeyFromObject(tr.record), tr.record) - g.Expect(err).To(MatchError(ContainSubstring("not found"))) - } - }, recordsRemovedMaxDuration, 5*time.Second, ctx).Should(Succeed()) - - By("checking provider zone records are all removed") - Eventually(func(g Gomega, ctx context.Context) { - zoneEndpoints, err := EndpointsForHost(ctx, testProvider, testHostname) - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(zoneEndpoints).To(HaveLen(0)) - }, 5*time.Second, 1*time.Second, ctx).Should(Succeed()) - - }) - }) - -}) diff --git a/test/e2e/multi_instance/suite_test.go b/test/e2e/multi_instance/suite_test.go deleted file mode 100644 index 183d6881..00000000 --- a/test/e2e/multi_instance/suite_test.go +++ /dev/null @@ -1,265 +0,0 @@ -//go:build e2e_multi_instance - -package multi_instance - -import ( - "context" - "fmt" - "os" - "strconv" - "strings" - "testing" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - . "github.com/onsi/gomega/gstruct" - - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes/scheme" - _ "k8s.io/client-go/plugin/pkg/client/auth" - "k8s.io/client-go/tools/clientcmd" - "sigs.k8s.io/controller-runtime/pkg/client" - logf "sigs.k8s.io/controller-runtime/pkg/log" - "sigs.k8s.io/controller-runtime/pkg/log/zap" - - "github.com/kuadrant/dns-operator/api/v1alpha1" - "github.com/kuadrant/dns-operator/internal/provider" - _ "github.com/kuadrant/dns-operator/internal/provider/aws" - _ "github.com/kuadrant/dns-operator/internal/provider/azure" - _ "github.com/kuadrant/dns-operator/internal/provider/google" - . "github.com/kuadrant/dns-operator/test/e2e/helpers" -) - -const ( - // configuration environment variables - dnsManagedZoneName = "TEST_DNS_MANAGED_ZONE_NAME" - dnsNamespaces = "TEST_DNS_NAMESPACES" - dnsClusterContexts = "TEST_DNS_CLUSTER_CONTEXTS" - deploymentCount = "DEPLOYMENT_COUNT" - clusterCount = "CLUSTER_COUNT" -) - -var ( - // testSuiteID is a randomly generated identifier for the test suite - testSuiteID string - // testZoneDomainName provided domain name for the testZoneID e.g. e2e.hcpapps.net - testZoneDomainName string - testManagedZoneName string - testNamespaces []string - testClusterContexts []string - testDNSProvider string - testClusters []testCluster -) - -// testCluster represents a cluster under test and contains a reference to a configured k8client and all it's managed zones. -type testCluster struct { - name string - testManagedZones []*v1alpha1.ManagedZone - k8sClient client.Client -} - -// testDNSRecord encapsulates a v1alpha1.DNSRecord created in a test case, the v1alpha1.ManagedZone it was created in and the config used to create it. -// The testConfig is used when asserting the expected values set in the providers. -type testDNSRecord struct { - cluster *testCluster - managedZone *v1alpha1.ManagedZone - record *v1alpha1.DNSRecord - config testConfig -} - -type testConfig struct { - testTargetIP string - testGeoCode string - testDefaultGeoCode string - hostnames testHostnames -} - -type testHostnames struct { - klb string - geoKlb string - defaultGeoKlb string - clusterKlb string -} - -func TestAPIs(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "E2E Multi Instance Tests Suite") -} - -var _ = BeforeSuite(func(ctx SpecContext) { - logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) - - err := setConfigFromEnvVars() - Expect(err).NotTo(HaveOccurred()) - - err = v1alpha1.AddToScheme(scheme.Scheme) - Expect(err).NotTo(HaveOccurred()) - - loadClusters(ctx) - Expect(testDNSProvider).NotTo(BeEmpty()) - Expect(testZoneDomainName).NotTo(BeEmpty()) - Expect(testClusters).NotTo(BeEmpty()) - for i := range testClusters { - Expect(testClusters[i].testManagedZones).NotTo(BeEmpty()) - } - - testSuiteID = "dns-op-e2e-multi-" + GenerateName() - - geoCode := "EU" - if testDNSProvider == "gcp" { - geoCode = "europe-west1" - } - SetTestEnv("testGeoCode", geoCode) -}) - -// setConfigFromEnvVars loads test suite runtime configurations from env vars. -// dnsManagedZoneName managed zone name expected to exist in each test namespace (i.e. dev-mz-aws). -// dnsNamespaces test namespaces, comma seperated list (i.e. dns-operator-1,dns-operator-2) -// deploymentCount number of test namespaces expected. Appends an index suffix to the dnsNamespaces, only used if dnsNamespaces is a single length array. -// -// Examples: -// dnsNamespaces=dns-operator deploymentCount= = dnsNamespaces=dns-operator -// dnsNamespaces=dns-operator-1,dns-operator-2 deploymentCount= = dnsNamespaces=dns-operator-1,dns-operator-2 -// dnsNamespaces=dns-operator deploymentCount=1 = dnsNamespaces=dns-operator-1 -// dnsNamespaces=dns-operator deploymentCount=2 = dnsNamespaces=dns-operator-1,dns-operator-2 -// dnsNamespaces=dns-operator-5,dns-operator-6 deploymentCount=1 = dnsNamespaces=dns-operator-5,dns-operator-6 -// -// dnsClusterContexts test cluster contexts, comma seperated list (i.e. kind-kuadrant-dns-local-1,kind-kuadrant-dns-local-2), -// if unset the current context is used and a single cluster is assumed. -// clusterCount number of test clusters expected. Appends an index suffix to the dnsClusterContexts, only used if dnsClusterContexts is a single length array. -// -// Examples: -// dnsClusterContexts=kind-kuadrant-dns-local clusterCount= = dnsClusterContexts=kind-kuadrant-dns-local -// dnsClusterContexts=kind-kuadrant-dns-local-1,kind-kuadrant-dns-local-2 clusterCount= = dnsClusterContexts=kind-kuadrant-dns-local-1,kind-kuadrant-dns-local-2 -// dnsClusterContexts=kind-kuadrant-dns-local clusterCount=1 = dnsClusterContexts=kind-kuadrant-dns-local-1 -// dnsClusterContexts=kind-kuadrant-dns-local clusterCount=2 = dnsClusterContexts=kind-kuadrant-dns-local-1,kind-kuadrant-dns-local-2 -// dnsClusterContexts=my-cluster-1,my-cluster-2 clusterCount=1 = dnsClusterContexts=my-cluster-1,my-cluster-2 - -func setConfigFromEnvVars() error { - // Load test suite configuration from the environment - if testManagedZoneName = os.Getenv(dnsManagedZoneName); testManagedZoneName == "" { - return fmt.Errorf("env variable '%s' must be set", dnsManagedZoneName) - } - - namespacesStr := os.Getenv(dnsNamespaces) - if namespacesStr == "" { - return fmt.Errorf("env variable '%s' must be set", dnsNamespaces) - } - - namespaces := strings.Split(namespacesStr, ",") - if len(namespaces) == 1 { - if dcStr := os.Getenv(deploymentCount); dcStr != "" { - dc, err := strconv.Atoi(dcStr) - if err != nil { - return fmt.Errorf("env variable '%s' must be an integar", deploymentCount) - } - for i := 1; i <= dc; i++ { - testNamespaces = append(testNamespaces, fmt.Sprintf("%s-%d", namespaces[0], i)) - } - } else { - testNamespaces = namespaces - } - } else { - testNamespaces = namespaces - } - - clusterContextsStr := os.Getenv(dnsClusterContexts) - if clusterContextsStr == "" { - testClusterContexts = []string{"current"} - return nil - } - - clusterContexts := strings.Split(clusterContextsStr, ",") - if len(clusterContexts) == 1 { - if dcStr := os.Getenv(clusterCount); dcStr != "" { - dc, err := strconv.Atoi(dcStr) - if err != nil { - return fmt.Errorf("env variable '%s' must be an integar", clusterCount) - } - for i := 1; i <= dc; i++ { - testClusterContexts = append(testClusterContexts, fmt.Sprintf("%s-%d", clusterContexts[0], i)) - } - } else { - testClusterContexts = clusterContexts - } - } else { - testClusterContexts = clusterContexts - } - - return nil -} - -// loadClusters iterates each of the configured test clusters, configures a k8s client, loads test managed zones and creates a `testCluster` resource. -func loadClusters(ctx context.Context) { - for _, c := range testClusterContexts { - cfgOverrides := &clientcmd.ConfigOverrides{} - if c != "current" { - cfgOverrides = &clientcmd.ConfigOverrides{CurrentContext: c} - } - cfg, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig( - clientcmd.NewDefaultClientConfigLoadingRules(), - cfgOverrides, - ).ClientConfig() - Expect(err).NotTo(HaveOccurred()) - - k8sClient, err := client.New(cfg, client.Options{Scheme: scheme.Scheme}) - Expect(err).NotTo(HaveOccurred()) - - tc := &testCluster{ - name: c, - k8sClient: k8sClient, - } - - loadManagedZones(ctx, tc) - - //Append the cluster to the list of test clusters - testClusters = append(testClusters, *tc) - } -} - -// loadManagedZones iterates each of the configured test namespaces, loads the expected managed zone (TEST_DNS_MANAGED_ZONE_NAME), and asserts the configuration of each is compatible. -// Sets the test suite testDNSProvider and testZoneDomainName directly from the managed zone spec and provider secret. -// If the managed zone does not exist in the namespace, an error is thrown. -// If the managed zone has a different domain name from any previously loaded managed zones, an error is thrown. -// If the managed zone has a different dns provider from any previously loaded managed zones, an error is thrown. -func loadManagedZones(ctx context.Context, tc *testCluster) { - for _, n := range testNamespaces { - mz := &v1alpha1.ManagedZone{} - - // Ensure managed zone exists and is ready - err := tc.k8sClient.Get(ctx, client.ObjectKey{Namespace: n, Name: testManagedZoneName}, mz) - Expect(err).NotTo(HaveOccurred()) - Expect(mz.Status.Conditions).To( - ContainElement(MatchFields(IgnoreExtras, Fields{ - "Type": Equal(string(v1alpha1.ConditionTypeReady)), - "Status": Equal(metav1.ConditionTrue), - "ObservedGeneration": Equal(mz.Generation), - })), - ) - - // Ensure all managed zone names match - if testZoneDomainName == "" { - testZoneDomainName = mz.Spec.DomainName - } else { - Expect(mz.Spec.DomainName).To(Equal(testZoneDomainName)) - } - - s := &v1.Secret{} - err = tc.k8sClient.Get(ctx, client.ObjectKey{Namespace: n, Name: mz.Spec.SecretRef.Name}, s) - Expect(err).NotTo(HaveOccurred()) - - p, err := provider.NameForProviderSecret(s) - Expect(err).NotTo(HaveOccurred()) - - // Ensure all managed zone are suing the same provider - if testDNSProvider == "" { - testDNSProvider = p - } else { - Expect(p).To(Equal(testDNSProvider)) - } - - //Append the managed zone to the list of test zones - tc.testManagedZones = append(tc.testManagedZones, mz) - } -} diff --git a/test/e2e/multi_record_test.go b/test/e2e/multi_record_test.go index 046a0f77..1fa967ae 100644 --- a/test/e2e/multi_record_test.go +++ b/test/e2e/multi_record_test.go @@ -4,23 +4,26 @@ package e2e import ( "context" + "fmt" "strings" "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" . "github.com/onsi/gomega/gstruct" + "github.com/onsi/gomega/types" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" externaldnsendpoint "sigs.k8s.io/external-dns/endpoint" "github.com/kuadrant/dns-operator/api/v1alpha1" + "github.com/kuadrant/dns-operator/internal/provider" . "github.com/kuadrant/dns-operator/test/e2e/helpers" ) // Test Cases covering multiple DNSRecords updating a set of records in a zone -var _ = Describe("Multi Record Test", func() { +var _ = Describe("Multi Record Test", Labels{"multi_record"}, func() { // testID is a randomly generated identifier for the test // it is used to name resources and/or namespaces so different // tests can be run in parallel in the same cluster @@ -30,17 +33,21 @@ var _ = Describe("Multi Record Test", func() { // testHostname generated hostname for this test e.g. t-gw-mgc-12345.t-e2e-12345.e2e.hcpapps.net var testHostname string - var dnsRecord1 *v1alpha1.DNSRecord - var dnsRecord2 *v1alpha1.DNSRecord var geoCode1 string var geoCode2 string + var testRecords []*testDNSRecord + + recordsReadyMaxDuration := time.Minute + recordsRemovedMaxDuration := time.Minute + BeforeEach(func(ctx SpecContext) { testID = "t-multi-" + GenerateName() testDomainName = strings.Join([]string{testSuiteID, testZoneDomainName}, ".") testHostname = strings.Join([]string{testID, testDomainName}, ".") + testRecords = []*testDNSRecord{} - if testDNSProvider == "gcp" { + if testDNSProvider == "google" { geoCode1 = "us-east1" geoCode2 = "europe-west1" } else { @@ -50,564 +57,478 @@ var _ = Describe("Multi Record Test", func() { }) AfterEach(func(ctx SpecContext) { - if dnsRecord1 != nil { - err := k8sClient.Delete(ctx, dnsRecord1, - client.PropagationPolicy(metav1.DeletePropagationForeground)) - Expect(client.IgnoreNotFound(err)).ToNot(HaveOccurred()) - } - if dnsRecord2 != nil { - err := k8sClient.Delete(ctx, dnsRecord2, + By("ensuring all dns records are deleted") + for _, tr := range testRecords { + err := tr.cluster.k8sClient.Delete(ctx, tr.record, client.PropagationPolicy(metav1.DeletePropagationForeground)) Expect(client.IgnoreNotFound(err)).ToNot(HaveOccurred()) } + + By("checking all dns records are removed") + Eventually(func(g Gomega, ctx context.Context) { + for _, tr := range testRecords { + err := tr.cluster.k8sClient.Get(ctx, client.ObjectKeyFromObject(tr.record), tr.record) + g.Expect(err).To(MatchError(ContainSubstring("not found"))) + } + }, time.Minute, 10*time.Second, ctx).Should(Succeed()) }) Context("simple", func() { - It("makes available a hostname that can be resolved", func(ctx SpecContext) { - By("creating two dns records") - testTargetIP1 := "127.0.0.1" - testTargetIP2 := "127.0.0.2" - - dnsRecord1 = &v1alpha1.DNSRecord{ - ObjectMeta: metav1.ObjectMeta{ - Name: testID + "-1", - Namespace: testNamespace, - }, - Spec: v1alpha1.DNSRecordSpec{ - RootHost: testHostname, - ManagedZoneRef: &v1alpha1.ManagedZoneReference{ - Name: testManagedZoneName, - }, - Endpoints: []*externaldnsendpoint.Endpoint{ - { - DNSName: testHostname, - Targets: []string{ - testTargetIP1, - }, - RecordType: "A", - RecordTTL: 60, + It("creates and deletes distributed dns records", func(ctx SpecContext) { + By(fmt.Sprintf("creating %d simple dnsrecords accross %d clusters", len(testNamespaces)*len(testClusters), len(testClusters))) + for ci, tc := range testClusters { + for mi, mz := range tc.testManagedZones { + config := testConfig{ + testTargetIP: fmt.Sprintf("127.0.%d.%d", ci+1, mi+1), + } + record := &v1alpha1.DNSRecord{ + ObjectMeta: metav1.ObjectMeta{ + Name: testID, + Namespace: mz.Namespace, }, - }, - HealthCheck: nil, - }, - } - - dnsRecord2 = &v1alpha1.DNSRecord{ - ObjectMeta: metav1.ObjectMeta{ - Name: testID + "-2", - Namespace: testNamespace, - }, - Spec: v1alpha1.DNSRecordSpec{ - RootHost: testHostname, - ManagedZoneRef: &v1alpha1.ManagedZoneReference{ - Name: testManagedZoneName, - }, - Endpoints: []*externaldnsendpoint.Endpoint{ - { - DNSName: testHostname, - Targets: []string{ - testTargetIP2, + Spec: v1alpha1.DNSRecordSpec{ + RootHost: testHostname, + ManagedZoneRef: &v1alpha1.ManagedZoneReference{ + Name: mz.Name, + }, + Endpoints: []*externaldnsendpoint.Endpoint{ + { + DNSName: testHostname, + Targets: []string{ + config.testTargetIP, + }, + RecordType: "A", + RecordTTL: 60, + }, }, - RecordType: "A", - RecordTTL: 60, + HealthCheck: nil, }, - }, - HealthCheck: nil, - }, + } + + By(fmt.Sprintf("creating dns record [name: `%s`, namespace: `%s`, mz: `%s`, endpoint: [dnsname: `%s`, target: `%s`]] on cluster [name: `%s`]", record.Name, record.Namespace, mz.Name, testHostname, config.testTargetIP, tc.name)) + err := tc.k8sClient.Create(ctx, record) + Expect(err).ToNot(HaveOccurred()) + + testRecords = append(testRecords, &testDNSRecord{ + cluster: &testClusters[ci], + managedZone: mz, + record: record, + config: config, + }) + } } - By("creating dnsrecord " + dnsRecord1.Name) - err := k8sClient.Create(ctx, dnsRecord1) - Expect(err).ToNot(HaveOccurred()) - - By("creating dnsrecord " + dnsRecord2.Name) - err = k8sClient.Create(ctx, dnsRecord2) - Expect(err).ToNot(HaveOccurred()) - - By("checking dns records become ready") + By(fmt.Sprintf("checking all dns records become ready within %s", recordsReadyMaxDuration)) Eventually(func(g Gomega, ctx context.Context) { - err := k8sClient.Get(ctx, client.ObjectKeyFromObject(dnsRecord1), dnsRecord1) - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(dnsRecord1.Status.Conditions).To( - ContainElement(MatchFields(IgnoreExtras, Fields{ - "Type": Equal(string(v1alpha1.ConditionTypeReady)), - "Status": Equal(metav1.ConditionTrue), - })), - ) - err = k8sClient.Get(ctx, client.ObjectKeyFromObject(dnsRecord2), dnsRecord2) - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(dnsRecord2.Status.Conditions).To( - ContainElement(MatchFields(IgnoreExtras, Fields{ - "Type": Equal(string(v1alpha1.ConditionTypeReady)), - "Status": Equal(metav1.ConditionTrue), - })), - ) - }, time.Minute, 10*time.Second, ctx).Should(Succeed()) - - By("checking dns records ownerID is set correctly") - Expect(dnsRecord1.Spec.OwnerID).To(BeEmpty()) - Expect(dnsRecord2.Spec.OwnerID).To(BeEmpty()) - Expect(dnsRecord1.Status.OwnerID).ToNot(BeEmpty()) - Expect(dnsRecord2.Status.OwnerID).ToNot(BeEmpty()) - Expect(dnsRecord1.Status.OwnerID).To(Equal(dnsRecord1.GetUIDHash())) - Expect(dnsRecord2.Status.OwnerID).To(Equal(dnsRecord2.GetUIDHash())) - - testProvider, err := ProviderForManagedZone(ctx, testManagedZone, k8sClient) + for _, tr := range testRecords { + err := tr.cluster.k8sClient.Get(ctx, client.ObjectKeyFromObject(tr.record), tr.record) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(tr.record.Status.Conditions).To( + ContainElement(MatchFields(IgnoreExtras, Fields{ + "Type": Equal(string(v1alpha1.ConditionTypeReady)), + "Status": Equal(metav1.ConditionTrue), + })), + ) + } + }, recordsReadyMaxDuration, 5*time.Second, ctx).Should(Succeed()) + + By("checking provider zone records are created as expected") + testProvider, err := ProviderForManagedZone(ctx, testClusters[0].testManagedZones[0], testClusters[0].k8sClient) Expect(err).NotTo(HaveOccurred()) - By("ensuring zone records are created as expected") - Eventually(func(g Gomega, ctx context.Context) { - zoneEndpoints, err := EndpointsForHost(ctx, testProvider, testHostname) - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(zoneEndpoints).To(HaveLen(2)) - g.Expect(zoneEndpoints).To(ContainElements( - PointTo(MatchFields(IgnoreExtras, Fields{ - "DNSName": Equal(testHostname), - "Targets": ConsistOf(testTargetIP1, testTargetIP2), - "RecordType": Equal("A"), - "SetIdentifier": Equal(""), - "RecordTTL": Equal(externaldnsendpoint.TTL(60)), - })), + zoneEndpoints, err := EndpointsForHost(ctx, testProvider, testHostname) + Expect(err).NotTo(HaveOccurred()) + Expect(zoneEndpoints).To(HaveLen(2)) + + var allOwners = []string{} + var allTargetIps = []string{} + for i := range testRecords { + allOwners = append(allOwners, testRecords[i].record.Status.OwnerID) + allTargetIps = append(allTargetIps, testRecords[i].config.testTargetIP) + } + + By("checking all target ips are present") + Expect(zoneEndpoints).To(ContainElements( + PointTo(MatchFields(IgnoreExtras, Fields{ + "DNSName": Equal(testHostname), + "Targets": ConsistOf(allTargetIps), + "RecordType": Equal("A"), + "SetIdentifier": Equal(""), + "RecordTTL": Equal(externaldnsendpoint.TTL(60)), + })), + )) + + By("checking all owner references are present") + for _, owner := range allOwners { + Expect(zoneEndpoints).To(ContainElements( PointTo(MatchFields(IgnoreExtras, Fields{ - "DNSName": Equal("kuadrant-a-" + testHostname), - "Targets": ContainElement(And( - ContainSubstring("heritage=external-dns,external-dns/owner="), - ContainSubstring(dnsRecord1.Status.OwnerID), - ContainSubstring(dnsRecord2.Status.OwnerID), - )), + "DNSName": Equal("kuadrant-a-" + testHostname), + "Targets": ContainElement(ContainSubstring(owner)), "RecordType": Equal("TXT"), "SetIdentifier": Equal(""), "RecordTTL": Equal(externaldnsendpoint.TTL(300)), })), )) - }, 10*time.Second, 1*time.Second, ctx).Should(Succeed()) + } - By("ensuring the authoritative nameserver resolves the hostname") + By("checking the authoritative nameserver resolves the hostname") // speed up things by using the authoritative nameserver authoritativeResolver := ResolverForDomainName(testZoneDomainName) Eventually(func(g Gomega, ctx context.Context) { ips, err := authoritativeResolver.LookupHost(ctx, testHostname) g.Expect(err).NotTo(HaveOccurred()) - g.Expect(ips).To(ConsistOf(testTargetIP1, testTargetIP2)) + g.Expect(ips).To(ContainElements(allTargetIps)) }, 300*time.Second, 10*time.Second, ctx).Should(Succeed()) - By("deleting dnsrecord " + dnsRecord2.Name) - err = k8sClient.Delete(ctx, dnsRecord2, + //Test Deletion of one of the records + recordToDelete := testRecords[0] + lastRecord := len(testRecords) == 1 + By(fmt.Sprintf("deleting dns record [name: `%s` namespace: `%s`]", recordToDelete.record.Name, recordToDelete.record.Namespace)) + err = recordToDelete.cluster.k8sClient.Delete(ctx, recordToDelete.record, client.PropagationPolicy(metav1.DeletePropagationForeground)) Expect(client.IgnoreNotFound(err)).ToNot(HaveOccurred()) - By("checking dnsrecord " + dnsRecord2.Name + " is removed") + By(fmt.Sprintf("checking dns record [name: `%s` namespace: `%s`] is removed within %s", recordToDelete.record.Name, recordToDelete.record.Namespace, recordsRemovedMaxDuration)) Eventually(func(g Gomega, ctx context.Context) { - err := k8sClient.Get(ctx, client.ObjectKeyFromObject(dnsRecord2), dnsRecord2) + err := recordToDelete.cluster.k8sClient.Get(ctx, client.ObjectKeyFromObject(recordToDelete.record), recordToDelete.record) g.Expect(err).To(HaveOccurred()) g.Expect(err).To(MatchError(ContainSubstring("not found"))) - }, 25*time.Second, 1*time.Second, ctx).Should(Succeed()) + }, recordsRemovedMaxDuration, 5*time.Second, ctx).Should(Succeed()) - By("ensuring zone records are updated as expected") + By("checking provider zone records are updated as expected") Eventually(func(g Gomega, ctx context.Context) { zoneEndpoints, err := EndpointsForHost(ctx, testProvider, testHostname) g.Expect(err).NotTo(HaveOccurred()) - g.Expect(zoneEndpoints).To(HaveLen(2)) - g.Expect(zoneEndpoints).To(ContainElements( - PointTo(MatchFields(IgnoreExtras, Fields{ - "DNSName": Equal(testHostname), - "Targets": ConsistOf(testTargetIP1), - "RecordType": Equal("A"), - "SetIdentifier": Equal(""), - "RecordTTL": Equal(externaldnsendpoint.TTL(60)), - })), - PointTo(MatchFields(IgnoreExtras, Fields{ - "DNSName": Equal("kuadrant-a-" + testHostname), - "Targets": ConsistOf("\"heritage=external-dns,external-dns/owner=" + dnsRecord1.Status.OwnerID + "\""), - "RecordType": Equal("TXT"), - "SetIdentifier": Equal(""), - "RecordTTL": Equal(externaldnsendpoint.TTL(300)), - })), - )) - }, 5*time.Second, 1*time.Second, ctx).Should(Succeed()) - - By("ensuring the authoritative nameserver resolves the hostname") - Eventually(func(g Gomega, ctx context.Context) { - ips, err := authoritativeResolver.LookupHost(ctx, testHostname) - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(ips).To(ConsistOf(testTargetIP1)) - }, 300*time.Second, 10*time.Second, ctx).Should(Succeed()) - - By("deleting dnsrecord " + dnsRecord1.Name) - err = k8sClient.Delete(ctx, dnsRecord1, - client.PropagationPolicy(metav1.DeletePropagationForeground)) - Expect(client.IgnoreNotFound(err)).ToNot(HaveOccurred()) + if lastRecord { + g.Expect(zoneEndpoints).To(HaveLen(0)) + } else { + g.Expect(zoneEndpoints).To(HaveLen(2)) + By(fmt.Sprintf("checking ip `%s` and owner `%s` are removed", recordToDelete.config.testTargetIP, recordToDelete.record.Status.OwnerID)) + g.Expect(zoneEndpoints).To(ContainElements( + PointTo(MatchFields(IgnoreExtras, Fields{ + "DNSName": Equal(testHostname), + "Targets": Not(ContainElement(recordToDelete.config.testTargetIP)), + "RecordType": Equal("A"), + "SetIdentifier": Equal(""), + "RecordTTL": Equal(externaldnsendpoint.TTL(60)), + })), + PointTo(MatchFields(IgnoreExtras, Fields{ + "DNSName": Equal("kuadrant-a-" + testHostname), + "Targets": Not(ContainElement(ContainSubstring(recordToDelete.record.Status.OwnerID))), + "RecordType": Equal("TXT"), + "SetIdentifier": Equal(""), + "RecordTTL": Equal(externaldnsendpoint.TTL(300)), + })), + )) + } + }, 5*time.Second, time.Second, ctx).Should(Succeed()) + + By("deleting all remaining dns records") + for _, tr := range testRecords { + err := tr.cluster.k8sClient.Delete(ctx, tr.record, + client.PropagationPolicy(metav1.DeletePropagationForeground)) + Expect(client.IgnoreNotFound(err)).ToNot(HaveOccurred()) + } - By("checking dnsrecord " + dnsRecord1.Name + " is removed") + By(fmt.Sprintf("checking all dns records are removed within %s", recordsRemovedMaxDuration)) Eventually(func(g Gomega, ctx context.Context) { - err := k8sClient.Get(ctx, client.ObjectKeyFromObject(dnsRecord1), dnsRecord1) - g.Expect(err).To(HaveOccurred()) - g.Expect(err).To(MatchError(ContainSubstring("not found"))) - }, 10*time.Second, 1*time.Second, ctx).Should(Succeed()) + for _, tr := range testRecords { + err := tr.cluster.k8sClient.Get(ctx, client.ObjectKeyFromObject(tr.record), tr.record) + g.Expect(err).To(MatchError(ContainSubstring("not found"))) + } + }, recordsRemovedMaxDuration, 5*time.Second, ctx).Should(Succeed()) - By("ensuring zone records are all removed as expected") + By("checking provider zone records are all removed") Eventually(func(g Gomega, ctx context.Context) { zoneEndpoints, err := EndpointsForHost(ctx, testProvider, testHostname) g.Expect(err).NotTo(HaveOccurred()) g.Expect(zoneEndpoints).To(HaveLen(0)) - }, 5*time.Second, 1*time.Second, ctx).Should(Succeed()) + }, 5*time.Second, time.Second, ctx).Should(Succeed()) + }) + }) Context("loadbalanced", func() { - It("makes available a hostname that can be resolved", func(ctx SpecContext) { + It("creates and deletes distributed dns records", func(ctx SpecContext) { if testDNSProvider == "azure" { Skip("not yet supported for azure") } - By("creating two dns records") - klbHostName := "klb." + testHostname - - testTargetIP1 := "127.0.0.1" - geo1KlbHostName := strings.ToLower(geoCode1) + "." + klbHostName - cluster1KlbHostName := "cluster1." + klbHostName - - testTargetIP2 := "127.0.0.2" - geo2KlbHostName := strings.ToLower(geoCode2) + "." + klbHostName - cluster2KlbHostName := "cluster2." + klbHostName - - dnsRecord1 = &v1alpha1.DNSRecord{ - ObjectMeta: metav1.ObjectMeta{ - Name: testID + "-1", - Namespace: testNamespace, - }, - Spec: v1alpha1.DNSRecordSpec{ - RootHost: testHostname, - ManagedZoneRef: &v1alpha1.ManagedZoneReference{ - Name: testManagedZoneName, - }, - Endpoints: []*externaldnsendpoint.Endpoint{ - { - DNSName: cluster1KlbHostName, - Targets: []string{ - testTargetIP1, - }, - RecordType: "A", - RecordTTL: 60, + testGeoRecords := map[string][]testDNSRecord{} + + By(fmt.Sprintf("creating %d loadbalanced dnsrecords accross %d clusters", len(testNamespaces)*len(testClusters), len(testClusters))) + for ci, tc := range testClusters { + for mi, mz := range tc.testManagedZones { + + var geoCode string + if (ci+mi)%2 == 0 { + geoCode = geoCode1 + } else { + geoCode = geoCode2 + } + + klbHostName := "klb." + testHostname + geoKlbHostName := strings.ToLower(geoCode) + "." + klbHostName + defaultGeoKlbHostName := strings.ToLower(geoCode1) + "." + klbHostName + clusterKlbHostName := fmt.Sprintf("cluster%d-%d.%s", ci+1, mi+1, klbHostName) + + config := testConfig{ + testTargetIP: fmt.Sprintf("127.0.%d.%d", ci+1, mi+1), + testGeoCode: geoCode, + testDefaultGeoCode: geoCode1, + hostnames: testHostnames{ + klb: klbHostName, + geoKlb: geoKlbHostName, + defaultGeoKlb: defaultGeoKlbHostName, + clusterKlb: clusterKlbHostName, }, - { - DNSName: testHostname, - Targets: []string{ - klbHostName, - }, - RecordType: "CNAME", - RecordTTL: 300, + } + + record := &v1alpha1.DNSRecord{ + ObjectMeta: metav1.ObjectMeta{ + Name: testID, + Namespace: mz.Namespace, }, - { - DNSName: geo1KlbHostName, - Targets: []string{ - cluster1KlbHostName, + Spec: v1alpha1.DNSRecordSpec{ + RootHost: testHostname, + ManagedZoneRef: &v1alpha1.ManagedZoneReference{ + Name: mz.Name, }, - RecordType: "CNAME", - RecordTTL: 60, - SetIdentifier: cluster1KlbHostName, - ProviderSpecific: externaldnsendpoint.ProviderSpecific{ + Endpoints: []*externaldnsendpoint.Endpoint{ { - Name: "weight", - Value: "200", + DNSName: clusterKlbHostName, + Targets: []string{ + config.testTargetIP, + }, + RecordType: "A", + RecordTTL: 60, }, - }, - }, - { - DNSName: klbHostName, - Targets: []string{ - geo1KlbHostName, - }, - RecordType: "CNAME", - RecordTTL: 300, - SetIdentifier: geoCode1, - ProviderSpecific: externaldnsendpoint.ProviderSpecific{ { - Name: "geo-code", - Value: geoCode1, + DNSName: testHostname, + Targets: []string{ + klbHostName, + }, + RecordType: "CNAME", + RecordTTL: 300, }, - }, - }, - { - DNSName: klbHostName, - Targets: []string{ - geo1KlbHostName, - }, - RecordType: "CNAME", - RecordTTL: 300, - SetIdentifier: "default", - ProviderSpecific: externaldnsendpoint.ProviderSpecific{ { - Name: "geo-code", - Value: "*", + DNSName: geoKlbHostName, + Targets: []string{ + clusterKlbHostName, + }, + RecordType: "CNAME", + RecordTTL: 60, + SetIdentifier: clusterKlbHostName, + ProviderSpecific: externaldnsendpoint.ProviderSpecific{ + { + Name: "weight", + Value: "200", + }, + }, }, - }, - }, - }, - HealthCheck: nil, - }, - } - - dnsRecord2 = &v1alpha1.DNSRecord{ - ObjectMeta: metav1.ObjectMeta{ - Name: testID + "-2", - Namespace: testNamespace, - }, - Spec: v1alpha1.DNSRecordSpec{ - RootHost: testHostname, - ManagedZoneRef: &v1alpha1.ManagedZoneReference{ - Name: testManagedZoneName, - }, - Endpoints: []*externaldnsendpoint.Endpoint{ - { - DNSName: cluster2KlbHostName, - Targets: []string{ - testTargetIP2, - }, - RecordType: "A", - RecordTTL: 60, - }, - { - DNSName: testHostname, - Targets: []string{ - klbHostName, - }, - RecordType: "CNAME", - RecordTTL: 300, - }, - { - DNSName: geo2KlbHostName, - Targets: []string{ - cluster2KlbHostName, - }, - RecordType: "CNAME", - RecordTTL: 60, - SetIdentifier: cluster2KlbHostName, - ProviderSpecific: externaldnsendpoint.ProviderSpecific{ { - Name: "weight", - Value: "200", + DNSName: klbHostName, + Targets: []string{ + geoKlbHostName, + }, + RecordType: "CNAME", + RecordTTL: 300, + SetIdentifier: config.testGeoCode, + ProviderSpecific: externaldnsendpoint.ProviderSpecific{ + { + Name: "geo-code", + Value: config.testGeoCode, + }, + }, }, - }, - }, - { - DNSName: klbHostName, - Targets: []string{ - geo2KlbHostName, - }, - RecordType: "CNAME", - RecordTTL: 300, - SetIdentifier: geoCode2, - ProviderSpecific: externaldnsendpoint.ProviderSpecific{ { - Name: "geo-code", - Value: geoCode2, + DNSName: klbHostName, + Targets: []string{ + defaultGeoKlbHostName, + }, + RecordType: "CNAME", + RecordTTL: 300, + SetIdentifier: "default", + ProviderSpecific: externaldnsendpoint.ProviderSpecific{ + { + Name: "geo-code", + Value: "*", + }, + }, }, }, + HealthCheck: nil, }, - //Note this dnsRecord has no default geo .... - }, - HealthCheck: nil, - }, + } + + By(fmt.Sprintf("creating dns record [name: `%s`, namespace: `%s`, managedZone: `%s`, endpoint: [dnsname: `%s`, target: `%s`, geoCode: `%s`]] on cluster [name: `%s`]", record.Name, record.Namespace, mz.Name, testHostname, config.testTargetIP, config.testGeoCode, tc.name)) + err := tc.k8sClient.Create(ctx, record) + Expect(err).ToNot(HaveOccurred()) + tr := &testDNSRecord{ + cluster: &testClusters[ci], + managedZone: mz, + record: record, + config: config, + } + testRecords = append(testRecords, tr) + testGeoRecords[config.testGeoCode] = append(testGeoRecords[config.testGeoCode], *tr) + } } - By("creating dnsrecord " + dnsRecord1.Name) - err := k8sClient.Create(ctx, dnsRecord1) - Expect(err).ToNot(HaveOccurred()) - - By("creating dnsrecord " + dnsRecord2.Name) - err = k8sClient.Create(ctx, dnsRecord2) - Expect(err).ToNot(HaveOccurred()) - - By("checking the dns records become ready") + By(fmt.Sprintf("checking all dns records become ready within %s", recordsReadyMaxDuration)) Eventually(func(g Gomega, ctx context.Context) { - err := k8sClient.Get(ctx, client.ObjectKeyFromObject(dnsRecord1), dnsRecord1) - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(dnsRecord1.Status.Conditions).To( - ContainElement(MatchFields(IgnoreExtras, Fields{ - "Type": Equal(string(v1alpha1.ConditionTypeReady)), - "Status": Equal(metav1.ConditionTrue), - })), - ) - err = k8sClient.Get(ctx, client.ObjectKeyFromObject(dnsRecord2), dnsRecord2) - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(dnsRecord2.Status.Conditions).To( - ContainElement(MatchFields(IgnoreExtras, Fields{ - "Type": Equal(string(v1alpha1.ConditionTypeReady)), - "Status": Equal(metav1.ConditionTrue), - })), - ) - }, time.Minute, 10*time.Second, ctx).Should(Succeed()) - - By("checking dns records ownerID is set correctly") - Expect(dnsRecord1.Spec.OwnerID).To(BeEmpty()) - Expect(dnsRecord2.Spec.OwnerID).To(BeEmpty()) - Expect(dnsRecord1.Status.OwnerID).ToNot(BeEmpty()) - Expect(dnsRecord2.Status.OwnerID).ToNot(BeEmpty()) - Expect(dnsRecord1.Status.OwnerID).To(Equal(dnsRecord1.GetUIDHash())) - Expect(dnsRecord2.Status.OwnerID).To(Equal(dnsRecord2.GetUIDHash())) - - testProvider, err := ProviderForManagedZone(ctx, testManagedZone, k8sClient) + for _, tr := range testRecords { + err := tr.cluster.k8sClient.Get(ctx, client.ObjectKeyFromObject(tr.record), tr.record) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(tr.record.Status.Conditions).To( + ContainElement(MatchFields(IgnoreExtras, Fields{ + "Type": Equal(string(v1alpha1.ConditionTypeReady)), + "Status": Equal(metav1.ConditionTrue), + })), + ) + } + }, recordsReadyMaxDuration, 5*time.Second, ctx).Should(Succeed()) + + By("checking provider zone records are created as expected") + testProvider, err := ProviderForManagedZone(ctx, testClusters[0].testManagedZones[0], testClusters[0].k8sClient) Expect(err).NotTo(HaveOccurred()) - By("ensuring zone records are created as expected") zoneEndpoints, err := EndpointsForHost(ctx, testProvider, testHostname) Expect(err).NotTo(HaveOccurred()) - if testDNSProvider == "gcp" { - Expect(zoneEndpoints).To(HaveLen(12)) - //Main Records - By("checking endpoint " + testHostname) - Expect(zoneEndpoints).To(ContainElement(PointTo(MatchFields(IgnoreExtras, Fields{ - "DNSName": Equal(testHostname), - "Targets": ConsistOf(klbHostName), - "RecordType": Equal("CNAME"), - "SetIdentifier": Equal(""), - "RecordTTL": Equal(externaldnsendpoint.TTL(300)), - })))) - By("checking endpoint " + klbHostName) - Expect(zoneEndpoints).To(ContainElement(PointTo(MatchFields(IgnoreExtras, Fields{ - "DNSName": Equal(klbHostName), - "Targets": ConsistOf(geo1KlbHostName, geo2KlbHostName), - "RecordType": Equal("CNAME"), - "SetIdentifier": Equal(""), - "RecordTTL": Equal(externaldnsendpoint.TTL(300)), - "ProviderSpecific": ContainElements( - externaldnsendpoint.ProviderSpecificProperty{Name: "routingpolicy", Value: "geo"}, - externaldnsendpoint.ProviderSpecificProperty{Name: geo1KlbHostName, Value: geoCode1}, - externaldnsendpoint.ProviderSpecificProperty{Name: geo2KlbHostName, Value: geoCode2}, - ), - })))) - By("checking endpoint " + geo1KlbHostName) - Expect(zoneEndpoints).To(ContainElement(PointTo(MatchFields(IgnoreExtras, Fields{ - "DNSName": Equal(geo1KlbHostName), - "Targets": ConsistOf(cluster1KlbHostName), - "RecordType": Equal("CNAME"), - "SetIdentifier": Equal(""), - "RecordTTL": Equal(externaldnsendpoint.TTL(60)), - "ProviderSpecific": ContainElements( - externaldnsendpoint.ProviderSpecificProperty{Name: "routingpolicy", Value: "weighted"}, - externaldnsendpoint.ProviderSpecificProperty{Name: cluster1KlbHostName, Value: "200"}, - ), - })))) - By("checking endpoint " + geo2KlbHostName) - Expect(zoneEndpoints).To(ContainElement(PointTo(MatchFields(IgnoreExtras, Fields{ - "DNSName": Equal(geo2KlbHostName), - "Targets": ConsistOf(cluster2KlbHostName), - "RecordType": Equal("CNAME"), - "SetIdentifier": Equal(""), - "RecordTTL": Equal(externaldnsendpoint.TTL(60)), - "ProviderSpecific": ContainElements( - externaldnsendpoint.ProviderSpecificProperty{Name: "routingpolicy", Value: "weighted"}, - externaldnsendpoint.ProviderSpecificProperty{Name: cluster2KlbHostName, Value: "200"}, - ), - })))) - By("checking endpoint " + cluster1KlbHostName) - Expect(zoneEndpoints).To(ContainElement(PointTo(MatchFields(IgnoreExtras, Fields{ - "DNSName": Equal(cluster1KlbHostName), - "Targets": ConsistOf(testTargetIP1), - "RecordType": Equal("A"), - "SetIdentifier": Equal(""), - "RecordTTL": Equal(externaldnsendpoint.TTL(60)), - })))) - By("checking endpoint " + cluster2KlbHostName) - Expect(zoneEndpoints).To(ContainElement(PointTo(MatchFields(IgnoreExtras, Fields{ - "DNSName": Equal(cluster2KlbHostName), - "Targets": ConsistOf(testTargetIP2), - "RecordType": Equal("A"), - "SetIdentifier": Equal(""), - "RecordTTL": Equal(externaldnsendpoint.TTL(60)), - })))) - //Txt Records - By("checking TXT owner endpoints") - Expect(zoneEndpoints).To(ContainElement(PointTo(MatchFields(IgnoreExtras, Fields{ - "DNSName": Equal("kuadrant-cname-" + testHostname), - "Targets": ContainElement(And( - ContainSubstring("heritage=external-dns,external-dns/owner="), - ContainSubstring(dnsRecord1.Status.OwnerID), - ContainSubstring(dnsRecord2.Status.OwnerID), - )), - "RecordType": Equal("TXT"), - "SetIdentifier": Equal(""), - "RecordTTL": Equal(externaldnsendpoint.TTL(300)), - })))) - Expect(zoneEndpoints).To(ContainElement(PointTo(MatchFields(IgnoreExtras, Fields{ - "DNSName": Equal("kuadrant-cname-" + klbHostName), - "Targets": ContainElement(And( + var expectedEndpointsLen int + if testDNSProvider == "google" { + expectedEndpointsLen = (2 + len(testGeoRecords) + len(testRecords)) * 2 + Expect(zoneEndpoints).To(HaveLen(expectedEndpointsLen)) + } else if testDNSProvider == "aws" { + expectedEndpointsLen = (2 + len(testGeoRecords) + (len(testRecords) * 2)) * 2 + Expect(zoneEndpoints).To(HaveLen(expectedEndpointsLen)) + } + + var totalEndpointsChecked = 0 + var allOwners = []string{} + var allOwnerMatcher = []types.GomegaMatcher{ + ContainSubstring("heritage=external-dns,external-dns/owner="), + } + var geoOwners = map[string][]string{} + var geoKlbHostname = map[string]string{} + var geoOwnerMatcher = map[string][]types.GomegaMatcher{} + for i := range testRecords { + ownerID := testRecords[i].record.Status.OwnerID + allOwners = append(allOwners, ownerID) + allOwnerMatcher = append(allOwnerMatcher, ContainSubstring(ownerID)) + + geoCode := testRecords[i].config.testGeoCode + geoOwners[geoCode] = append(geoOwners[geoCode], ownerID) + geoKlbHostname[geoCode] = testRecords[i].config.hostnames.geoKlb + if _, ok := geoOwnerMatcher[geoCode]; !ok { + geoOwnerMatcher[geoCode] = []types.GomegaMatcher{ ContainSubstring("heritage=external-dns,external-dns/owner="), - ContainSubstring(dnsRecord1.Status.OwnerID), - ContainSubstring(dnsRecord2.Status.OwnerID), - )), - "RecordType": Equal("TXT"), - "SetIdentifier": Equal(""), - "RecordTTL": Equal(externaldnsendpoint.TTL(300)), - })))) - Expect(zoneEndpoints).To(ContainElement(PointTo(MatchFields(IgnoreExtras, Fields{ - "DNSName": Equal("kuadrant-cname-" + geo1KlbHostName), - "Targets": ConsistOf("\"heritage=external-dns,external-dns/owner=" + dnsRecord1.Status.OwnerID + "\""), - "RecordType": Equal("TXT"), - "SetIdentifier": Equal(""), - "RecordTTL": Equal(externaldnsendpoint.TTL(300)), - })))) - Expect(zoneEndpoints).To(ContainElement(PointTo(MatchFields(IgnoreExtras, Fields{ - "DNSName": Equal("kuadrant-cname-" + geo2KlbHostName), - "Targets": ConsistOf("\"heritage=external-dns,external-dns/owner=" + dnsRecord2.Status.OwnerID + "\""), - "RecordType": Equal("TXT"), - "SetIdentifier": Equal(""), - "RecordTTL": Equal(externaldnsendpoint.TTL(300)), - })))) + } + } + geoOwnerMatcher[geoCode] = append(geoOwnerMatcher[geoCode], ContainSubstring(ownerID)) + } + + By("[Common] checking common endpoints") + // A CNAME record for testHostname should always exist and be owned by all endpoints + By("[Common] checking " + testHostname + " endpoint") + Expect(zoneEndpoints).To(ContainElement(PointTo(MatchFields(IgnoreExtras, Fields{ + "DNSName": Equal(testHostname), + "Targets": ConsistOf(testRecords[0].config.hostnames.klb), + "RecordType": Equal("CNAME"), + "SetIdentifier": Equal(""), + "RecordTTL": Equal(externaldnsendpoint.TTL(300)), + })))) + totalEndpointsChecked++ + By("[Common] checking " + testHostname + " TXT owner endpoint") + Expect(zoneEndpoints).To(ContainElement(PointTo(MatchFields(IgnoreExtras, Fields{ + "DNSName": Equal("kuadrant-cname-" + testHostname), + "Targets": ContainElement(And(allOwnerMatcher...)), + "RecordType": Equal("TXT"), + "SetIdentifier": Equal(""), + "RecordTTL": Equal(externaldnsendpoint.TTL(300)), + })))) + totalEndpointsChecked++ + + By("[Geo] checking geo endpoints") + if testDNSProvider == "google" { + // A CNAME record for klbHostName should always exist, be owned by all endpoints and target all geo hostnames + klbHostName := testRecords[0].config.hostnames.klb + + allKlbGeoHostnames := []string{} + gcpGeoProps := []externaldnsendpoint.ProviderSpecificProperty{ + {Name: "routingpolicy", Value: "geo"}, + } + for g, h := range geoKlbHostname { + allKlbGeoHostnames = append(allKlbGeoHostnames, h) + gcpGeoProps = append(gcpGeoProps, externaldnsendpoint.ProviderSpecificProperty{Name: h, Value: g}) + } + + By("[Geo] checking " + klbHostName + " endpoint") Expect(zoneEndpoints).To(ContainElement(PointTo(MatchFields(IgnoreExtras, Fields{ - "DNSName": Equal("kuadrant-a-" + cluster1KlbHostName), - "Targets": ConsistOf("\"heritage=external-dns,external-dns/owner=" + dnsRecord1.Status.OwnerID + "\""), - "RecordType": Equal("TXT"), - "SetIdentifier": Equal(""), - "RecordTTL": Equal(externaldnsendpoint.TTL(300)), + "DNSName": Equal(klbHostName), + "Targets": ConsistOf(allKlbGeoHostnames), + "RecordType": Equal("CNAME"), + "SetIdentifier": Equal(""), + "RecordTTL": Equal(externaldnsendpoint.TTL(300)), + "ProviderSpecific": ContainElements(gcpGeoProps), })))) + totalEndpointsChecked++ + By("[Geo] checking " + klbHostName + " TXT owner endpoint") Expect(zoneEndpoints).To(ContainElement(PointTo(MatchFields(IgnoreExtras, Fields{ - "DNSName": Equal("kuadrant-a-" + cluster2KlbHostName), - "Targets": ConsistOf("\"heritage=external-dns,external-dns/owner=" + dnsRecord2.Status.OwnerID + "\""), + "DNSName": Equal("kuadrant-cname-" + klbHostName), + "Targets": ContainElement(And(allOwnerMatcher...)), "RecordType": Equal("TXT"), "SetIdentifier": Equal(""), "RecordTTL": Equal(externaldnsendpoint.TTL(300)), })))) + totalEndpointsChecked++ } if testDNSProvider == "aws" { - Expect(zoneEndpoints).To(HaveLen(16)) - //Main Records - By("checking endpoint " + testHostname) - Expect(zoneEndpoints).To(ContainElement(PointTo(MatchFields(IgnoreExtras, Fields{ - "DNSName": Equal(testHostname), - "Targets": ConsistOf(klbHostName), - "RecordType": Equal("CNAME"), - "SetIdentifier": Equal(""), - "RecordTTL": Equal(externaldnsendpoint.TTL(300)), - })))) - By("checking endpoint " + klbHostName + " - " + geoCode1) - Expect(zoneEndpoints).To(ContainElement(PointTo(MatchFields(IgnoreExtras, Fields{ - "DNSName": Equal(klbHostName), - "Targets": ConsistOf(geo1KlbHostName), - "RecordType": Equal("CNAME"), - "SetIdentifier": Equal(geoCode1), - "RecordTTL": Equal(externaldnsendpoint.TTL(300)), - "ProviderSpecific": Equal(externaldnsendpoint.ProviderSpecific{ - {Name: "alias", Value: "false"}, - {Name: "aws/geolocation-country-code", Value: "US"}, - }), - })))) - By("checking endpoint " + klbHostName + " - " + geoCode2) - Expect(zoneEndpoints).To(ContainElement(PointTo(MatchFields(IgnoreExtras, Fields{ - "DNSName": Equal(klbHostName), - "Targets": ConsistOf(geo2KlbHostName), - "RecordType": Equal("CNAME"), - "SetIdentifier": Equal(geoCode2), - "RecordTTL": Equal(externaldnsendpoint.TTL(300)), - "ProviderSpecific": Equal(externaldnsendpoint.ProviderSpecific{ - {Name: "alias", Value: "false"}, - {Name: "aws/geolocation-continent-code", Value: "EU"}, - }), - })))) - By("checking endpoint " + klbHostName + " - " + geoCode1) + // A CNAME record for klbHostName should exist for each geo and be owned by all endpoints in that geo + klbHostName := testRecords[0].config.hostnames.klb + for geoCode, geoRecords := range testGeoRecords { + geoKlbHostName := geoRecords[0].config.hostnames.geoKlb + + By("[Geo] checking " + klbHostName + " -> " + geoCode + " -> " + geoKlbHostName + " - endpoint") + + awsGeoCodeKey := "aws/geolocation-country-code" + if !provider.IsISO3166Alpha2Code(geoCode) { + awsGeoCodeKey = "aws/geolocation-continent-code" + } + + Expect(zoneEndpoints).To(ContainElement(PointTo(MatchFields(IgnoreExtras, Fields{ + "DNSName": Equal(klbHostName), + "Targets": ConsistOf(geoKlbHostName), + "RecordType": Equal("CNAME"), + "SetIdentifier": Equal(geoCode), + "RecordTTL": Equal(externaldnsendpoint.TTL(300)), + "ProviderSpecific": Equal(externaldnsendpoint.ProviderSpecific{ + {Name: "alias", Value: "false"}, + {Name: awsGeoCodeKey, Value: geoCode}, + }), + })))) + totalEndpointsChecked++ + By("[Geo] checking " + klbHostName + " -> " + geoCode + " - TXT owner endpoint") + Expect(zoneEndpoints).To(ContainElement(PointTo(MatchFields(IgnoreExtras, Fields{ + "DNSName": Equal("kuadrant-cname-" + klbHostName), + "Targets": ContainElement(And(geoOwnerMatcher[geoCode]...)), + "RecordType": Equal("TXT"), + "SetIdentifier": Equal(geoCode), + "RecordTTL": Equal(externaldnsendpoint.TTL(300)), + "ProviderSpecific": Equal(externaldnsendpoint.ProviderSpecific{ + {Name: awsGeoCodeKey, Value: geoCode}, + }), + })))) + totalEndpointsChecked++ + } + + defaultGeoKlbHostName := testRecords[0].config.hostnames.defaultGeoKlb + defaultGeoCode := testRecords[0].config.testDefaultGeoCode + + By("[Geo] checking endpoint " + klbHostName + " -> default") Expect(zoneEndpoints).To(ContainElement(PointTo(MatchFields(IgnoreExtras, Fields{ "DNSName": Equal(klbHostName), - "Targets": ConsistOf(geo1KlbHostName), + "Targets": ConsistOf(defaultGeoKlbHostName), "RecordType": Equal("CNAME"), "SetIdentifier": Equal("default"), "RecordTTL": Equal(externaldnsendpoint.TTL(300)), @@ -616,82 +537,11 @@ var _ = Describe("Multi Record Test", func() { {Name: "aws/geolocation-country-code", Value: "*"}, }), })))) - By("checking endpoint " + geo1KlbHostName) - Expect(zoneEndpoints).To(ContainElement(PointTo(MatchFields(IgnoreExtras, Fields{ - "DNSName": Equal(geo1KlbHostName), - "Targets": ConsistOf(cluster1KlbHostName), - "RecordType": Equal("CNAME"), - "SetIdentifier": Equal(cluster1KlbHostName), - "RecordTTL": Equal(externaldnsendpoint.TTL(60)), - "ProviderSpecific": Equal(externaldnsendpoint.ProviderSpecific{ - {Name: "alias", Value: "false"}, - {Name: "aws/weight", Value: "200"}, - }), - })))) - By("checking endpoint " + geo2KlbHostName) - Expect(zoneEndpoints).To(ContainElement(PointTo(MatchFields(IgnoreExtras, Fields{ - "DNSName": Equal(geo2KlbHostName), - "Targets": ConsistOf(cluster2KlbHostName), - "RecordType": Equal("CNAME"), - "SetIdentifier": Equal(cluster2KlbHostName), - "RecordTTL": Equal(externaldnsendpoint.TTL(60)), - "ProviderSpecific": Equal(externaldnsendpoint.ProviderSpecific{ - {Name: "alias", Value: "false"}, - {Name: "aws/weight", Value: "200"}, - }), - })))) - By("checking endpoint " + cluster1KlbHostName) - Expect(zoneEndpoints).To(ContainElement(PointTo(MatchFields(IgnoreExtras, Fields{ - "DNSName": Equal(cluster1KlbHostName), - "Targets": ConsistOf(testTargetIP1), - "RecordType": Equal("A"), - "SetIdentifier": Equal(""), - "RecordTTL": Equal(externaldnsendpoint.TTL(60)), - })))) - By("checking endpoint " + cluster2KlbHostName) - Expect(zoneEndpoints).To(ContainElement(PointTo(MatchFields(IgnoreExtras, Fields{ - "DNSName": Equal(cluster2KlbHostName), - "Targets": ConsistOf(testTargetIP2), - "RecordType": Equal("A"), - "SetIdentifier": Equal(""), - "RecordTTL": Equal(externaldnsendpoint.TTL(60)), - })))) - //Txt Records - By("checking TXT owner endpoints") - Expect(zoneEndpoints).To(ContainElement(PointTo(MatchFields(IgnoreExtras, Fields{ - "DNSName": Equal("kuadrant-cname-" + testHostname), - "Targets": ContainElement(And( - ContainSubstring("heritage=external-dns,external-dns/owner="), - ContainSubstring(dnsRecord1.Status.OwnerID), - ContainSubstring(dnsRecord2.Status.OwnerID), - )), - "RecordType": Equal("TXT"), - "SetIdentifier": Equal(""), - "RecordTTL": Equal(externaldnsendpoint.TTL(300)), - })))) - Expect(zoneEndpoints).To(ContainElement(PointTo(MatchFields(IgnoreExtras, Fields{ - "DNSName": Equal("kuadrant-cname-" + klbHostName), - "Targets": ConsistOf("\"heritage=external-dns,external-dns/owner=" + dnsRecord1.Status.OwnerID + "\""), - "RecordType": Equal("TXT"), - "SetIdentifier": Equal(geoCode1), - "RecordTTL": Equal(externaldnsendpoint.TTL(300)), - "ProviderSpecific": Equal(externaldnsendpoint.ProviderSpecific{ - {Name: "aws/geolocation-country-code", Value: "US"}, - }), - })))) - Expect(zoneEndpoints).To(ContainElement(PointTo(MatchFields(IgnoreExtras, Fields{ - "DNSName": Equal("kuadrant-cname-" + klbHostName), - "Targets": ConsistOf("\"heritage=external-dns,external-dns/owner=" + dnsRecord2.Status.OwnerID + "\""), - "RecordType": Equal("TXT"), - "SetIdentifier": Equal(geoCode2), - "RecordTTL": Equal(externaldnsendpoint.TTL(300)), - "ProviderSpecific": Equal(externaldnsendpoint.ProviderSpecific{ - {Name: "aws/geolocation-continent-code", Value: "EU"}, - }), - })))) + totalEndpointsChecked++ + By("[Geo] checking " + klbHostName + " -> default - TXT owner endpoint") Expect(zoneEndpoints).To(ContainElement(PointTo(MatchFields(IgnoreExtras, Fields{ "DNSName": Equal("kuadrant-cname-" + klbHostName), - "Targets": ConsistOf("\"heritage=external-dns,external-dns/owner=" + dnsRecord1.Status.OwnerID + "\""), + "Targets": ContainElement(And(geoOwnerMatcher[defaultGeoCode]...)), "RecordType": Equal("TXT"), "SetIdentifier": Equal("default"), "RecordTTL": Equal(externaldnsendpoint.TTL(300)), @@ -699,88 +549,127 @@ var _ = Describe("Multi Record Test", func() { {Name: "aws/geolocation-country-code", Value: "*"}, }), })))) + totalEndpointsChecked++ + } + + By("[Weight] checking weighted endpoints") + if testDNSProvider == "google" { + // A weighted CNAME record should exist for each geo, be owned by all endpoints in that geo, and target the hostname of all clusters in that geo + for geoCode, geoRecords := range testGeoRecords { + geoKlbHostName := geoRecords[0].config.hostnames.geoKlb + + allGeoClusterHostnames := []string{} + gcpWeightProps := []externaldnsendpoint.ProviderSpecificProperty{ + {Name: "routingpolicy", Value: "weighted"}, + } + for i := range geoRecords { + geoClusterHostname := geoRecords[i].config.hostnames.clusterKlb + allGeoClusterHostnames = append(allGeoClusterHostnames, geoClusterHostname) + gcpWeightProps = append(gcpWeightProps, externaldnsendpoint.ProviderSpecificProperty{Name: geoClusterHostname, Value: "200"}) + } + + By("[Weight] checking " + geoKlbHostName + " endpoint") + Expect(zoneEndpoints).To(ContainElement(PointTo(MatchFields(IgnoreExtras, Fields{ + "DNSName": Equal(geoKlbHostName), + "Targets": ConsistOf(allGeoClusterHostnames), + "RecordType": Equal("CNAME"), + "SetIdentifier": Equal(""), + "RecordTTL": Equal(externaldnsendpoint.TTL(60)), + "ProviderSpecific": ContainElements(gcpWeightProps), + })))) + totalEndpointsChecked++ + By("[Weight] checking " + geoKlbHostName + " TXT owner endpoint") + Expect(zoneEndpoints).To(ContainElement(PointTo(MatchFields(IgnoreExtras, Fields{ + "DNSName": Equal("kuadrant-cname-" + geoKlbHostName), + "Targets": ContainElement(And(geoOwnerMatcher[geoCode]...)), + "RecordType": Equal("TXT"), + "SetIdentifier": Equal(""), + "RecordTTL": Equal(externaldnsendpoint.TTL(300)), + })))) + totalEndpointsChecked++ + } + } + if testDNSProvider == "aws" { + // A weighted CNAME record should exist for each dns record in each geo and be owned only by that endpoint + for _, geoRecords := range testGeoRecords { + geoKlbHostName := geoRecords[0].config.hostnames.geoKlb + for i := range geoRecords { + clusterKlbHostName := geoRecords[i].config.hostnames.clusterKlb + ownerID := geoRecords[i].record.Status.OwnerID + By("[Weight] checking " + geoKlbHostName + " -> " + clusterKlbHostName + " - endpoint") + Expect(zoneEndpoints).To(ContainElement(PointTo(MatchFields(IgnoreExtras, Fields{ + "DNSName": Equal(geoKlbHostName), + "Targets": ConsistOf(clusterKlbHostName), + "RecordType": Equal("CNAME"), + "SetIdentifier": Equal(clusterKlbHostName), + "RecordTTL": Equal(externaldnsendpoint.TTL(60)), + "ProviderSpecific": Equal(externaldnsendpoint.ProviderSpecific{ + {Name: "alias", Value: "false"}, + {Name: "aws/weight", Value: "200"}, + }), + })))) + totalEndpointsChecked++ + By("[Weight] checking " + geoKlbHostName + " -> " + clusterKlbHostName + " -> " + ownerID + " TXT owner endpoint") + Expect(zoneEndpoints).To(ContainElement(PointTo(MatchFields(IgnoreExtras, Fields{ + "DNSName": Equal("kuadrant-cname-" + geoKlbHostName), + "Targets": ConsistOf("\"heritage=external-dns,external-dns/owner=" + ownerID + "\""), + "RecordType": Equal("TXT"), + "SetIdentifier": Equal(clusterKlbHostName), + "RecordTTL": Equal(externaldnsendpoint.TTL(300)), + "ProviderSpecific": Equal(externaldnsendpoint.ProviderSpecific{ + {Name: "aws/weight", Value: "200"}, + }), + })))) + totalEndpointsChecked++ + } + } + } + + By("[Cluster] checking cluster endpoints") + // An A record with the cluster target IP should exist for each dns record and owned only by that endpoint + for i := range testRecords { + clusterKlbHostName := testRecords[i].config.hostnames.clusterKlb + clusterTargetIP := testRecords[i].config.testTargetIP + ownerID := testRecords[i].record.Status.OwnerID + By("[Cluster] checking " + clusterKlbHostName + " endpoint") Expect(zoneEndpoints).To(ContainElement(PointTo(MatchFields(IgnoreExtras, Fields{ - "DNSName": Equal("kuadrant-cname-" + geo1KlbHostName), - "Targets": ConsistOf("\"heritage=external-dns,external-dns/owner=" + dnsRecord1.Status.OwnerID + "\""), - "RecordType": Equal("TXT"), - "SetIdentifier": Equal(cluster1KlbHostName), - "RecordTTL": Equal(externaldnsendpoint.TTL(300)), - "ProviderSpecific": Equal(externaldnsendpoint.ProviderSpecific{ - {Name: "aws/weight", Value: "200"}, - }), - })))) - Expect(zoneEndpoints).To(ContainElement(PointTo(MatchFields(IgnoreExtras, Fields{ - "DNSName": Equal("kuadrant-cname-" + geo2KlbHostName), - "Targets": ConsistOf("\"heritage=external-dns,external-dns/owner=" + dnsRecord2.Status.OwnerID + "\""), - "RecordType": Equal("TXT"), - "SetIdentifier": Equal(cluster2KlbHostName), - "RecordTTL": Equal(externaldnsendpoint.TTL(300)), - "ProviderSpecific": Equal(externaldnsendpoint.ProviderSpecific{ - {Name: "aws/weight", Value: "200"}, - }), - })))) - Expect(zoneEndpoints).To(ContainElement(PointTo(MatchFields(IgnoreExtras, Fields{ - "DNSName": Equal("kuadrant-a-" + cluster1KlbHostName), - "Targets": ConsistOf("\"heritage=external-dns,external-dns/owner=" + dnsRecord1.Status.OwnerID + "\""), - "RecordType": Equal("TXT"), + "DNSName": Equal(clusterKlbHostName), + "Targets": ConsistOf(clusterTargetIP), + "RecordType": Equal("A"), "SetIdentifier": Equal(""), - "RecordTTL": Equal(externaldnsendpoint.TTL(300)), + "RecordTTL": Equal(externaldnsendpoint.TTL(60)), })))) + totalEndpointsChecked++ + By("[Cluster] checking " + clusterKlbHostName + " TXT owner endpoint") Expect(zoneEndpoints).To(ContainElement(PointTo(MatchFields(IgnoreExtras, Fields{ - "DNSName": Equal("kuadrant-a-" + cluster2KlbHostName), - "Targets": ConsistOf("\"heritage=external-dns,external-dns/owner=" + dnsRecord2.Status.OwnerID + "\""), + "DNSName": Equal("kuadrant-a-" + clusterKlbHostName), + "Targets": ConsistOf("\"heritage=external-dns,external-dns/owner=" + ownerID + "\""), "RecordType": Equal("TXT"), "SetIdentifier": Equal(""), "RecordTTL": Equal(externaldnsendpoint.TTL(300)), })))) + totalEndpointsChecked++ } - By("ensuring the authoritative nameserver resolves the hostname") - // speed up things by using the authoritative nameserver - authoritativeResolver := ResolverForDomainName(testZoneDomainName) - Eventually(func(g Gomega, ctx context.Context) { - ips, err := authoritativeResolver.LookupHost(ctx, testHostname) - g.Expect(err).NotTo(HaveOccurred()) - GinkgoWriter.Printf("[debug] ips: %v\n", ips) - g.Expect(ips).To(Or(ContainElement(testTargetIP1), ContainElement(testTargetIP2))) - }, 300*time.Second, 10*time.Second, ctx).Should(Succeed()) - - By("deleting dnsrecord " + dnsRecord2.Name) - err = k8sClient.Delete(ctx, dnsRecord2, - client.PropagationPolicy(metav1.DeletePropagationForeground)) - Expect(client.IgnoreNotFound(err)).ToNot(HaveOccurred()) + By("checking all endpoints were validated") + Expect(totalEndpointsChecked).To(Equal(expectedEndpointsLen)) - By("checking dnsrecord " + dnsRecord2.Name + " is removed") - Eventually(func(g Gomega, ctx context.Context) { - err := k8sClient.Get(ctx, client.ObjectKeyFromObject(dnsRecord2), dnsRecord2) - g.Expect(err).To(HaveOccurred()) - g.Expect(err).To(MatchError(ContainSubstring("not found"))) - }, 10*time.Second, 1*time.Second, ctx).Should(Succeed()) - - By("ensuring the authoritative nameserver resolves the hostname") - Eventually(func(g Gomega, ctx context.Context) { - ips, err := authoritativeResolver.LookupHost(ctx, testHostname) - g.Expect(err).NotTo(HaveOccurred()) - GinkgoWriter.Printf("[debug] ips: %v\n", ips) - g.Expect(ips).To(ConsistOf(testTargetIP1)) - }, 300*time.Second, 10*time.Second, ctx).Should(Succeed()) - - By("ensuring zone records are updated as expected") - //ToDo mnairn Add more checks in here - - By("deleting dnsrecord " + dnsRecord1.Name) - err = k8sClient.Delete(ctx, dnsRecord1, - client.PropagationPolicy(metav1.DeletePropagationForeground)) - Expect(client.IgnoreNotFound(err)).ToNot(HaveOccurred()) + By("deleting all remaining dns records") + for _, tr := range testRecords { + err := tr.cluster.k8sClient.Delete(ctx, tr.record, + client.PropagationPolicy(metav1.DeletePropagationForeground)) + Expect(client.IgnoreNotFound(err)).ToNot(HaveOccurred()) + } - By("checking dnsrecord " + dnsRecord1.Name + " is removed") + By(fmt.Sprintf("checking all dns records are removed within %s", recordsRemovedMaxDuration)) Eventually(func(g Gomega, ctx context.Context) { - err := k8sClient.Get(ctx, client.ObjectKeyFromObject(dnsRecord1), dnsRecord1) - g.Expect(err).To(HaveOccurred()) - g.Expect(err).To(MatchError(ContainSubstring("not found"))) - }, 10*time.Second, 1*time.Second, ctx).Should(Succeed()) + for _, tr := range testRecords { + err := tr.cluster.k8sClient.Get(ctx, client.ObjectKeyFromObject(tr.record), tr.record) + g.Expect(err).To(MatchError(ContainSubstring("not found"))) + } + }, recordsRemovedMaxDuration, 5*time.Second, ctx).Should(Succeed()) - By("ensuring zone records are all removed as expected") + By("checking provider zone records are all removed") Eventually(func(g Gomega, ctx context.Context) { zoneEndpoints, err := EndpointsForHost(ctx, testProvider, testHostname) g.Expect(err).NotTo(HaveOccurred()) diff --git a/test/e2e/provider_errors_test.go b/test/e2e/provider_errors_test.go index 9a7efb73..a746c66e 100644 --- a/test/e2e/provider_errors_test.go +++ b/test/e2e/provider_errors_test.go @@ -30,12 +30,17 @@ var _ = Describe("DNSRecord Provider Errors", Labels{"provider_errors"}, func() // testHostname generated hostname for this test e.g. t-gw-mgc-12345.t-e2e-12345.e2e.hcpapps.net var testHostname string + var k8sClient client.Client + var testManagedZone *v1alpha1.ManagedZone + var dnsRecord *v1alpha1.DNSRecord BeforeEach(func(ctx SpecContext) { testID = "t-errors-" + GenerateName() testDomainName = strings.Join([]string{testSuiteID, testZoneDomainName}, ".") testHostname = strings.Join([]string{testID, testDomainName}, ".") + k8sClient = testClusters[0].k8sClient + testManagedZone = testClusters[0].testManagedZones[0] }) AfterEach(func(ctx SpecContext) { @@ -49,8 +54,8 @@ var _ = Describe("DNSRecord Provider Errors", Labels{"provider_errors"}, func() It("correctly handles invalid geo", func(ctx SpecContext) { var validGeoCode string var expectedProviderErr string - if testDNSProvider == "gcp" { - //GCP + if testDNSProvider == "google" { + //Google expectedProviderErr = "location': 'notageocode', invalid" validGeoCode = "us-east1" } else if testDNSProvider == "azure" { @@ -80,7 +85,7 @@ var _ = Describe("DNSRecord Provider Errors", Labels{"provider_errors"}, func() invalidEndpoint, } - dnsRecord = testBuildDNSRecord(testID, testNamespace, testManagedZoneName, "test-owner", testHostname) + dnsRecord = testBuildDNSRecord(testID, testManagedZone.Namespace, testManagedZone.Name, "test-owner", testHostname) dnsRecord.Spec.Endpoints = testEndpoints By("creating dnsrecord " + dnsRecord.Name + " with invalid geo endpoint") @@ -141,8 +146,8 @@ var _ = Describe("DNSRecord Provider Errors", Labels{"provider_errors"}, func() It("correctly handles invalid weight", func(ctx SpecContext) { var expectedProviderErr string - if testDNSProvider == "gcp" { - //GCP + if testDNSProvider == "google" { + //Google expectedProviderErr = "weight': '-1.0' Reason: backendError, Message: Invalid Value" } else if testDNSProvider == "azure" { Skip("not yet supported for azure") @@ -171,7 +176,7 @@ var _ = Describe("DNSRecord Provider Errors", Labels{"provider_errors"}, func() invalidEndpoint, } - dnsRecord = testBuildDNSRecord(testID, testNamespace, testManagedZoneName, "test-owner", testHostname) + dnsRecord = testBuildDNSRecord(testID, testManagedZone.Namespace, testManagedZone.Name, "test-owner", testHostname) dnsRecord.Spec.Endpoints = testEndpoints By("creating dnsrecord " + dnsRecord.Name + " with invalid weight endpoint") diff --git a/test/e2e/single_record_test.go b/test/e2e/single_record_test.go index 5616a039..9caf73ef 100644 --- a/test/e2e/single_record_test.go +++ b/test/e2e/single_record_test.go @@ -30,15 +30,19 @@ var _ = Describe("Single Record Test", func() { // testHostname generated hostname for this test e.g. t-gw-mgc-12345.t-e2e-12345.e2e.hcpapps.net var testHostname string - var dnsRecord *v1alpha1.DNSRecord + var k8sClient client.Client + var testManagedZone *v1alpha1.ManagedZone var geoCode string + var dnsRecord *v1alpha1.DNSRecord + BeforeEach(func(ctx SpecContext) { testID = "t-single-" + GenerateName() testDomainName = strings.Join([]string{testSuiteID, testZoneDomainName}, ".") testHostname = strings.Join([]string{testID, testDomainName}, ".") - - if testDNSProvider == "gcp" { + k8sClient = testClusters[0].k8sClient + testManagedZone = testClusters[0].testManagedZones[0] + if testDNSProvider == "google" { geoCode = "us-east1" } else { geoCode = "US" @@ -61,12 +65,12 @@ var _ = Describe("Single Record Test", func() { dnsRecord = &v1alpha1.DNSRecord{ ObjectMeta: metav1.ObjectMeta{ Name: testID, - Namespace: testNamespace, + Namespace: testManagedZone.Namespace, }, Spec: v1alpha1.DNSRecordSpec{ RootHost: testWCHostname, ManagedZoneRef: &v1alpha1.ManagedZoneReference{ - Name: testManagedZoneName, + Name: testManagedZone.Name, }, Endpoints: []*externaldnsendpoint.Endpoint{ { @@ -156,12 +160,12 @@ var _ = Describe("Single Record Test", func() { dnsRecord = &v1alpha1.DNSRecord{ ObjectMeta: metav1.ObjectMeta{ Name: testID, - Namespace: testNamespace, + Namespace: testManagedZone.Namespace, }, Spec: v1alpha1.DNSRecordSpec{ RootHost: testHostname, ManagedZoneRef: &v1alpha1.ManagedZoneReference{ - Name: testManagedZoneName, + Name: testManagedZone.Name, }, Endpoints: []*externaldnsendpoint.Endpoint{ { @@ -246,12 +250,12 @@ var _ = Describe("Single Record Test", func() { dnsRecord = &v1alpha1.DNSRecord{ ObjectMeta: metav1.ObjectMeta{ Name: testID, - Namespace: testNamespace, + Namespace: testManagedZone.Namespace, }, Spec: v1alpha1.DNSRecordSpec{ RootHost: testHostname, ManagedZoneRef: &v1alpha1.ManagedZoneReference{ - Name: testManagedZoneName, + Name: testManagedZone.Name, }, Endpoints: []*externaldnsendpoint.Endpoint{ { @@ -345,7 +349,7 @@ var _ = Describe("Single Record Test", func() { Expect(err).NotTo(HaveOccurred()) zoneEndpoints, err := EndpointsForHost(ctx, testProvider, testHostname) Expect(err).NotTo(HaveOccurred()) - if testDNSProvider == "gcp" { + if testDNSProvider == "google" { Expect(zoneEndpoints).To(HaveLen(8)) Expect(zoneEndpoints).To(ContainElements( PointTo(MatchFields(IgnoreExtras, Fields{ diff --git a/test/e2e/suite_test.go b/test/e2e/suite_test.go index 08f706a4..9541c8f4 100644 --- a/test/e2e/suite_test.go +++ b/test/e2e/suite_test.go @@ -1,16 +1,21 @@ -//go:build e2e || multi_instance +//go:build e2e package e2e import ( + "context" "fmt" "os" - "slices" + "strconv" + "strings" "testing" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + . "github.com/onsi/gomega/gstruct" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/scheme" _ "k8s.io/client-go/plugin/pkg/client/auth" "k8s.io/client-go/tools/clientcmd" @@ -19,6 +24,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log/zap" "github.com/kuadrant/dns-operator/api/v1alpha1" + "github.com/kuadrant/dns-operator/internal/provider" _ "github.com/kuadrant/dns-operator/internal/provider/aws" _ "github.com/kuadrant/dns-operator/internal/provider/azure" _ "github.com/kuadrant/dns-operator/internal/provider/google" @@ -27,26 +33,57 @@ import ( const ( // configuration environment variables - dnsZoneDomainNameEnvvar = "TEST_DNS_ZONE_DOMAIN_NAME" - dnsManagedZoneName = "TEST_DNS_MANAGED_ZONE_NAME" - dnsNamespace = "TEST_DNS_NAMESPACE" - dnsProvider = "TEST_DNS_PROVIDER" + dnsManagedZoneName = "TEST_DNS_MANAGED_ZONE_NAME" + dnsNamespaces = "TEST_DNS_NAMESPACES" + dnsClusterContexts = "TEST_DNS_CLUSTER_CONTEXTS" + deploymentCount = "DEPLOYMENT_COUNT" + clusterCount = "CLUSTER_COUNT" ) var ( - k8sClient client.Client // testSuiteID is a randomly generated identifier for the test suite testSuiteID string // testZoneDomainName provided domain name for the testZoneID e.g. e2e.hcpapps.net - testZoneDomainName string - testManagedZoneName string - testNamespace string - testDNSProvider string - supportedProviders = []string{"aws", "gcp", "azure"} + testZoneDomainName string + testManagedZoneName string + testNamespaces []string + testClusterContexts []string + testDNSProvider string + testClusters []testCluster + supportedHealthCheckProviders = []string{"aws"} - testManagedZone *v1alpha1.ManagedZone ) +// testCluster represents a cluster under test and contains a reference to a configured k8client and all it's managed zones. +type testCluster struct { + name string + testManagedZones []*v1alpha1.ManagedZone + k8sClient client.Client +} + +// testDNSRecord encapsulates a v1alpha1.DNSRecord created in a test case, the v1alpha1.ManagedZone it was created in and the config used to create it. +// The testConfig is used when asserting the expected values set in the providers. +type testDNSRecord struct { + cluster *testCluster + managedZone *v1alpha1.ManagedZone + record *v1alpha1.DNSRecord + config testConfig +} + +type testConfig struct { + testTargetIP string + testGeoCode string + testDefaultGeoCode string + hostnames testHostnames +} + +type testHostnames struct { + klb string + geoKlb string + defaultGeoKlb string + clusterKlb string +} + func TestAPIs(t *testing.T) { RegisterFailHandler(Fail) RunSpecs(t, "E2E Tests Suite") @@ -61,45 +98,174 @@ var _ = BeforeSuite(func(ctx SpecContext) { err = v1alpha1.AddToScheme(scheme.Scheme) Expect(err).NotTo(HaveOccurred()) - cfg, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig( - clientcmd.NewDefaultClientConfigLoadingRules(), - &clientcmd.ConfigOverrides{}, - ).ClientConfig() - Expect(err).NotTo(HaveOccurred()) - - k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) - Expect(err).NotTo(HaveOccurred()) - Expect(k8sClient).NotTo(BeNil()) - - testManagedZone = &v1alpha1.ManagedZone{} - err = k8sClient.Get(ctx, client.ObjectKey{Namespace: testNamespace, Name: testManagedZoneName}, testManagedZone) - Expect(err).NotTo(HaveOccurred()) + loadClusters(ctx) + Expect(testDNSProvider).NotTo(BeEmpty()) + Expect(testZoneDomainName).NotTo(BeEmpty()) + Expect(testClusters).NotTo(BeEmpty()) + for i := range testClusters { + Expect(testClusters[i].testManagedZones).NotTo(BeEmpty()) + } testSuiteID = "dns-op-e2e-" + GenerateName() geoCode := "EU" - if testDNSProvider == "gcp" { + if testDNSProvider == "google" { geoCode = "europe-west1" } SetTestEnv("testGeoCode", geoCode) }) +// setConfigFromEnvVars loads test suite runtime configurations from env vars. +// dnsManagedZoneName managed zone name expected to exist in each test namespace (i.e. dev-mz-aws). +// dnsNamespaces test namespaces, comma seperated list (i.e. dns-operator-1,dns-operator-2) +// deploymentCount number of test namespaces expected. Appends an index suffix to the dnsNamespaces, only used if dnsNamespaces is a single length array. +// +// Examples: +// dnsNamespaces=dns-operator deploymentCount= = dnsNamespaces=dns-operator +// dnsNamespaces=dns-operator-1,dns-operator-2 deploymentCount= = dnsNamespaces=dns-operator-1,dns-operator-2 +// dnsNamespaces=dns-operator deploymentCount=1 = dnsNamespaces=dns-operator-1 +// dnsNamespaces=dns-operator deploymentCount=2 = dnsNamespaces=dns-operator-1,dns-operator-2 +// dnsNamespaces=dns-operator-5,dns-operator-6 deploymentCount=1 = dnsNamespaces=dns-operator-5,dns-operator-6 +// +// dnsClusterContexts test cluster contexts, comma seperated list (i.e. kind-kuadrant-dns-local-1,kind-kuadrant-dns-local-2), +// if unset the current context is used and a single cluster is assumed. +// clusterCount number of test clusters expected. Appends an index suffix to the dnsClusterContexts, only used if dnsClusterContexts is a single length array. +// +// Examples: +// dnsClusterContexts=kind-kuadrant-dns-local clusterCount= = dnsClusterContexts=kind-kuadrant-dns-local +// dnsClusterContexts=kind-kuadrant-dns-local-1,kind-kuadrant-dns-local-2 clusterCount= = dnsClusterContexts=kind-kuadrant-dns-local-1,kind-kuadrant-dns-local-2 +// dnsClusterContexts=kind-kuadrant-dns-local clusterCount=1 = dnsClusterContexts=kind-kuadrant-dns-local-1 +// dnsClusterContexts=kind-kuadrant-dns-local clusterCount=2 = dnsClusterContexts=kind-kuadrant-dns-local-1,kind-kuadrant-dns-local-2 +// dnsClusterContexts=my-cluster-1,my-cluster-2 clusterCount=1 = dnsClusterContexts=my-cluster-1,my-cluster-2 + func setConfigFromEnvVars() error { // Load test suite configuration from the environment - if testZoneDomainName = os.Getenv(dnsZoneDomainNameEnvvar); testZoneDomainName == "" { - return fmt.Errorf("env variable '%s' must be set", dnsZoneDomainNameEnvvar) - } if testManagedZoneName = os.Getenv(dnsManagedZoneName); testManagedZoneName == "" { return fmt.Errorf("env variable '%s' must be set", dnsManagedZoneName) } - if testNamespace = os.Getenv(dnsNamespace); testNamespace == "" { - return fmt.Errorf("env variable '%s' must be set", dnsNamespace) + + namespacesStr := os.Getenv(dnsNamespaces) + if namespacesStr == "" { + //ToDo mnairn: Temporarily keeping a check for "TEST_DNS_NAMESPACE" to allow PR e2e test to work. Remove later. + namespacesStr = os.Getenv("TEST_DNS_NAMESPACE") + if namespacesStr == "" { + return fmt.Errorf("env variable '%s' must be set", dnsNamespaces) + } } - if testDNSProvider = os.Getenv(dnsProvider); testDNSProvider == "" { - return fmt.Errorf("env variable '%s' must be set", dnsProvider) + + namespaces := strings.Split(namespacesStr, ",") + if len(namespaces) == 1 { + if dcStr := os.Getenv(deploymentCount); dcStr != "" { + dc, err := strconv.Atoi(dcStr) + if err != nil { + return fmt.Errorf("env variable '%s' must be an integar", deploymentCount) + } + for i := 1; i <= dc; i++ { + testNamespaces = append(testNamespaces, fmt.Sprintf("%s-%d", namespaces[0], i)) + } + } else { + testNamespaces = namespaces + } + } else { + testNamespaces = namespaces } - if !slices.Contains(supportedProviders, testDNSProvider) { - return fmt.Errorf("unsupported provider '%s' must be one of '%s'", testDNSProvider, supportedProviders) + + clusterContextsStr := os.Getenv(dnsClusterContexts) + if clusterContextsStr == "" { + testClusterContexts = []string{"current"} + return nil } + + clusterContexts := strings.Split(clusterContextsStr, ",") + if len(clusterContexts) == 1 { + if dcStr := os.Getenv(clusterCount); dcStr != "" { + dc, err := strconv.Atoi(dcStr) + if err != nil { + return fmt.Errorf("env variable '%s' must be an integar", clusterCount) + } + for i := 1; i <= dc; i++ { + testClusterContexts = append(testClusterContexts, fmt.Sprintf("%s-%d", clusterContexts[0], i)) + } + } else { + testClusterContexts = clusterContexts + } + } else { + testClusterContexts = clusterContexts + } + return nil } + +// loadClusters iterates each of the configured test clusters, configures a k8s client, loads test managed zones and creates a `testCluster` resource. +func loadClusters(ctx context.Context) { + for _, c := range testClusterContexts { + cfgOverrides := &clientcmd.ConfigOverrides{} + if c != "current" { + cfgOverrides = &clientcmd.ConfigOverrides{CurrentContext: c} + } + cfg, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig( + clientcmd.NewDefaultClientConfigLoadingRules(), + cfgOverrides, + ).ClientConfig() + Expect(err).NotTo(HaveOccurred()) + + k8sClient, err := client.New(cfg, client.Options{Scheme: scheme.Scheme}) + Expect(err).NotTo(HaveOccurred()) + + tc := &testCluster{ + name: c, + k8sClient: k8sClient, + } + + loadManagedZones(ctx, tc) + + //Append the cluster to the list of test clusters + testClusters = append(testClusters, *tc) + } +} + +// loadManagedZones iterates each of the configured test namespaces, loads the expected managed zone (TEST_DNS_MANAGED_ZONE_NAME), and asserts the configuration of each is compatible. +// Sets the test suite testDNSProvider and testZoneDomainName directly from the managed zone spec and provider secret. +// If the managed zone does not exist in the namespace, an error is thrown. +// If the managed zone has a different domain name from any previously loaded managed zones, an error is thrown. +// If the managed zone has a different dns provider from any previously loaded managed zones, an error is thrown. +func loadManagedZones(ctx context.Context, tc *testCluster) { + for _, n := range testNamespaces { + mz := &v1alpha1.ManagedZone{} + + // Ensure managed zone exists and is ready + err := tc.k8sClient.Get(ctx, client.ObjectKey{Namespace: n, Name: testManagedZoneName}, mz) + Expect(err).NotTo(HaveOccurred()) + Expect(mz.Status.Conditions).To( + ContainElement(MatchFields(IgnoreExtras, Fields{ + "Type": Equal(string(v1alpha1.ConditionTypeReady)), + "Status": Equal(metav1.ConditionTrue), + "ObservedGeneration": Equal(mz.Generation), + })), + ) + + // Ensure all managed zone names match + if testZoneDomainName == "" { + testZoneDomainName = mz.Spec.DomainName + } else { + Expect(mz.Spec.DomainName).To(Equal(testZoneDomainName)) + } + + s := &v1.Secret{} + err = tc.k8sClient.Get(ctx, client.ObjectKey{Namespace: n, Name: mz.Spec.SecretRef.Name}, s) + Expect(err).NotTo(HaveOccurred()) + + p, err := provider.NameForProviderSecret(s) + Expect(err).NotTo(HaveOccurred()) + + // Ensure all managed zone are suing the same provider + if testDNSProvider == "" { + testDNSProvider = p + } else { + Expect(p).To(Equal(testDNSProvider)) + } + + //Append the managed zone to the list of test zones + tc.testManagedZones = append(tc.testManagedZones, mz) + } +} From 2ba6aca37e3dcf949a7b939c61801a4fcc0067d2 Mon Sep 17 00:00:00 2001 From: Michael Nairn Date: Mon, 29 Jul 2024 11:06:29 +0100 Subject: [PATCH 3/3] local-setup: Remove local-setup-multi in favour of adding the cluster count option to local-setup ``` make local-setup DEPLOY=true CLUSTER_COUNT=2 ``` Only noticable change is that all clusters generated by local-setup, single or multi cluster, have a suffix for the count. ``` make local-setup DEPLOY=true kind get clusters kuadrant-dns-local-1 ``` Signed-off-by: Michael Nairn --- Makefile | 18 +++++++++--------- README.md | 2 +- test/e2e/README.md | 8 ++++---- test/e2e/suite_test.go | 21 +++++++++++---------- 4 files changed, 25 insertions(+), 24 deletions(-) diff --git a/Makefile b/Makefile index e9ca4fa2..fdc3b95b 100644 --- a/Makefile +++ b/Makefile @@ -159,11 +159,11 @@ test-e2e: ginkgo test-e2e-multi: ginkgo $(GINKGO) -tags=e2e -v --label-filter=multi_record -v ./test/e2e -.PHONY: local-setup -local-setup: DEPLOY=false -local-setup: TEST_NAMESPACE=dnstest -local-setup: DEPLOYMENT_SCOPE=cluster -local-setup: $(KIND) ## Setup local development kind cluster, dependencies and optionally deploy the dns operator DEPLOY=false|true +.PHONY: local-setup-cluster +local-setup-cluster: DEPLOY=false +local-setup-cluster: TEST_NAMESPACE=dnstest +local-setup-cluster: DEPLOYMENT_SCOPE=cluster +local-setup-cluster: $(KIND) ## Setup local development kind cluster, dependencies and optionally deploy the dns operator DEPLOY=false|true @echo "local-setup: KIND_CLUSTER_NAME=${KIND_CLUSTER_NAME} DEPLOY=${DEPLOY} DEPLOYMENT_SCOPE=${DEPLOYMENT_SCOPE} TEST_NAMESPACE=${TEST_NAMESPACE}" @$(MAKE) -s kind-delete-cluster @$(MAKE) -s kind-create-cluster @@ -188,11 +188,11 @@ local-setup: $(KIND) ## Setup local development kind cluster, dependencies and o $(KUBECTL) get managedzones -A @echo "local-setup: Complete!!" -.PHONY: local-setup-multi -local-setup-multi: CLUSTER_COUNT=1 -local-setup-multi: ## Setup multiple local development kind clusters +.PHONY: local-setup +local-setup: CLUSTER_COUNT=1 +local-setup: ## Setup local development kind cluster(s) @n=1 ; while [[ $$n -le $(CLUSTER_COUNT) ]] ; do \ - $(MAKE) -s local-setup KIND_CLUSTER_NAME=${KIND_CLUSTER_NAME_PREFIX}-$$n;\ + $(MAKE) -s local-setup-cluster KIND_CLUSTER_NAME=${KIND_CLUSTER_NAME_PREFIX}-$$n;\ ((n = n + 1)) ;\ done ;\ diff --git a/README.md b/README.md index 7dbe161b..3765c471 100644 --- a/README.md +++ b/README.md @@ -84,7 +84,7 @@ kubectl logs -f deployments/dns-operator-controller-manager -n dns-operator-syst The e2e test suite can be executed against any cluster running the DNS Operator with configuration added for any supported provider. ``` -make test-e2e TEST_DNS_MANAGED_ZONE_NAME= TEST_DNS_NAMESPACES= +make test-e2e TEST_DNS_MANAGED_ZONE_NAME= TEST_DNS_NAMESPACES= ``` | Environment Variable | Description | diff --git a/test/e2e/README.md b/test/e2e/README.md index 90b4f2b7..068ac848 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -43,16 +43,16 @@ dnstest dev-mz-gcp mn.google.hcpapps.net ### Cluster scoped on multiple clusters -Deploy the operator on two kind cluster each with one operator instance watching all namespaces: +Deploy the operator on two kind clusters each with one operator instance watching all namespaces: ```shell -make local-setup-multi DEPLOY=true CLUSTER_COUNT=2 +make local-setup DEPLOY=true CLUSTER_COUNT=2 ``` ### Namespace scoped on multiple clusters -Deploy the operator on two local kind cluster with two operator instances in two namespaces watching their own namespace only: +Deploy the operator on two local kind clusters with two operator instances in two namespaces watching their own namespace only: ```shell -make local-setup-multi DEPLOY=true DEPLOYMENT_SCOPE=namespace DEPLOYMENT_COUNT=2 CLUSTER_COUNT=2 +make local-setup DEPLOY=true DEPLOYMENT_SCOPE=namespace DEPLOYMENT_COUNT=2 CLUSTER_COUNT=2 ``` ## Run the test suite diff --git a/test/e2e/suite_test.go b/test/e2e/suite_test.go index 9541c8f4..1875f40c 100644 --- a/test/e2e/suite_test.go +++ b/test/e2e/suite_test.go @@ -116,27 +116,28 @@ var _ = BeforeSuite(func(ctx SpecContext) { }) // setConfigFromEnvVars loads test suite runtime configurations from env vars. +// // dnsManagedZoneName managed zone name expected to exist in each test namespace (i.e. dev-mz-aws). // dnsNamespaces test namespaces, comma seperated list (i.e. dns-operator-1,dns-operator-2) // deploymentCount number of test namespaces expected. Appends an index suffix to the dnsNamespaces, only used if dnsNamespaces is a single length array. // // Examples: -// dnsNamespaces=dns-operator deploymentCount= = dnsNamespaces=dns-operator -// dnsNamespaces=dns-operator-1,dns-operator-2 deploymentCount= = dnsNamespaces=dns-operator-1,dns-operator-2 -// dnsNamespaces=dns-operator deploymentCount=1 = dnsNamespaces=dns-operator-1 -// dnsNamespaces=dns-operator deploymentCount=2 = dnsNamespaces=dns-operator-1,dns-operator-2 -// dnsNamespaces=dns-operator-5,dns-operator-6 deploymentCount=1 = dnsNamespaces=dns-operator-5,dns-operator-6 +// inputs: TEST_DNS_NAMESPACES=dns-operator DEPLOYMENT_COUNT= configResult: dnsNamespaces=dns-operator +// inputs: TEST_DNS_NAMESPACES=dns-operator-1,dns-operator-2 DEPLOYMENT_COUNT= configResult: dnsNamespaces=dns-operator-1,dns-operator-2 +// inputs: TEST_DNS_NAMESPACES=dns-operator DEPLOYMENT_COUNT=1 configResult: dnsNamespaces=dns-operator-1 +// inputs: TEST_DNS_NAMESPACES=dns-operator DEPLOYMENT_COUNT=2 configResult: dnsNamespaces=dns-operator-1,dns-operator-2 +// inputs: TEST_DNS_NAMESPACES=dns-operator-5,dns-operator-6 DEPLOYMENT_COUNT=1 configResult: dnsNamespaces=dns-operator-5,dns-operator-6 // // dnsClusterContexts test cluster contexts, comma seperated list (i.e. kind-kuadrant-dns-local-1,kind-kuadrant-dns-local-2), // if unset the current context is used and a single cluster is assumed. // clusterCount number of test clusters expected. Appends an index suffix to the dnsClusterContexts, only used if dnsClusterContexts is a single length array. // // Examples: -// dnsClusterContexts=kind-kuadrant-dns-local clusterCount= = dnsClusterContexts=kind-kuadrant-dns-local -// dnsClusterContexts=kind-kuadrant-dns-local-1,kind-kuadrant-dns-local-2 clusterCount= = dnsClusterContexts=kind-kuadrant-dns-local-1,kind-kuadrant-dns-local-2 -// dnsClusterContexts=kind-kuadrant-dns-local clusterCount=1 = dnsClusterContexts=kind-kuadrant-dns-local-1 -// dnsClusterContexts=kind-kuadrant-dns-local clusterCount=2 = dnsClusterContexts=kind-kuadrant-dns-local-1,kind-kuadrant-dns-local-2 -// dnsClusterContexts=my-cluster-1,my-cluster-2 clusterCount=1 = dnsClusterContexts=my-cluster-1,my-cluster-2 +// inputs: TEST_DNS_CLUSTER_CONTEXTS=kind-kuadrant-dns-local CLUSTER_COUNT= configResult: dnsClusterContexts=kind-kuadrant-dns-local +// inputs: TEST_DNS_CLUSTER_CONTEXTS=kind-kuadrant-dns-local-1,kind-kuadrant-dns-local-2 CLUSTER_COUNT= configResult: dnsClusterContexts=kind-kuadrant-dns-local-1,kind-kuadrant-dns-local-2 +// inputs: TEST_DNS_CLUSTER_CONTEXTS=kind-kuadrant-dns-local CLUSTER_COUNT=1 configResult: dnsClusterContexts=kind-kuadrant-dns-local-1 +// inputs: TEST_DNS_CLUSTER_CONTEXTS=kind-kuadrant-dns-local CLUSTER_COUNT=2 configResult: dnsClusterContexts=kind-kuadrant-dns-local-1,kind-kuadrant-dns-local-2 +// inputs: TEST_DNS_CLUSTER_CONTEXTS=my-cluster-1,my-cluster-2 CLUSTER_COUNT=1 configResult: dnsClusterContexts=my-cluster-1,my-cluster-2 func setConfigFromEnvVars() error { // Load test suite configuration from the environment