e2e tests and GH workflow updates

Kuadrant · Dec 5, 2023 · 824e130 · 824e130
1 parent 1dc8c23
commit 824e130
Show file tree

Hide file tree

Showing 3 changed files with 66 additions and 64 deletions.
diff --git a/.github/workflows/ci-e2e.yaml b/.github/workflows/ci-e2e.yaml
@@ -44,24 +44,22 @@ jobs:
       - uses: actions/setup-go@v4
         with:
           go-version: "1.21.x"
-      - name: Create controller-config.env
+      - name: Create AWS provider secret
         run: |
-          echo "AWS_DNS_PUBLIC_ZONE_ID=Z086929132US3PB46EOLR" >> controller-config.env
-          echo "ZONE_ROOT_DOMAIN=e2e.hcpapps.net" >> controller-config.env
-          echo "ZONE_NAME=e2e-google-hcpapps-net" >> controller-config.env 
-          echo "ZONE_DNS_NAME=e2e.google.hcpapps.net" >> controller-config.env
-
-      - name: Create aws-credentials.env
-        run: |
-          echo "AWS_ACCESS_KEY_ID=${{ secrets.E2E_AWS_ACCESS_KEY_ID }}" >> aws-credentials.env
-          echo "AWS_SECRET_ACCESS_KEY=${{ secrets.E2E_AWS_SECRET_ACCESS_KEY }}" >> aws-credentials.env
-          echo "AWS_REGION=${{ env.AWS_REGION }}" >> aws-credentials.env
-      - name: Create gcp-credentials.env
+          echo "{\"zoneIDFilter\": [], \"domainFilter\": [\"e2e.hcpapps.net\"]}" > e2e-test-aws-provider-config.json
+          kubectl create secret generic e2e-test-aws-credentials -n multi-cluster-gateways \
+            --type=kuadrant.io/aws \
+            --from-literal=AWS_ACCESS_KEY_ID=${{ secrets.E2E_AWS_ACCESS_KEY_ID }} \
+            --from-literal=AWS_SECRET_ACCESS_KEY=${{ secrets.E2E_AWS_SECRET_ACCESS_KEY }} \
+            --from-file=CONFIG=e2e-test-aws-provider-config.json
+      - name: Create GCP provider secret
         run: |
-          secret="${{ secrets.E2E_GOOGLE }}"
-          decoded=$(echo "$secret" | base64 --decode )
-          echo "GOOGLE=$decoded" >> gcp-credentials.env
-          echo "PROJECT_ID=${{ secrets.E2E_PROJECT_ID }}" >> gcp-credentials.env   
+          echo "{\"zoneIDFilter\": [], \"domainFilter\": [\"e2e.google.hcpapps.net\"]}" > e2e-test-gcp-provider-config.json
+          kubectl create secret generic e2e-test-gcp-credentials -n multi-cluster-gateways \
+            --type=kuadrant.io/gcp \
+            --from-literal=PROJECT_ID=${{ secrets.E2E_PROJECT_ID }} \
+            --from-file=GOOGLE=${{ secrets.E2E_GOOGLE }} \
+            --from-file=CONFIG=e2e-test-gcp-provider-config.json
       - name: Setup environment
         run: |
           export OCM_SINGLE=1
@@ -75,16 +73,12 @@ jobs:
           kubectl --context kind-mgc-control-plane -n multicluster-gateway-controller-system wait --timeout=300s --for=condition=Available deployment/mgc-policy-controller
           kubectl --context kind-mgc-control-plane logs --all-containers --ignore-errors deployment/mgc-controller-manager -n multicluster-gateway-controller-system
           kubectl --context kind-mgc-control-plane logs --all-containers --ignore-errors deployment/mgc-policy-controller -n multicluster-gateway-controller-system
-          kubectl get managedzones -n multi-cluster-gateways mgc-dev-mz-aws -o yaml
-          kubectl --context kind-mgc-control-plane -n multi-cluster-gateways wait --timeout=60s --for=condition=Ready managedzone/mgc-dev-mz-aws
-          kubectl get managedzones -n multi-cluster-gateways mgc-dev-mz-gcp -o yaml
-          kubectl --context kind-mgc-control-plane -n multi-cluster-gateways wait --timeout=60s --for=condition=Ready managedzone/mgc-dev-mz-gcp
       - name: Run suite AWS
         run: |
           export OCM_SINGLE=1
           export TEST_HUB_NAMESPACE=multi-cluster-gateways
           export TEST_HUB_KUBE_CONTEXT=kind-mgc-control-plane
-          export TEST_DNS_PROVIDER_SECRET_NAME=mgc-aws-credentials
+          export TEST_DNS_PROVIDER_SECRET_NAME=e2e-test-aws-credentials
           export TEST_DNS_ZONE_DOMAIN_NAME=e2e.hcpapps.net
           export TEST_DNS_ZONE_ID=Z086929132US3PB46EOLR
           make test-e2e
@@ -93,7 +87,7 @@ jobs:
           export OCM_SINGLE=1
           export TEST_HUB_NAMESPACE=multi-cluster-gateways
           export TEST_HUB_KUBE_CONTEXT=kind-mgc-control-plane
-          export TEST_DNS_PROVIDER_SECRET_NAME=mgc-gcp-credentials
+          export TEST_DNS_PROVIDER_SECRET_NAME=e2e-test-gcp-credentials
           export TEST_DNS_ZONE_DOMAIN_NAME=e2e.google.hcpapps.net
           export TEST_DNS_ZONE_ID=e2e-google-hcpapps-net
           make test-e2e

diff --git a/test/e2e/dnspolicy_istio_gateway_test.go → ...dnspolicy_providers_istio_gateway_test.go b/test/e2e/dnspolicy_istio_gateway_test.go → ...dnspolicy_providers_istio_gateway_test.go
@@ -43,9 +43,11 @@ var _ = Describe("DNSPolicy Targeting an Istio Gateway", func() {
 
 	var gw *gatewayapiv1.Gateway
 	var httproute *gatewayapiv1.HTTPRoute
+	var dnsPolicy *v1alpha2.DNSPolicy
+	var mz *v1alpha2.ManagedZone
 
 	BeforeEach(func(ctx SpecContext) {
-		testID = "t-dns-istio-" + tconfig.GenerateName()
+		testID = "t-dns-" + tconfig.GenerateName()
 		//ToDo Have this generate a new namespace instead of using the Hub Namespace and consider using a spoke client.
 		// This currently still relies on the provider credentials secret being created ahead of time in a target namespace
 		testNamespace = tconfig.HubNamespace()
@@ -55,6 +57,8 @@ var _ = Describe("DNSPolicy Targeting an Istio Gateway", func() {
 		testHostname = gatewayapiv1.Hostname(strings.Join([]string{testID, testDomainName}, "."))
 		k8sClient = tconfig.HubClient()
 
+		GinkgoWriter.Printf("[debug] testHostname: '%s'\n", testHostname)
+
 		By("creating an Istio Gateway")
 		gw = NewGatewayBuilder(testID, IstioGatewayClassName, testNamespace).
 			WithListener(gatewayapiv1.Listener{
@@ -104,8 +108,23 @@ var _ = Describe("DNSPolicy Targeting an Istio Gateway", func() {
 	})
 
 	AfterEach(func(ctx SpecContext) {
-		if gw != nil {
-			err := k8sClient.Delete(ctx, gw,
+		if dnsPolicy != nil {
+			err := k8sClient.Delete(ctx, dnsPolicy,
+				client.PropagationPolicy(metav1.DeletePropagationForeground))
+			Expect(client.IgnoreNotFound(err)).ToNot(HaveOccurred())
+			Eventually(func(g Gomega) { // wait until it's gone to allow time for DNSRecords to be cleaned up
+				err := k8sClient.Get(ctx, client.ObjectKeyFromObject(dnsPolicy), dnsPolicy)
+				g.Expect(err).To(HaveOccurred())
+				g.Expect(err).To(MatchError(ContainSubstring("not found")))
+
+				recordList := &v1alpha2.DNSRecordList{}
+				err = k8sClient.List(ctx, recordList, &client.MatchingLabels{"kuadrant.io/gateway": gw.GetName()}, &client.ListOptions{Namespace: testNamespace})
+				g.Expect(err).NotTo(HaveOccurred())
+				g.Expect(recordList.Items).To(BeEmpty())
+			}, TestTimeoutMedium, time.Second).Should(Succeed())
+		}
+		if mz != nil {
+			err := k8sClient.Delete(ctx, mz,
 				client.PropagationPolicy(metav1.DeletePropagationForeground))
 			Expect(client.IgnoreNotFound(err)).ToNot(HaveOccurred())
 		}
@@ -114,11 +133,14 @@ var _ = Describe("DNSPolicy Targeting an Istio Gateway", func() {
 				client.PropagationPolicy(metav1.DeletePropagationForeground))
 			Expect(client.IgnoreNotFound(err)).ToNot(HaveOccurred())
 		}
+		if gw != nil {
+			err := k8sClient.Delete(ctx, gw,
+				client.PropagationPolicy(metav1.DeletePropagationForeground))
+			Expect(client.IgnoreNotFound(err)).ToNot(HaveOccurred())
+		}
 	})
 
-	Context("ManagedZone Provider", func() {
-		var mz *v1alpha2.ManagedZone
-		var dnsPolicy *v1alpha2.DNSPolicy
+	FContext("ManagedZone Provider", func() {
 
 		BeforeEach(func(ctx SpecContext) {
 			By("creating a ManagedZone")
@@ -131,24 +153,6 @@ var _ = Describe("DNSPolicy Targeting an Istio Gateway", func() {
 			Expect(err).ToNot(HaveOccurred())
 		})
 
-		AfterEach(func(ctx SpecContext) {
-			if dnsPolicy != nil {
-				err := k8sClient.Delete(ctx, dnsPolicy,
-					client.PropagationPolicy(metav1.DeletePropagationForeground))
-				Expect(client.IgnoreNotFound(err)).ToNot(HaveOccurred())
-				Eventually(func(g Gomega) { // wait until it's gone to allow time for DNSRecords to be cleaned up
-					err := k8sClient.Get(ctx, client.ObjectKeyFromObject(dnsPolicy), dnsPolicy)
-					g.Expect(err).To(HaveOccurred())
-					g.Expect(err).To(MatchError(ContainSubstring("not found")))
-				}, TestTimeoutMedium, time.Second).Should(Succeed())
-			}
-			if mz != nil {
-				err := k8sClient.Delete(ctx, mz,
-					client.PropagationPolicy(metav1.DeletePropagationForeground))
-				Expect(client.IgnoreNotFound(err)).ToNot(HaveOccurred())
-			}
-		})
-
 		It("makes the hostname resolvable when a dnspolicy and httproute are attached", func(ctx SpecContext) {
 
 			By("creating a DNSPolicy with ManagedZone provider")
@@ -207,6 +211,15 @@ var _ = Describe("DNSPolicy Targeting an Istio Gateway", func() {
 						"RecordTTL":     Equal(v1alpha2.TTL(60)),
 					})),
 				))
+				g.Expect(dnsrecord.Status.Endpoints).Should(ConsistOf(
+					PointTo(MatchFields(IgnoreExtras, Fields{
+						"DNSName":       Equal(string(testHostname)),
+						"Targets":       Not(BeEmpty()),
+						"RecordType":    Equal("A"),
+						"SetIdentifier": Equal(""),
+						"RecordTTL":     Equal(v1alpha2.TTL(60)),
+					})),
+				))
 			}, TestTimeoutLong, time.Second, ctx).Should(Succeed())
 
 			By("ensuring the authoritative nameserver resolves the hostname")
@@ -226,15 +239,6 @@ var _ = Describe("DNSPolicy Targeting an Istio Gateway", func() {
 	})
 
 	Context("Secret Provider", func() {
-		var dnsPolicy *v1alpha2.DNSPolicy
-
-		AfterEach(func(ctx SpecContext) {
-			if dnsPolicy != nil {
-				err := k8sClient.Delete(ctx, dnsPolicy,
-					client.PropagationPolicy(metav1.DeletePropagationForeground))
-				Expect(client.IgnoreNotFound(err)).ToNot(HaveOccurred())
-			}
-		})
 
 		It("makes the hostname resolvable when a dnspolicy and httproute are attached", func(ctx SpecContext) {
 
@@ -294,6 +298,16 @@ var _ = Describe("DNSPolicy Targeting an Istio Gateway", func() {
 						"RecordTTL":     Equal(v1alpha2.TTL(60)),
 					})),
 				))
+				// We need to wait for the status to be updated otherwise google can leave things behind
+				g.Expect(dnsrecord.Status.Endpoints).Should(ConsistOf(
+					PointTo(MatchFields(IgnoreExtras, Fields{
+						"DNSName":       Equal(string(testHostname)),
+						"Targets":       Not(BeEmpty()),
+						"RecordType":    Equal("A"),
+						"SetIdentifier": Equal(""),
+						"RecordTTL":     Equal(v1alpha2.TTL(60)),
+					})),
+				))
 			}, TestTimeoutLong, time.Second, ctx).Should(Succeed())
 
 			By("ensuring the authoritative nameserver resolves the hostname")
@@ -313,15 +327,6 @@ var _ = Describe("DNSPolicy Targeting an Istio Gateway", func() {
 	})
 
 	Context("None Provider", func() {
-		var dnsPolicy *v1alpha2.DNSPolicy
-
-		AfterEach(func(ctx SpecContext) {
-			if dnsPolicy != nil {
-				err := k8sClient.Delete(ctx, dnsPolicy,
-					client.PropagationPolicy(metav1.DeletePropagationForeground))
-				Expect(client.IgnoreNotFound(err)).ToNot(HaveOccurred())
-			}
-		})
 
 		It("should create dns record with no zone assigned and record should not become ready", func(ctx SpecContext) {
 
@@ -380,6 +385,7 @@ var _ = Describe("DNSPolicy Targeting an Istio Gateway", func() {
 						"RecordTTL":     Equal(v1alpha2.TTL(60)),
 					})),
 				))
+				g.Expect(dnsrecord.Status.Endpoints).Should(BeEmpty())
 			}, TestTimeoutLong, time.Second, ctx).Should(Succeed())
 
 		})

diff --git a/test/e2e/gateway_single_spoke_test.go b/test/e2e/gateway_single_spoke_test.go
@@ -64,6 +64,8 @@ var _ = Describe("Gateway single target cluster", func() {
 		testHostnameOther = gatewayapiv1.Hostname(strings.Join([]string{"other-" + testID, string(testHostname)}, "."))
 		testHostnameWildcard = gatewayapiv1.Hostname(strings.Join([]string{"*", string(testHostname)}, "."))
 
+		GinkgoWriter.Printf("[debug] testHostname: '%s'\n", testHostname)
+
 		By("creating a Placement for the Gateway resource")
 		placement = &ocm_cluster_v1beta1.Placement{
 			ObjectMeta: metav1.ObjectMeta{Name: testID, Namespace: tconfig.HubNamespace()},