From 824e130e1dc396baca836459f2491154f015b92a Mon Sep 17 00:00:00 2001 From: Michael Nairn Date: Tue, 5 Dec 2023 16:59:36 +0000 Subject: [PATCH] e2e tests and GH workflow updates --- .github/workflows/ci-e2e.yaml | 38 ++++---- ...dnspolicy_providers_istio_gateway_test.go} | 90 ++++++++++--------- test/e2e/gateway_single_spoke_test.go | 2 + 3 files changed, 66 insertions(+), 64 deletions(-) rename test/e2e/{dnspolicy_istio_gateway_test.go => dnspolicy_providers_istio_gateway_test.go} (86%) diff --git a/.github/workflows/ci-e2e.yaml b/.github/workflows/ci-e2e.yaml index 717fc3ff..145db180 100644 --- a/.github/workflows/ci-e2e.yaml +++ b/.github/workflows/ci-e2e.yaml @@ -44,24 +44,22 @@ jobs: - uses: actions/setup-go@v4 with: go-version: "1.21.x" - - name: Create controller-config.env + - name: Create AWS provider secret run: | - echo "AWS_DNS_PUBLIC_ZONE_ID=Z086929132US3PB46EOLR" >> controller-config.env - echo "ZONE_ROOT_DOMAIN=e2e.hcpapps.net" >> controller-config.env - echo "ZONE_NAME=e2e-google-hcpapps-net" >> controller-config.env - echo "ZONE_DNS_NAME=e2e.google.hcpapps.net" >> controller-config.env - - - name: Create aws-credentials.env - run: | - echo "AWS_ACCESS_KEY_ID=${{ secrets.E2E_AWS_ACCESS_KEY_ID }}" >> aws-credentials.env - echo "AWS_SECRET_ACCESS_KEY=${{ secrets.E2E_AWS_SECRET_ACCESS_KEY }}" >> aws-credentials.env - echo "AWS_REGION=${{ env.AWS_REGION }}" >> aws-credentials.env - - name: Create gcp-credentials.env + echo "{\"zoneIDFilter\": [], \"domainFilter\": [\"e2e.hcpapps.net\"]}" > e2e-test-aws-provider-config.json + kubectl create secret generic e2e-test-aws-credentials -n multi-cluster-gateways \ + --type=kuadrant.io/aws \ + --from-literal=AWS_ACCESS_KEY_ID=${{ secrets.E2E_AWS_ACCESS_KEY_ID }} \ + --from-literal=AWS_SECRET_ACCESS_KEY=${{ secrets.E2E_AWS_SECRET_ACCESS_KEY }} \ + --from-file=CONFIG=e2e-test-aws-provider-config.json + - name: Create GCP provider secret run: | - secret="${{ secrets.E2E_GOOGLE }}" - decoded=$(echo "$secret" | base64 --decode ) - echo "GOOGLE=$decoded" >> gcp-credentials.env - echo "PROJECT_ID=${{ secrets.E2E_PROJECT_ID }}" >> gcp-credentials.env + echo "{\"zoneIDFilter\": [], \"domainFilter\": [\"e2e.google.hcpapps.net\"]}" > e2e-test-gcp-provider-config.json + kubectl create secret generic e2e-test-gcp-credentials -n multi-cluster-gateways \ + --type=kuadrant.io/gcp \ + --from-literal=PROJECT_ID=${{ secrets.E2E_PROJECT_ID }} \ + --from-file=GOOGLE=${{ secrets.E2E_GOOGLE }} \ + --from-file=CONFIG=e2e-test-gcp-provider-config.json - name: Setup environment run: | export OCM_SINGLE=1 @@ -75,16 +73,12 @@ jobs: kubectl --context kind-mgc-control-plane -n multicluster-gateway-controller-system wait --timeout=300s --for=condition=Available deployment/mgc-policy-controller kubectl --context kind-mgc-control-plane logs --all-containers --ignore-errors deployment/mgc-controller-manager -n multicluster-gateway-controller-system kubectl --context kind-mgc-control-plane logs --all-containers --ignore-errors deployment/mgc-policy-controller -n multicluster-gateway-controller-system - kubectl get managedzones -n multi-cluster-gateways mgc-dev-mz-aws -o yaml - kubectl --context kind-mgc-control-plane -n multi-cluster-gateways wait --timeout=60s --for=condition=Ready managedzone/mgc-dev-mz-aws - kubectl get managedzones -n multi-cluster-gateways mgc-dev-mz-gcp -o yaml - kubectl --context kind-mgc-control-plane -n multi-cluster-gateways wait --timeout=60s --for=condition=Ready managedzone/mgc-dev-mz-gcp - name: Run suite AWS run: | export OCM_SINGLE=1 export TEST_HUB_NAMESPACE=multi-cluster-gateways export TEST_HUB_KUBE_CONTEXT=kind-mgc-control-plane - export TEST_DNS_PROVIDER_SECRET_NAME=mgc-aws-credentials + export TEST_DNS_PROVIDER_SECRET_NAME=e2e-test-aws-credentials export TEST_DNS_ZONE_DOMAIN_NAME=e2e.hcpapps.net export TEST_DNS_ZONE_ID=Z086929132US3PB46EOLR make test-e2e @@ -93,7 +87,7 @@ jobs: export OCM_SINGLE=1 export TEST_HUB_NAMESPACE=multi-cluster-gateways export TEST_HUB_KUBE_CONTEXT=kind-mgc-control-plane - export TEST_DNS_PROVIDER_SECRET_NAME=mgc-gcp-credentials + export TEST_DNS_PROVIDER_SECRET_NAME=e2e-test-gcp-credentials export TEST_DNS_ZONE_DOMAIN_NAME=e2e.google.hcpapps.net export TEST_DNS_ZONE_ID=e2e-google-hcpapps-net make test-e2e diff --git a/test/e2e/dnspolicy_istio_gateway_test.go b/test/e2e/dnspolicy_providers_istio_gateway_test.go similarity index 86% rename from test/e2e/dnspolicy_istio_gateway_test.go rename to test/e2e/dnspolicy_providers_istio_gateway_test.go index ba4f8100..48472108 100644 --- a/test/e2e/dnspolicy_istio_gateway_test.go +++ b/test/e2e/dnspolicy_providers_istio_gateway_test.go @@ -43,9 +43,11 @@ var _ = Describe("DNSPolicy Targeting an Istio Gateway", func() { var gw *gatewayapiv1.Gateway var httproute *gatewayapiv1.HTTPRoute + var dnsPolicy *v1alpha2.DNSPolicy + var mz *v1alpha2.ManagedZone BeforeEach(func(ctx SpecContext) { - testID = "t-dns-istio-" + tconfig.GenerateName() + testID = "t-dns-" + tconfig.GenerateName() //ToDo Have this generate a new namespace instead of using the Hub Namespace and consider using a spoke client. // This currently still relies on the provider credentials secret being created ahead of time in a target namespace testNamespace = tconfig.HubNamespace() @@ -55,6 +57,8 @@ var _ = Describe("DNSPolicy Targeting an Istio Gateway", func() { testHostname = gatewayapiv1.Hostname(strings.Join([]string{testID, testDomainName}, ".")) k8sClient = tconfig.HubClient() + GinkgoWriter.Printf("[debug] testHostname: '%s'\n", testHostname) + By("creating an Istio Gateway") gw = NewGatewayBuilder(testID, IstioGatewayClassName, testNamespace). WithListener(gatewayapiv1.Listener{ @@ -104,8 +108,23 @@ var _ = Describe("DNSPolicy Targeting an Istio Gateway", func() { }) AfterEach(func(ctx SpecContext) { - if gw != nil { - err := k8sClient.Delete(ctx, gw, + if dnsPolicy != nil { + err := k8sClient.Delete(ctx, dnsPolicy, + client.PropagationPolicy(metav1.DeletePropagationForeground)) + Expect(client.IgnoreNotFound(err)).ToNot(HaveOccurred()) + Eventually(func(g Gomega) { // wait until it's gone to allow time for DNSRecords to be cleaned up + err := k8sClient.Get(ctx, client.ObjectKeyFromObject(dnsPolicy), dnsPolicy) + g.Expect(err).To(HaveOccurred()) + g.Expect(err).To(MatchError(ContainSubstring("not found"))) + + recordList := &v1alpha2.DNSRecordList{} + err = k8sClient.List(ctx, recordList, &client.MatchingLabels{"kuadrant.io/gateway": gw.GetName()}, &client.ListOptions{Namespace: testNamespace}) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(recordList.Items).To(BeEmpty()) + }, TestTimeoutMedium, time.Second).Should(Succeed()) + } + if mz != nil { + err := k8sClient.Delete(ctx, mz, client.PropagationPolicy(metav1.DeletePropagationForeground)) Expect(client.IgnoreNotFound(err)).ToNot(HaveOccurred()) } @@ -114,11 +133,14 @@ var _ = Describe("DNSPolicy Targeting an Istio Gateway", func() { client.PropagationPolicy(metav1.DeletePropagationForeground)) Expect(client.IgnoreNotFound(err)).ToNot(HaveOccurred()) } + if gw != nil { + err := k8sClient.Delete(ctx, gw, + client.PropagationPolicy(metav1.DeletePropagationForeground)) + Expect(client.IgnoreNotFound(err)).ToNot(HaveOccurred()) + } }) - Context("ManagedZone Provider", func() { - var mz *v1alpha2.ManagedZone - var dnsPolicy *v1alpha2.DNSPolicy + FContext("ManagedZone Provider", func() { BeforeEach(func(ctx SpecContext) { By("creating a ManagedZone") @@ -131,24 +153,6 @@ var _ = Describe("DNSPolicy Targeting an Istio Gateway", func() { Expect(err).ToNot(HaveOccurred()) }) - AfterEach(func(ctx SpecContext) { - if dnsPolicy != nil { - err := k8sClient.Delete(ctx, dnsPolicy, - client.PropagationPolicy(metav1.DeletePropagationForeground)) - Expect(client.IgnoreNotFound(err)).ToNot(HaveOccurred()) - Eventually(func(g Gomega) { // wait until it's gone to allow time for DNSRecords to be cleaned up - err := k8sClient.Get(ctx, client.ObjectKeyFromObject(dnsPolicy), dnsPolicy) - g.Expect(err).To(HaveOccurred()) - g.Expect(err).To(MatchError(ContainSubstring("not found"))) - }, TestTimeoutMedium, time.Second).Should(Succeed()) - } - if mz != nil { - err := k8sClient.Delete(ctx, mz, - client.PropagationPolicy(metav1.DeletePropagationForeground)) - Expect(client.IgnoreNotFound(err)).ToNot(HaveOccurred()) - } - }) - It("makes the hostname resolvable when a dnspolicy and httproute are attached", func(ctx SpecContext) { By("creating a DNSPolicy with ManagedZone provider") @@ -207,6 +211,15 @@ var _ = Describe("DNSPolicy Targeting an Istio Gateway", func() { "RecordTTL": Equal(v1alpha2.TTL(60)), })), )) + g.Expect(dnsrecord.Status.Endpoints).Should(ConsistOf( + PointTo(MatchFields(IgnoreExtras, Fields{ + "DNSName": Equal(string(testHostname)), + "Targets": Not(BeEmpty()), + "RecordType": Equal("A"), + "SetIdentifier": Equal(""), + "RecordTTL": Equal(v1alpha2.TTL(60)), + })), + )) }, TestTimeoutLong, time.Second, ctx).Should(Succeed()) By("ensuring the authoritative nameserver resolves the hostname") @@ -226,15 +239,6 @@ var _ = Describe("DNSPolicy Targeting an Istio Gateway", func() { }) Context("Secret Provider", func() { - var dnsPolicy *v1alpha2.DNSPolicy - - AfterEach(func(ctx SpecContext) { - if dnsPolicy != nil { - err := k8sClient.Delete(ctx, dnsPolicy, - client.PropagationPolicy(metav1.DeletePropagationForeground)) - Expect(client.IgnoreNotFound(err)).ToNot(HaveOccurred()) - } - }) It("makes the hostname resolvable when a dnspolicy and httproute are attached", func(ctx SpecContext) { @@ -294,6 +298,16 @@ var _ = Describe("DNSPolicy Targeting an Istio Gateway", func() { "RecordTTL": Equal(v1alpha2.TTL(60)), })), )) + // We need to wait for the status to be updated otherwise google can leave things behind + g.Expect(dnsrecord.Status.Endpoints).Should(ConsistOf( + PointTo(MatchFields(IgnoreExtras, Fields{ + "DNSName": Equal(string(testHostname)), + "Targets": Not(BeEmpty()), + "RecordType": Equal("A"), + "SetIdentifier": Equal(""), + "RecordTTL": Equal(v1alpha2.TTL(60)), + })), + )) }, TestTimeoutLong, time.Second, ctx).Should(Succeed()) By("ensuring the authoritative nameserver resolves the hostname") @@ -313,15 +327,6 @@ var _ = Describe("DNSPolicy Targeting an Istio Gateway", func() { }) Context("None Provider", func() { - var dnsPolicy *v1alpha2.DNSPolicy - - AfterEach(func(ctx SpecContext) { - if dnsPolicy != nil { - err := k8sClient.Delete(ctx, dnsPolicy, - client.PropagationPolicy(metav1.DeletePropagationForeground)) - Expect(client.IgnoreNotFound(err)).ToNot(HaveOccurred()) - } - }) It("should create dns record with no zone assigned and record should not become ready", func(ctx SpecContext) { @@ -380,6 +385,7 @@ var _ = Describe("DNSPolicy Targeting an Istio Gateway", func() { "RecordTTL": Equal(v1alpha2.TTL(60)), })), )) + g.Expect(dnsrecord.Status.Endpoints).Should(BeEmpty()) }, TestTimeoutLong, time.Second, ctx).Should(Succeed()) }) diff --git a/test/e2e/gateway_single_spoke_test.go b/test/e2e/gateway_single_spoke_test.go index 1bd76b10..407b065f 100644 --- a/test/e2e/gateway_single_spoke_test.go +++ b/test/e2e/gateway_single_spoke_test.go @@ -64,6 +64,8 @@ var _ = Describe("Gateway single target cluster", func() { testHostnameOther = gatewayapiv1.Hostname(strings.Join([]string{"other-" + testID, string(testHostname)}, ".")) testHostnameWildcard = gatewayapiv1.Hostname(strings.Join([]string{"*", string(testHostname)}, ".")) + GinkgoWriter.Printf("[debug] testHostname: '%s'\n", testHostname) + By("creating a Placement for the Gateway resource") placement = &ocm_cluster_v1beta1.Placement{ ObjectMeta: metav1.ObjectMeta{Name: testID, Namespace: tconfig.HubNamespace()},