diff --git a/.gitignore b/.gitignore index 4b5eaa88..2d5320f3 100644 --- a/.gitignore +++ b/.gitignore @@ -24,4 +24,6 @@ bin *~ test-resources/test-suite.kind-config.yaml -test-resources/test-suite.metallb-pool.yaml \ No newline at end of file +test-resources/test-suite.metallb-pool.yaml + +local-dev \ No newline at end of file diff --git a/Makefile b/Makefile index 960f60cb..aeef7fe9 100644 --- a/Makefile +++ b/Makefile @@ -18,12 +18,33 @@ INGRESS_VERSION=4.9.1 HARBOR_VERSION=1.14.3 KIND_CLUSTER ?= remote-controller +KIND_NETWORK ?= remote-controller TIMEOUT = 30m HELM = helm KUBECTL = kubectl JQ = jq +ARCH := $(shell uname | tr '[:upper:]' '[:lower:]') + +KIND = $(realpath ./local-dev/kind) +KIND_VERSION = v0.25.0 + +.PHONY: local-dev/kind +local-dev/kind: +ifeq ($(KIND_VERSION), $(shell kind version 2>/dev/null | sed -nE 's/kind (v[0-9.]+).*/\1/p')) + $(info linking local kind version $(KIND_VERSION)) + ln -sf $(shell command -v kind) ./local-dev/kind +else +ifneq ($(KIND_VERSION), $(shell ./local-dev/kind version 2>/dev/null | sed -nE 's/kind (v[0-9.]+).*/\1/p')) + $(info downloading kind version $(KIND_VERSION) for $(ARCH)) + mkdir -p local-dev + rm local-dev/kind || true + curl -sSLo local-dev/kind https://kind.sigs.k8s.io/dl/$(KIND_VERSION)/kind-$(ARCH)-amd64 + chmod a+x local-dev/kind +endif +endif + # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) ifeq (,$(shell go env GOBIN)) GOBIN=$(shell go env GOPATH)/bin @@ -140,7 +161,7 @@ endif .PHONY: install-metallb install-metallb: - LAGOON_KIND_CIDR_BLOCK=$$(docker network inspect $(KIND_CLUSTER) | $(JQ) '. [0].IPAM.Config[0].Subnet' | tr -d '"') && \ + LAGOON_KIND_CIDR_BLOCK=$$(docker network inspect $(KIND_NETWORK) | $(JQ) '. [0].IPAM.Config[0].Subnet' | tr -d '"') && \ export LAGOON_KIND_NETWORK_RANGE=$$(echo $${LAGOON_KIND_CIDR_BLOCK%???} | awk -F'.' '{print $$1,$$2,$$3,240}' OFS='.')/29 && \ $(HELM) upgrade \ --install \ @@ -152,7 +173,7 @@ install-metallb: metallb \ metallb/metallb && \ $$(envsubst < test-resources/test-suite.metallb-pool.yaml.tpl > test-resources/test-suite.metallb-pool.yaml) && \ - $(KUBECTL) apply -f test-resources/test-suite.metallb-pool.yaml \ + $(KUBECTL) apply -f test-resources/test-suite.metallb-pool.yaml # cert-manager is used to allow self-signed certificates to be generated automatically by ingress in the same way lets-encrypt would .PHONY: install-certmanager @@ -246,11 +267,12 @@ install-lagoon-remote: install-registry .PHONY: create-kind-cluster create-kind-cluster: - docker network inspect $(KIND_CLUSTER) >/dev/null || docker network create $(KIND_CLUSTER) \ - && LAGOON_KIND_CIDR_BLOCK=$$(docker network inspect $(KIND_CLUSTER) | $(JQ) '. [0].IPAM.Config[0].Subnet' | tr -d '"') \ + docker network inspect $(KIND_NETWORK) >/dev/null || docker network create $(KIND_NETWORK) \ + && LAGOON_KIND_CIDR_BLOCK=$$(docker network inspect $(KIND_NETWORK) | $(JQ) '. [0].IPAM.Config[0].Subnet' | tr -d '"') \ && export KIND_NODE_IP=$$(echo $${LAGOON_KIND_CIDR_BLOCK%???} | awk -F'.' '{print $$1,$$2,$$3,240}' OFS='.') \ && envsubst < test-resources/test-suite.kind-config.yaml.tpl > test-resources/test-suite.kind-config.yaml \ - && kind create cluster --wait=60s --name=$(KIND_CLUSTER) --config=test-resources/test-suite.kind-config.yaml + && export KIND_EXPERIMENTAL_DOCKER_NETWORK=$(KIND_NETWORK) \ + && $(KIND) create cluster --wait=60s --name=$(KIND_CLUSTER) --config=test-resources/test-suite.kind-config.yaml # Create a kind cluster locally and run the test e2e test suite against it .PHONY: kind/test-e2e # Run the e2e tests against a Kind k8s instance that is spun up locally @@ -259,7 +281,7 @@ kind/test-e2e: create-kind-cluster install-lagoon-remote kind/re-test-e2e .PHONY: local-kind/test-e2e # Run the e2e tests against a Kind k8s instance that is spun up locally kind/re-test-e2e: export KIND_CLUSTER=$(KIND_CLUSTER) && \ - kind export kubeconfig --name=$(KIND_CLUSTER) && \ + $(KIND) export kubeconfig --name=$(KIND_CLUSTER) && \ export HARBOR_VERSION=$(HARBOR_VERSION) && \ export OVERRIDE_BUILD_DEPLOY_DIND_IMAGE=$(OVERRIDE_BUILD_DEPLOY_DIND_IMAGE) && \ $(MAKE) test-e2e @@ -267,7 +289,7 @@ kind/re-test-e2e: .PHONY: clean kind/clean: docker compose down && \ - kind delete cluster --name=$(KIND_CLUSTER) && docker network rm $(KIND_CLUSTER) + $(KIND) delete cluster --name=$(KIND_CLUSTER) && docker network rm $(KIND_NETWORK) # Utilize Kind or modify the e2e tests to load the image locally, enabling compatibility with other vendors. .PHONY: test-e2e # Run the e2e tests against a Kind k8s instance that is spun up inside github action. diff --git a/config/default/manager_auth_proxy_patch.yaml b/config/default/manager_auth_proxy_patch.yaml index 567b1d51..74a2a25a 100644 --- a/config/default/manager_auth_proxy_patch.yaml +++ b/config/default/manager_auth_proxy_patch.yaml @@ -24,6 +24,7 @@ spec: - "--enable-deprecated-apis" - "--lagoon-feature-flag-support-k8upv2" - "--skip-tls-verify" + - "--cleanup-harbor-repository-on-delete" # enabled for tests ports: - containerPort: 8443 name: https diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 29dae687..07585ff9 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -17,6 +17,7 @@ limitations under the License. package e2e import ( + "encoding/json" "fmt" "os" "os/exec" @@ -54,6 +55,10 @@ var ( "lagoon_tasks_running_current", "lagoon_tasks_started_total", } + + createPolicyWant = `{"algorithm":"or","rules":[{"action":"retain","params":{"latestPulledN":3},"scope_selectors":{"repository":[{"decoration":"repoMatches","kind":"doublestar","pattern":"[^pr\\-]*/*"}]},"tag_selectors":[{"decoration":"matches","extras":"{\"untagged\":true}","kind":"doublestar","pattern":"**"}],"template":"latestPulledN"},{"action":"retain","params":{"latestPulledN":1},"scope_selectors":{"repository":[{"decoration":"repoMatches","kind":"doublestar","pattern":"pr-*"}]},"tag_selectors":[{"decoration":"matches","extras":"{\"untagged\":true}","kind":"doublestar","pattern":"**"}],"template":"latestPulledN"}],"scope":{"level":"project"},"trigger":{"kind":"Schedule","settings":{"cron":"0 3 3 * * 3"}}}` + deletePolicyWant = `{"algorithm":"or","rules":[],"scope":{"level":"project"},"trigger":{"kind":"Schedule","settings":{"cron":""}}}` + projectRepositoriesWant = `[{"artifact_count":1,"name":"nginx-example/main/nginx","pull_count":1}]` ) func init() { @@ -62,12 +67,20 @@ func init() { } var _ = Describe("controller", Ordered, func() { + // get the ingress lb ip for use later + ip, err := utils.GetIngressLB() + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + BeforeAll(func() { By("start local services") Expect(utils.StartLocalServices()).To(Succeed()) + By("removing manager namespace") + cmd := exec.Command("kubectl", "delete", "ns", namespace) + _, _ = utils.Run(cmd) + By("creating manager namespace") - cmd := exec.Command("kubectl", "create", "ns", namespace) + cmd = exec.Command("kubectl", "create", "ns", namespace) _, _ = utils.Run(cmd) // when running a re-test, it is best to make sure the old namespace doesn't exist @@ -75,6 +88,9 @@ var _ = Describe("controller", Ordered, func() { // remove the old namespace cmd = exec.Command("kubectl", "delete", "ns", "nginx-example-main") _, _ = utils.Run(cmd) + By("delete harbor project") + _ = utils.DeleteHarborProject(ip, "nginx-example") + // clean up the k8up crds utils.UninstallK8upCRDs() }) @@ -84,10 +100,6 @@ var _ = Describe("controller", Ordered, func() { By("stop metrics consumer") utils.StopMetricsConsumer() - By("removing manager namespace") - cmd := exec.Command("kubectl", "delete", "ns", namespace) - _, _ = utils.Run(cmd) - By("stop local services") utils.StopLocalServices() }) @@ -164,22 +176,7 @@ var _ = Describe("controller", Ordered, func() { for _, name := range []string{"7m5zypx", "8m5zypx", "9m5zypx", "1m5zypx"} { if name == "9m5zypx" { By("creating a LagoonBuild resource via rabbitmq") - cmd = exec.Command( - "curl", - "-s", - "-u", - "guest:guest", - "-H", - "'Accept: application/json'", - "-H", - "'Content-Type:application/json'", - "-X", - "POST", - "-d", - fmt.Sprintf("@test/e2e/testdata/lagoon-build-%s.json", name), - "http://172.17.0.1:15672/api/exchanges/%2f/lagoon-tasks/publish", - ) - _, err = utils.Run(cmd) + err = utils.PublishMessage(fmt.Sprintf("@test/e2e/testdata/lagoon-build-%s.json", name)) ExpectWithOffset(1, err).NotTo(HaveOccurred()) } else { By("creating a LagoonBuild resource") @@ -334,22 +331,7 @@ var _ = Describe("controller", Ordered, func() { time.Sleep(5 * time.Second) By(fmt.Sprintf("creating a %s restore task via rabbitmq", name)) - cmd = exec.Command( - "curl", - "-s", - "-u", - "guest:guest", - "-H", - "'Accept: application/json'", - "-H", - "'Content-Type:application/json'", - "-X", - "POST", - "-d", - fmt.Sprintf("@test/e2e/testdata/%s-restore.json", name), - "http://172.17.0.1:15672/api/exchanges/%2f/lagoon-tasks/publish", - ) - _, err = utils.Run(cmd) + err = utils.PublishMessage(fmt.Sprintf("@test/e2e/testdata/%s-restore.json", name)) ExpectWithOffset(1, err).NotTo(HaveOccurred()) time.Sleep(10 * time.Second) @@ -400,23 +382,40 @@ var _ = Describe("controller", Ordered, func() { } EventuallyWithOffset(1, verifyRobotCredentialsRotate, duration, interval).Should(Succeed()) + By("check harbor project policies before creating policy") + _, err = utils.QueryHarborProjectPolicies(ip, "nginx-example") + if err != nil { + if !strings.Contains(err.Error(), "project metadata value is empty: retention_id") { + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + } + } + By("creating harbor policy update via rabbitmq") + err = utils.PublishMessage("@test/e2e/testdata/create-retention-policy.json") + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + time.Sleep(10 * time.Second) + By("check harbor project policies after creating policy") + after, err := utils.QueryHarborProjectPolicies(ip, "nginx-example") + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + err = comparePolicy(createPolicyWant, after) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + By("delete harbor policy via rabbitmq") + err = utils.PublishMessage("@test/e2e/testdata/remove-retention-policy.json") + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + time.Sleep(10 * time.Second) + By("check harbor project policies after deleting policy") + after, err = utils.QueryHarborProjectPolicies(ip, "nginx-example") + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + err = comparePolicy(deletePolicyWant, after) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + By("check harbor project repositories before deleting environment") + before, err := utils.QueryHarborRepositories(ip, "nginx-example") + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + err = compareRepositories(projectRepositoriesWant, before) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) By("delete environment via rabbitmq") - cmd = exec.Command( - "curl", - "-s", - "-u", - "guest:guest", - "-H", - "'Accept: application/json'", - "-H", - "'Content-Type:application/json'", - "-X", - "POST", - "-d", - "@test/e2e/testdata/remove-environment.json", - "http://172.17.0.1:15672/api/exchanges/%2f/lagoon-tasks/publish", - ) - _, err = utils.Run(cmd) + err = utils.PublishMessage("@test/e2e/testdata/remove-environment.json") ExpectWithOffset(1, err).NotTo(HaveOccurred()) By("validating that the namespace deletes") @@ -435,6 +434,12 @@ var _ = Describe("controller", Ordered, func() { } EventuallyWithOffset(1, verifyNamespaceRemoved, duration, interval).Should(Succeed()) + By("check harbor project repositories after deleting environment") + after, err = utils.QueryHarborRepositories(ip, "nginx-example") + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + err = compareRepositories("null", after) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + By("validating that unauthenticated metrics requests fail") runCmd := `curl -s -k https://remote-controller-controller-manager-metrics-service.remote-controller-system.svc.cluster.local:8443/metrics | grep -v "#" | grep "lagoon_"` _, err = utils.RunCommonsCommand(namespace, runCmd) @@ -451,3 +456,45 @@ var _ = Describe("controller", Ordered, func() { }) }) }) + +func comparePolicy(want, got string) error { + // this removes the next scheduled time from the payload as it can vary + var m map[string]interface{} + if err := json.Unmarshal([]byte(got), &m); err != nil { + return err + } + delete(m["scope"].(map[string]interface{}), "ref") + delete(m["trigger"].(map[string]interface{})["settings"].(map[string]interface{}), "next_scheduled_time") + delete(m, "id") + p, err := json.Marshal(m) + if err != nil { + return err + } + if want != string(p) { + return fmt.Errorf("resulting policies don't match:\nwant: %s\ngot: %s", want, string(p)) + } + return nil +} + +func compareRepositories(want, got string) error { + if got == "null" && want == "null" { + return nil + } + // this removes the next scheduled time from the payload as it can vary + var m []interface{} + if err := json.Unmarshal([]byte(got), &m); err != nil { + return err + } + delete(m[0].(map[string]interface{}), "id") + delete(m[0].(map[string]interface{}), "creation_time") + delete(m[0].(map[string]interface{}), "update_time") + delete(m[0].(map[string]interface{}), "project_id") + p, err := json.Marshal(m) + if err != nil { + return err + } + if want != string(p) { + return fmt.Errorf("resulting policies don't match:\nwant: %s\ngot: %s", want, string(p)) + } + return nil +} diff --git a/test/e2e/testdata/create-retention-policy.json b/test/e2e/testdata/create-retention-policy.json new file mode 100644 index 00000000..b0167eff --- /dev/null +++ b/test/e2e/testdata/create-retention-policy.json @@ -0,0 +1,20 @@ +{"properties":{"delivery_mode":2},"routing_key":"ci-local-controller-kubernetes:misc", + "payload":"{ + \"misc\":{ + \"miscResource\":\"eyJ0eXBlIjoiaGFyYm9yUmV0ZW50aW9uUG9saWN5IiwiZXZlbnRUeXBlIjoidXBkYXRlUG9saWN5IiwiZGF0YSI6eyJwcm9qZWN0Ijp7ImlkIjoxLCJuYW1lIjoibmdpbngtZXhhbXBsZSJ9LCJwb2xpY3kiOnsicnVsZXMiOlt7Im5hbWUiOiJhbGwgYnJhbmNoZXMsIGV4Y2x1ZGluZyBwdWxscmVxdWVzdHMiLCJwYXR0ZXJuIjoiW15wclxcLV0qLyoiLCJsYXRlc3RQdWxsZWQiOjN9LHsibmFtZSI6InB1bGxyZXF1ZXN0cyIsInBhdHRlcm4iOiJwci0qIiwibGF0ZXN0UHVsbGVkIjoxfV0sInNjaGVkdWxlIjoiMyAzICogKiAzIn19fQ==\" + }, + \"key\":\"deploytarget:harborpolicy:update\", + \"environment\":{ + \"name\":\"main\", + \"openshiftProjectName\":\"nginx-example-main\" + }, + \"project\":{ + \"name\":\"nginx-example\" + }, + \"advancedTask\":{} + }", +"payload_encoding":"string" +} + + + diff --git a/test/e2e/testdata/lagoon-build-1m5zypx.yaml b/test/e2e/testdata/lagoon-build-1m5zypx.yaml index ba50e261..e96436c2 100644 --- a/test/e2e/testdata/lagoon-build-1m5zypx.yaml +++ b/test/e2e/testdata/lagoon-build-1m5zypx.yaml @@ -13,6 +13,7 @@ spec: project: name: nginx-example environment: main + id: 1 organization: id: 123 name: test-org diff --git a/test/e2e/testdata/lagoon-build-7m5zypx.yaml b/test/e2e/testdata/lagoon-build-7m5zypx.yaml index f1ed59e8..893b69ff 100644 --- a/test/e2e/testdata/lagoon-build-7m5zypx.yaml +++ b/test/e2e/testdata/lagoon-build-7m5zypx.yaml @@ -13,6 +13,7 @@ spec: gitReference: origin/main project: name: nginx-example + id: 1 environment: main uiLink: https://dashboard.amazeeio.cloud/projects/project/project-environment/deployments/lagoon-build-7m5zypx routerPattern: 'main-nginx-example' diff --git a/test/e2e/testdata/lagoon-build-8m5zypx.yaml b/test/e2e/testdata/lagoon-build-8m5zypx.yaml index 1ac0dccf..9ed85fae 100644 --- a/test/e2e/testdata/lagoon-build-8m5zypx.yaml +++ b/test/e2e/testdata/lagoon-build-8m5zypx.yaml @@ -14,6 +14,7 @@ spec: project: name: nginx-example environment: main + id: 1 organization: id: 123 name: test-org diff --git a/test/e2e/testdata/lagoon-build-9m5zypx.json b/test/e2e/testdata/lagoon-build-9m5zypx.json index bcd95b64..e1e95be2 100644 --- a/test/e2e/testdata/lagoon-build-9m5zypx.json +++ b/test/e2e/testdata/lagoon-build-9m5zypx.json @@ -15,6 +15,11 @@ \"gitReference\": \"origin\/main\", \"project\": { \"name\": \"nginx-example\", + \"id\": 1, + \"organization\": { + \"id\": 123, + \"name\": \"test-org\" + }, \"environment\": \"main\", \"uiLink\": \"https:\/\/dashboard.amazeeio.cloud\/projects\/project\/project-environment\/deployments\/lagoon-build-9m5zypx\", \"routerPattern\": \"main-nginx-example\", diff --git a/test/e2e/testdata/remove-retention-policy.json b/test/e2e/testdata/remove-retention-policy.json new file mode 100644 index 00000000..84f75377 --- /dev/null +++ b/test/e2e/testdata/remove-retention-policy.json @@ -0,0 +1,20 @@ +{"properties":{"delivery_mode":2},"routing_key":"ci-local-controller-kubernetes:misc", + "payload":"{ + \"misc\":{ + \"miscResource\":\"eyJ0eXBlIjoiaGFyYm9yUmV0ZW50aW9uUG9saWN5IiwiZXZlbnRUeXBlIjoicmVtb3ZlUG9saWN5IiwiZGF0YSI6eyJwcm9qZWN0Ijp7ImlkIjoxLCJuYW1lIjoibmdpbngtZXhhbXBsZSJ9fX0=\" + }, + \"key\":\"deploytarget:harborpolicy:update\", + \"environment\":{ + \"name\":\"main\", + \"openshiftProjectName\":\"nginx-example-main\" + }, + \"project\":{ + \"name\":\"nginx-example\" + }, + \"advancedTask\":{} + }", +"payload_encoding":"string" +} + + + diff --git a/test/utils/utils.go b/test/utils/utils.go index 92e885ef..3575fb35 100644 --- a/test/utils/utils.go +++ b/test/utils/utils.go @@ -17,11 +17,18 @@ limitations under the License. package utils import ( + "context" + "crypto/tls" + "encoding/json" "fmt" + "net/http" "os" "os/exec" "strings" + harborclientv5 "github.com/mittwald/goharbor-client/v5/apiv2" + "github.com/mittwald/goharbor-client/v5/apiv2/pkg/config" + "github.com/onsi/ginkgo/v2" ) @@ -93,6 +100,85 @@ func RunCommonsCommand(ns, runCmd string) ([]byte, error) { return Run(cmd) } +func GetIngressLB() (string, error) { + cmd := exec.Command("kubectl", "-n", "ingress-nginx", + "get", "services", "ingress-nginx-controller", + "-o", "jsonpath={.status.loadBalancer.ingress[0].ip}", + ) + status, err := Run(cmd) + if err != nil { + return "", err + } + return string(status), nil +} + +// Removes a CRD file but doesn't cause a failure if already deleted +func PublishMessage(file string) error { + cmd := exec.Command( + "curl", + "-s", + "-u", + "guest:guest", + "-H", + "'Accept: application/json'", + "-H", + "'Content-Type:application/json'", + "-X", + "POST", + "-d", + file, + "http://172.17.0.1:15672/api/exchanges/%2f/lagoon-tasks/publish", + ) + _, err := Run(cmd) + return err +} + +func harborClient(ip string) (*harborclientv5.RESTClient, error) { + harborConfig := &config.Options{ + Page: 1, + PageSize: 100, + } + http.DefaultTransport.(*http.Transport).TLSClientConfig = &tls.Config{InsecureSkipVerify: true} + return harborclientv5.NewRESTClientForHost(fmt.Sprintf("https://registry.%s.nip.io/api/", ip), "admin", "Harbor12345", harborConfig) +} + +func QueryHarborRepositories(ip, projectName string) (string, error) { + c2, err := harborClient(ip) + if err != nil { + return "", fmt.Errorf("here1 %v", err) + } + ctx := context.Background() + listRepositories, err := c2.ListRepositories(ctx, projectName) + if err != nil { + return "", fmt.Errorf("here2 %v", err) + } + r1, _ := json.Marshal(listRepositories) + return string(r1), nil +} + +func QueryHarborProjectPolicies(ip, projectName string) (string, error) { + c2, err := harborClient(ip) + if err != nil { + return "", err + } + ctx := context.Background() + existingPolicy, err := c2.GetRetentionPolicyByProject(ctx, projectName) + if err != nil { + return "", err + } + r1, _ := json.Marshal(existingPolicy) + return string(r1), nil +} + +func DeleteHarborProject(ip, projectName string) error { + c2, err := harborClient(ip) + if err != nil { + return err + } + ctx := context.Background() + return c2.DeleteProject(ctx, projectName) +} + // Run executes the provided command within this context func Run(cmd *exec.Cmd) ([]byte, error) { dir, _ := GetProjectDir()