diff --git a/.github/workflows/remote-controller.yaml b/.github/workflows/remote-controller.yaml index 3ce7912e..a094fe41 100644 --- a/.github/workflows/remote-controller.yaml +++ b/.github/workflows/remote-controller.yaml @@ -94,8 +94,8 @@ jobs: - name: Configure node IP in kind-config.yaml run: | - docker network create remote-controller - LAGOON_KIND_CIDR_BLOCK=$(docker network inspect remote-controller | jq '. [0].IPAM.Config[0].Subnet' | tr -d '"') + docker network create kind + LAGOON_KIND_CIDR_BLOCK=$(docker network inspect kind | jq '. [0].IPAM.Config[0].Subnet' | tr -d '"') export KIND_NODE_IP=$(echo ${LAGOON_KIND_CIDR_BLOCK%???} | awk -F'.' '{print $1,$2,$3,240}' OFS='.') envsubst < test-resources/test-suite.kind-config.yaml.tpl > test-resources/test-suite.kind-config.yaml @@ -110,7 +110,7 @@ jobs: - name: Check node IP matches kind configuration run: | - LAGOON_KIND_CIDR_BLOCK=$(docker network inspect remote-controller | jq '. [0].IPAM.Config[0].Subnet' | tr -d '"') + LAGOON_KIND_CIDR_BLOCK=$(docker network inspect kind | jq '. [0].IPAM.Config[0].Subnet' | tr -d '"') NODE_IP=$(echo ${LAGOON_KIND_CIDR_BLOCK%???} | awk -F'.' '{print $1,$2,$3,240}' OFS='.') echo Checking for NODE_IP "$NODE_IP" grep $NODE_IP test-resources/test-suite.kind-config.yaml @@ -127,4 +127,4 @@ jobs: - name: Run github/test-e2e run: | - make github/test-e2e HARBOR_VERSION=${{matrix.harbor}} OVERRIDE_BUILD_DEPLOY_DIND_IMAGE="${{matrix.lagoon_build_image}}" \ No newline at end of file + make github/test-e2e HARBOR_VERSION=${{matrix.harbor}} OVERRIDE_BUILD_DEPLOY_DIND_IMAGE="${{matrix.lagoon_build_image}}" KIND_NETWORK=kind \ No newline at end of file diff --git a/.gitignore b/.gitignore index 4b5eaa88..2d5320f3 100644 --- a/.gitignore +++ b/.gitignore @@ -24,4 +24,6 @@ bin *~ test-resources/test-suite.kind-config.yaml -test-resources/test-suite.metallb-pool.yaml \ No newline at end of file +test-resources/test-suite.metallb-pool.yaml + +local-dev \ No newline at end of file diff --git a/Makefile b/Makefile index 960f60cb..0c4b89cf 100644 --- a/Makefile +++ b/Makefile @@ -18,12 +18,33 @@ INGRESS_VERSION=4.9.1 HARBOR_VERSION=1.14.3 KIND_CLUSTER ?= remote-controller +KIND_NETWORK ?= remote-controller TIMEOUT = 30m HELM = helm KUBECTL = kubectl JQ = jq +ARCH := $(shell uname | tr '[:upper:]' '[:lower:]') + +KIND = $(realpath ./local-dev/kind) +KIND_VERSION = v0.25.0 + +.PHONY: local-dev/kind +local-dev/kind: +ifeq ($(KIND_VERSION), $(shell kind version 2>/dev/null | sed -nE 's/kind (v[0-9.]+).*/\1/p')) + $(info linking local kind version $(KIND_VERSION)) + ln -sf $(shell command -v kind) ./local-dev/kind +else +ifneq ($(KIND_VERSION), $(shell ./local-dev/kind version 2>/dev/null | sed -nE 's/kind (v[0-9.]+).*/\1/p')) + $(info downloading kind version $(KIND_VERSION) for $(ARCH)) + mkdir -p local-dev + rm local-dev/kind || true + curl -sSLo local-dev/kind https://kind.sigs.k8s.io/dl/$(KIND_VERSION)/kind-$(ARCH)-amd64 + chmod a+x local-dev/kind +endif +endif + # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) ifeq (,$(shell go env GOBIN)) GOBIN=$(shell go env GOPATH)/bin @@ -140,7 +161,7 @@ endif .PHONY: install-metallb install-metallb: - LAGOON_KIND_CIDR_BLOCK=$$(docker network inspect $(KIND_CLUSTER) | $(JQ) '. [0].IPAM.Config[0].Subnet' | tr -d '"') && \ + LAGOON_KIND_CIDR_BLOCK=$$(docker network inspect $(KIND_NETWORK) | $(JQ) '. [0].IPAM.Config[0].Subnet' | tr -d '"') && \ export LAGOON_KIND_NETWORK_RANGE=$$(echo $${LAGOON_KIND_CIDR_BLOCK%???} | awk -F'.' '{print $$1,$$2,$$3,240}' OFS='.')/29 && \ $(HELM) upgrade \ --install \ @@ -152,7 +173,7 @@ install-metallb: metallb \ metallb/metallb && \ $$(envsubst < test-resources/test-suite.metallb-pool.yaml.tpl > test-resources/test-suite.metallb-pool.yaml) && \ - $(KUBECTL) apply -f test-resources/test-suite.metallb-pool.yaml \ + $(KUBECTL) apply -f test-resources/test-suite.metallb-pool.yaml # cert-manager is used to allow self-signed certificates to be generated automatically by ingress in the same way lets-encrypt would .PHONY: install-certmanager @@ -246,11 +267,12 @@ install-lagoon-remote: install-registry .PHONY: create-kind-cluster create-kind-cluster: - docker network inspect $(KIND_CLUSTER) >/dev/null || docker network create $(KIND_CLUSTER) \ - && LAGOON_KIND_CIDR_BLOCK=$$(docker network inspect $(KIND_CLUSTER) | $(JQ) '. [0].IPAM.Config[0].Subnet' | tr -d '"') \ + docker network inspect $(KIND_NETWORK) >/dev/null || docker network create $(KIND_NETWORK) \ + && LAGOON_KIND_CIDR_BLOCK=$$(docker network inspect $(KIND_NETWORK) | $(JQ) '. [0].IPAM.Config[0].Subnet' | tr -d '"') \ && export KIND_NODE_IP=$$(echo $${LAGOON_KIND_CIDR_BLOCK%???} | awk -F'.' '{print $$1,$$2,$$3,240}' OFS='.') \ && envsubst < test-resources/test-suite.kind-config.yaml.tpl > test-resources/test-suite.kind-config.yaml \ - && kind create cluster --wait=60s --name=$(KIND_CLUSTER) --config=test-resources/test-suite.kind-config.yaml + && export KIND_EXPERIMENTAL_DOCKER_NETWORK=$(KIND_NETWORK) \ + && $(KIND) create cluster --wait=60s --name=$(KIND_CLUSTER) --config=test-resources/test-suite.kind-config.yaml # Create a kind cluster locally and run the test e2e test suite against it .PHONY: kind/test-e2e # Run the e2e tests against a Kind k8s instance that is spun up locally @@ -259,7 +281,7 @@ kind/test-e2e: create-kind-cluster install-lagoon-remote kind/re-test-e2e .PHONY: local-kind/test-e2e # Run the e2e tests against a Kind k8s instance that is spun up locally kind/re-test-e2e: export KIND_CLUSTER=$(KIND_CLUSTER) && \ - kind export kubeconfig --name=$(KIND_CLUSTER) && \ + $(KIND) export kubeconfig --name=$(KIND_CLUSTER) && \ export HARBOR_VERSION=$(HARBOR_VERSION) && \ export OVERRIDE_BUILD_DEPLOY_DIND_IMAGE=$(OVERRIDE_BUILD_DEPLOY_DIND_IMAGE) && \ $(MAKE) test-e2e @@ -267,14 +289,14 @@ kind/re-test-e2e: .PHONY: clean kind/clean: docker compose down && \ - kind delete cluster --name=$(KIND_CLUSTER) && docker network rm $(KIND_CLUSTER) + $(KIND) delete cluster --name=$(KIND_CLUSTER) && docker network rm $(KIND_NETWORK) # Utilize Kind or modify the e2e tests to load the image locally, enabling compatibility with other vendors. .PHONY: test-e2e # Run the e2e tests against a Kind k8s instance that is spun up inside github action. test-e2e: export HARBOR_VERSION=$(HARBOR_VERSION) && \ export OVERRIDE_BUILD_DEPLOY_DIND_IMAGE=$(OVERRIDE_BUILD_DEPLOY_DIND_IMAGE) && \ - go test ./test/e2e/ -v -ginkgo.v + go test ./test/e2e/ -v -ginkgo.v -timeout 20m .PHONY: github/test-e2e github/test-e2e: install-lagoon-remote test-e2e diff --git a/cmd/main.go b/cmd/main.go index 09831d0a..0ff4c5ce 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -686,6 +686,7 @@ func main() { enableDebug, lffSupportK8UPv2, cache, + harborConfig, ) c := cron.New() diff --git a/config/default/manager_auth_proxy_patch.yaml b/config/default/manager_auth_proxy_patch.yaml index 567b1d51..74a2a25a 100644 --- a/config/default/manager_auth_proxy_patch.yaml +++ b/config/default/manager_auth_proxy_patch.yaml @@ -24,6 +24,7 @@ spec: - "--enable-deprecated-apis" - "--lagoon-feature-flag-support-k8upv2" - "--skip-tls-verify" + - "--cleanup-harbor-repository-on-delete" # enabled for tests ports: - containerPort: 8443 name: https diff --git a/go.mod b/go.mod index c5a6d564..c02d5770 100644 --- a/go.mod +++ b/go.mod @@ -40,6 +40,7 @@ require ( github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927 // indirect + github.com/cxmcc/unixsums v0.0.0-20131125091133-89564297d82f // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect github.com/evanphx/json-patch/v5 v5.9.0 // indirect diff --git a/go.sum b/go.sum index 98bf770c..0c53a08c 100644 --- a/go.sum +++ b/go.sum @@ -339,6 +339,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46t github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/cxmcc/unixsums v0.0.0-20131125091133-89564297d82f h1:PkAFGgVtJnasAxOaiEY1RYPx8W+7X7l66vi8T2apKCM= +github.com/cxmcc/unixsums v0.0.0-20131125091133-89564297d82f/go.mod h1:XJq7OckzkOtlgeEKFwkH2gFbc1+1WRFUBf7QnvfyrzQ= github.com/cyphar/filepath-securejoin v0.2.2/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4= github.com/cyphar/filepath-securejoin v0.2.3/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= github.com/d2g/dhcp4 v0.0.0-20170904100407-a1d1b6c41b1c/go.mod h1:Ct2BUK8SB0YC1SMSibvLzxjeJLnrYEVLULFNiHY9YfQ= diff --git a/internal/controllers/v1beta1/build_controller.go b/internal/controllers/v1beta1/build_controller.go index 9a523bee..1876d099 100644 --- a/internal/controllers/v1beta1/build_controller.go +++ b/internal/controllers/v1beta1/build_controller.go @@ -326,7 +326,7 @@ func (r *LagoonBuildReconciler) createNamespaceBuild(ctx context.Context, continue } // send the status change to lagoon - r.updateDeploymentAndEnvironmentTask(ctx, opLog, runningBuild, nil, buildCondition, "cancelled") + r.updateDeploymentAndEnvironmentTask(opLog, runningBuild, nil, buildCondition, "cancelled") continue } // handle processing running but no pod/failed pod builds diff --git a/internal/controllers/v1beta1/build_deletionhandlers.go b/internal/controllers/v1beta1/build_deletionhandlers.go index 3b85dd8a..3e4e46c3 100644 --- a/internal/controllers/v1beta1/build_deletionhandlers.go +++ b/internal/controllers/v1beta1/build_deletionhandlers.go @@ -212,16 +212,16 @@ Build cancelled } // send any messages to lagoon message queues // update the deployment with the status of cancelled in lagoon - r.buildStatusLogsToLagoonLogs(ctx, opLog, &lagoonBuild, &lagoonEnv, lagoonv1beta1.BuildStatusCancelled, "cancelled") - r.updateDeploymentAndEnvironmentTask(ctx, opLog, &lagoonBuild, &lagoonEnv, lagoonv1beta1.BuildStatusCancelled, "cancelled") - r.buildLogsToLagoonLogs(ctx, opLog, &lagoonBuild, allContainerLogs, lagoonv1beta1.BuildStatusCancelled) + r.buildStatusLogsToLagoonLogs(opLog, &lagoonBuild, &lagoonEnv, lagoonv1beta1.BuildStatusCancelled, "cancelled") + r.updateDeploymentAndEnvironmentTask(opLog, &lagoonBuild, &lagoonEnv, lagoonv1beta1.BuildStatusCancelled, "cancelled") + r.buildLogsToLagoonLogs(opLog, &lagoonBuild, allContainerLogs, lagoonv1beta1.BuildStatusCancelled) } return nil } // buildLogsToLagoonLogs sends the build logs to the lagoon-logs message queue // it contains the actual pod log output that is sent to elasticsearch, it is what eventually is displayed in the UI -func (r *LagoonBuildReconciler) buildLogsToLagoonLogs(ctx context.Context, +func (r *LagoonBuildReconciler) buildLogsToLagoonLogs( opLog logr.Logger, lagoonBuild *lagoonv1beta1.LagoonBuild, logs []byte, @@ -267,7 +267,7 @@ func (r *LagoonBuildReconciler) buildLogsToLagoonLogs(ctx context.Context, // updateDeploymentAndEnvironmentTask sends the status of the build and deployment to the controllerhandler message queue in lagoon, // this is for the handler in lagoon to process. -func (r *LagoonBuildReconciler) updateDeploymentAndEnvironmentTask(ctx context.Context, +func (r *LagoonBuildReconciler) updateDeploymentAndEnvironmentTask( opLog logr.Logger, lagoonBuild *lagoonv1beta1.LagoonBuild, lagoonEnv *corev1.ConfigMap, @@ -361,7 +361,7 @@ func (r *LagoonBuildReconciler) updateDeploymentAndEnvironmentTask(ctx context.C } // buildStatusLogsToLagoonLogs sends the logs to lagoon-logs message queue, used for general messaging -func (r *LagoonBuildReconciler) buildStatusLogsToLagoonLogs(ctx context.Context, +func (r *LagoonBuildReconciler) buildStatusLogsToLagoonLogs( opLog logr.Logger, lagoonBuild *lagoonv1beta1.LagoonBuild, lagoonEnv *corev1.ConfigMap, diff --git a/internal/controllers/v1beta1/build_helpers.go b/internal/controllers/v1beta1/build_helpers.go index 56ed05f2..7a2796a3 100644 --- a/internal/controllers/v1beta1/build_helpers.go +++ b/internal/controllers/v1beta1/build_helpers.go @@ -210,7 +210,7 @@ func (r *LagoonBuildReconciler) getOrCreateNamespace(ctx context.Context, namesp return fmt.Errorf("error getting harbor version, check your harbor configuration. Error was: %v", err) } if lagoonHarbor.UseV2Functions(curVer) { - hProject, err := lagoonHarbor.CreateProjectV2(ctx, lagoonBuild.Spec.Project.Name) + hProject, err := lagoonHarbor.CreateProjectV2(ctx, *namespace) if err != nil { return fmt.Errorf("error creating harbor project: %v", err) } @@ -330,8 +330,9 @@ func (r *LagoonBuildReconciler) processBuild(ctx context.Context, opLog logr.Log } var serviceaccountTokenSecret string + reg, _ := regexp.Compile("^lagoon-deployer-token") for _, secret := range serviceAccount.Secrets { - match, _ := regexp.MatchString("^lagoon-deployer-token", secret.Name) + match := reg.MatchString(secret.Name) if match { serviceaccountTokenSecret = secret.Name break @@ -896,13 +897,13 @@ func (r *LagoonBuildReconciler) updateQueuedBuild( // send any messages to lagoon message queues // update the deployment with the status, lagoon v2.12.0 supports queued status, otherwise use pending if lagoonv1beta1.CheckLagoonVersion(&lagoonBuild, "2.12.0") { - r.buildStatusLogsToLagoonLogs(ctx, opLog, &lagoonBuild, &lagoonEnv, lagoonv1beta1.BuildStatusQueued, fmt.Sprintf("queued %v/%v", queuePosition, queueLength)) - r.updateDeploymentAndEnvironmentTask(ctx, opLog, &lagoonBuild, &lagoonEnv, lagoonv1beta1.BuildStatusQueued, fmt.Sprintf("queued %v/%v", queuePosition, queueLength)) - r.buildLogsToLagoonLogs(ctx, opLog, &lagoonBuild, allContainerLogs, lagoonv1beta1.BuildStatusQueued) + r.buildStatusLogsToLagoonLogs(opLog, &lagoonBuild, &lagoonEnv, lagoonv1beta1.BuildStatusQueued, fmt.Sprintf("queued %v/%v", queuePosition, queueLength)) + r.updateDeploymentAndEnvironmentTask(opLog, &lagoonBuild, &lagoonEnv, lagoonv1beta1.BuildStatusQueued, fmt.Sprintf("queued %v/%v", queuePosition, queueLength)) + r.buildLogsToLagoonLogs(opLog, &lagoonBuild, allContainerLogs, lagoonv1beta1.BuildStatusQueued) } else { - r.buildStatusLogsToLagoonLogs(ctx, opLog, &lagoonBuild, &lagoonEnv, lagoonv1beta1.BuildStatusPending, fmt.Sprintf("queued %v/%v", queuePosition, queueLength)) - r.updateDeploymentAndEnvironmentTask(ctx, opLog, &lagoonBuild, &lagoonEnv, lagoonv1beta1.BuildStatusPending, fmt.Sprintf("queued %v/%v", queuePosition, queueLength)) - r.buildLogsToLagoonLogs(ctx, opLog, &lagoonBuild, allContainerLogs, lagoonv1beta1.BuildStatusPending) + r.buildStatusLogsToLagoonLogs(opLog, &lagoonBuild, &lagoonEnv, lagoonv1beta1.BuildStatusPending, fmt.Sprintf("queued %v/%v", queuePosition, queueLength)) + r.updateDeploymentAndEnvironmentTask(opLog, &lagoonBuild, &lagoonEnv, lagoonv1beta1.BuildStatusPending, fmt.Sprintf("queued %v/%v", queuePosition, queueLength)) + r.buildLogsToLagoonLogs(opLog, &lagoonBuild, allContainerLogs, lagoonv1beta1.BuildStatusPending) } return nil @@ -955,10 +956,10 @@ Build cancelled } // send any messages to lagoon message queues // update the deployment with the status of cancelled in lagoon - r.buildStatusLogsToLagoonLogs(ctx, opLog, &lagoonBuild, &lagoonEnv, lagoonv1beta1.BuildStatusCancelled, "cancelled") - r.updateDeploymentAndEnvironmentTask(ctx, opLog, &lagoonBuild, &lagoonEnv, lagoonv1beta1.BuildStatusCancelled, "cancelled") + r.buildStatusLogsToLagoonLogs(opLog, &lagoonBuild, &lagoonEnv, lagoonv1beta1.BuildStatusCancelled, "cancelled") + r.updateDeploymentAndEnvironmentTask(opLog, &lagoonBuild, &lagoonEnv, lagoonv1beta1.BuildStatusCancelled, "cancelled") if cancelled { - r.buildLogsToLagoonLogs(ctx, opLog, &lagoonBuild, allContainerLogs, lagoonv1beta1.BuildStatusCancelled) + r.buildLogsToLagoonLogs(opLog, &lagoonBuild, allContainerLogs, lagoonv1beta1.BuildStatusCancelled) } // delete the build from the lagoon namespace in kubernetes entirely err = r.Delete(ctx, &lagoonBuild) diff --git a/internal/controllers/v1beta1/task_controller.go b/internal/controllers/v1beta1/task_controller.go index b9572290..387b9867 100644 --- a/internal/controllers/v1beta1/task_controller.go +++ b/internal/controllers/v1beta1/task_controller.go @@ -97,7 +97,7 @@ func (r *LagoonTaskReconciler) Reconcile(ctx context.Context, req ctrl.Request) // The object is being deleted if helpers.ContainsString(lagoonTask.ObjectMeta.Finalizers, taskFinalizer) { // our finalizer is present, so lets handle any external dependency - if err := r.deleteExternalResources(ctx, &lagoonTask, req.NamespacedName.Namespace); err != nil { + if err := r.deleteExternalResources(); err != nil { // if fail to delete the external dependency here, return with error // so that it can be retried return ctrl.Result{}, err @@ -130,7 +130,7 @@ func (r *LagoonTaskReconciler) SetupWithManager(mgr ctrl.Manager) error { Complete(r) } -func (r *LagoonTaskReconciler) deleteExternalResources(ctx context.Context, lagoonTask *lagoonv1beta1.LagoonTask, namespace string) error { +func (r *LagoonTaskReconciler) deleteExternalResources() error { // delete any external resources if required return nil } @@ -266,34 +266,34 @@ func (r *LagoonTaskReconciler) getTaskPodDeployment(ctx context.Context, lagoonT lagoonTask.Spec.Task.Command, } dep.Spec.Template.Spec.RestartPolicy = "Never" - taskPod := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: lagoonTask.ObjectMeta.Name, - Namespace: lagoonTask.ObjectMeta.Namespace, - Labels: map[string]string{ - "lagoon.sh/jobType": "task", - "lagoon.sh/taskName": lagoonTask.ObjectMeta.Name, - "lagoon.sh/crdVersion": crdVersion, - "lagoon.sh/controller": r.ControllerNamespace, - }, - OwnerReferences: []metav1.OwnerReference{ - { - APIVersion: fmt.Sprintf("%v", lagoonv1beta1.GroupVersion), - Kind: "LagoonTask", - Name: lagoonTask.ObjectMeta.Name, - UID: lagoonTask.UID, - }, + } + taskPod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: lagoonTask.ObjectMeta.Name, + Namespace: lagoonTask.ObjectMeta.Namespace, + Labels: map[string]string{ + "lagoon.sh/jobType": "task", + "lagoon.sh/taskName": lagoonTask.ObjectMeta.Name, + "lagoon.sh/crdVersion": crdVersion, + "lagoon.sh/controller": r.ControllerNamespace, + }, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: fmt.Sprintf("%v", lagoonv1beta1.GroupVersion), + Kind: "LagoonTask", + Name: lagoonTask.ObjectMeta.Name, + UID: lagoonTask.UID, }, }, - Spec: dep.Spec.Template.Spec, - } - // set the organization labels on task pods - if lagoonTask.Spec.Project.Organization != nil { - taskPod.ObjectMeta.Labels["organization.lagoon.sh/id"] = fmt.Sprintf("%d", *lagoonTask.Spec.Project.Organization.ID) - taskPod.ObjectMeta.Labels["organization.lagoon.sh/name"] = lagoonTask.Spec.Project.Organization.Name - } - return taskPod, nil + }, + Spec: dep.Spec.Template.Spec, + } + // set the organization labels on task pods + if lagoonTask.Spec.Project.Organization != nil { + taskPod.ObjectMeta.Labels["organization.lagoon.sh/id"] = fmt.Sprintf("%d", *lagoonTask.Spec.Project.Organization.ID) + taskPod.ObjectMeta.Labels["organization.lagoon.sh/name"] = lagoonTask.Spec.Project.Organization.Name } + return taskPod, nil } } if !hasService { @@ -425,8 +425,9 @@ func (r *LagoonTaskReconciler) createAdvancedTask(ctx context.Context, lagoonTas return err } var serviceaccountTokenSecret string + reg, _ := regexp.Compile("^lagoon-deployer-token") for _, secret := range serviceAccount.Secrets { - match, _ := regexp.MatchString("^lagoon-deployer-token", secret.Name) + match := reg.MatchString(secret.Name) if match { serviceaccountTokenSecret = secret.Name break diff --git a/internal/controllers/v1beta2/build_helpers.go b/internal/controllers/v1beta2/build_helpers.go index a72d03e7..8923fab4 100644 --- a/internal/controllers/v1beta2/build_helpers.go +++ b/internal/controllers/v1beta2/build_helpers.go @@ -214,7 +214,7 @@ func (r *LagoonBuildReconciler) getOrCreateNamespace(ctx context.Context, namesp return fmt.Errorf("error getting harbor version, check your harbor configuration. Error was: %v", err) } if lagoonHarbor.UseV2Functions(curVer) { - hProject, err := lagoonHarbor.CreateProjectV2(ctx, lagoonBuild.Spec.Project.Name) + hProject, err := lagoonHarbor.CreateProjectV2(ctx, *namespace) if err != nil { return fmt.Errorf("error creating harbor project: %v", err) } diff --git a/internal/controllers/v1beta2/task_controller.go b/internal/controllers/v1beta2/task_controller.go index 6eea9829..3183fa1f 100644 --- a/internal/controllers/v1beta2/task_controller.go +++ b/internal/controllers/v1beta2/task_controller.go @@ -83,7 +83,7 @@ func (r *LagoonTaskReconciler) Reconcile(ctx context.Context, req ctrl.Request) // The object is being deleted if helpers.ContainsString(lagoonTask.ObjectMeta.Finalizers, taskFinalizer) { // our finalizer is present, so lets handle any external dependency - if err := r.deleteExternalResources(ctx, &lagoonTask, req.NamespacedName.Namespace); err != nil { + if err := r.deleteExternalResources(ctx, &lagoonTask, ""); err != nil { // if fail to delete the external dependency here, return with error // so that it can be retried return ctrl.Result{}, err diff --git a/internal/harbor/harbor_credentialrotation.go b/internal/harbor/harbor_credentialrotation.go index bad6a37a..1eae05f9 100644 --- a/internal/harbor/harbor_credentialrotation.go +++ b/internal/harbor/harbor_credentialrotation.go @@ -71,7 +71,7 @@ func (h *Harbor) RotateRobotCredential(ctx context.Context, cl client.Client, ns return false, fmt.Errorf("error checking harbor version: %v", err) } if h.UseV2Functions(curVer) { - hProject, err := h.CreateProjectV2(ctx, ns.Labels["lagoon.sh/project"]) + hProject, err := h.CreateProjectV2(ctx, ns) if err != nil { return false, fmt.Errorf("error getting or creating project: %v", err) } diff --git a/internal/harbor/harbor_project.go b/internal/harbor/harbor_project.go new file mode 100644 index 00000000..bef80800 --- /dev/null +++ b/internal/harbor/harbor_project.go @@ -0,0 +1,173 @@ +package harbor + +import ( + "context" + "fmt" + "strings" + + harborclientv5model "github.com/mittwald/goharbor-client/v5/apiv2/model" + "github.com/uselagoon/remote-controller/internal/helpers" + corev1 "k8s.io/api/core/v1" +) + +// CreateProjectV2 will create a project if one doesn't exist, but will update as required. +func (h *Harbor) CreateProjectV2(ctx context.Context, namespace corev1.Namespace) (*harborclientv5model.Project, error) { + projectName := namespace.Labels["lagoon.sh/project"] + exists, err := h.ClientV5.ProjectExists(ctx, projectName) + if err != nil { + h.Log.Info(fmt.Sprintf("Error checking project %s exists, err: %v", projectName, err)) + return nil, err + } + if !exists { + err := h.ClientV5.NewProject(ctx, &harborclientv5model.ProjectReq{ + ProjectName: projectName, + }) + if err != nil { + h.Log.Info(fmt.Sprintf("Error creating project %s, err: %v", projectName, err)) + return nil, err + } + project, err := h.ClientV5.GetProject(ctx, projectName) + if err != nil { + h.Log.Info(fmt.Sprintf("Error getting project %s, err: %v", projectName, err)) + return nil, err + } + stor := int64(-1) + tStr := "true" + project.Metadata = &harborclientv5model.ProjectMetadata{ + AutoScan: &tStr, + ReuseSysCVEAllowlist: &tStr, + Public: "false", + } + err = h.ClientV5.UpdateProject(ctx, project, &stor) + if err != nil { + h.Log.Info(fmt.Sprintf("Error updating project %s, err: %v", projectName, err)) + return nil, err + } + } + project, err := h.ClientV5.GetProject(ctx, projectName) + if err != nil { + h.Log.Info(fmt.Sprintf("Error getting project %s, err: %v", projectName, err)) + return nil, err + } + + // TODO: Repository support not required yet + // this is a place holder + // w, err := h.ClientV5.ListRepositories(ctx, int(project.ProjectID)) + // if err != nil { + // return nil, err + // } + // for _, x := range w { + // fmt.Println(x) + // } + + if h.WebhookAddition { + wps, err := h.ClientV5.ListProjectWebhookPolicies(ctx, int(project.ProjectID)) + if err != nil { + h.Log.Info(fmt.Sprintf("Error listing project %s webhooks", project.Name)) + return nil, err + } + exists := false + for _, wp := range wps { + // if the webhook policy already exists with the name we want + // then update it with any changes that may be required + if wp.Name == "Lagoon Default Webhook" { + exists = true + wp.Targets = []*harborclientv5model.WebhookTargetObject{ + { + Type: "http", + SkipCertVerify: true, + Address: h.WebhookURL, + }, + } + wp.Enabled = true + wp.EventTypes = []string{"SCANNING_FAILED", "SCANNING_COMPLETED"} + err = h.ClientV5.UpdateProjectWebhookPolicy(ctx, int(wp.ProjectID), wp) + if err != nil { + h.Log.Info(fmt.Sprintf("Error updating project %s webhook", project.Name)) + return nil, err + } + } + } + if !exists { + // otherwise create the webhook if it doesn't exist + newPolicy := &harborclientv5model.WebhookPolicy{ + Name: "Lagoon Default Webhook", + ProjectID: int64(project.ProjectID), + Enabled: true, + Targets: []*harborclientv5model.WebhookTargetObject{ + { + Type: "http", + SkipCertVerify: true, + Address: h.WebhookURL, + }, + }, + EventTypes: []string{"SCANNING_FAILED", "SCANNING_COMPLETED"}, + } + err = h.ClientV5.AddProjectWebhookPolicy(ctx, int(project.ProjectID), newPolicy) + if err != nil { + h.Log.Info(fmt.Sprintf("Error adding project %s webhook", project.Name)) + return nil, err + } + } + } + return project, nil +} + +// DeleteRepository will delete repositories related to an environment +func (h *Harbor) DeleteRepository(ctx context.Context, projectName, branch string) { + environmentName := helpers.ShortenEnvironment(projectName, helpers.MakeSafe(branch)) + h.Config.PageSize = 100 + pageCount := int64(1) + listRepositories := h.ListRepositories(ctx, projectName) + for _, repo := range listRepositories { + if strings.Contains(repo.Name, fmt.Sprintf("%s/%s", projectName, environmentName)) { + repoName := strings.Replace(repo.Name, fmt.Sprintf("%s/", projectName), "", 1) + err := h.ClientV5.DeleteRepository(ctx, projectName, repoName) + if err != nil { + h.Log.Info(fmt.Sprintf("Error deleting harbor repository %s", repo.Name)) + } + h.Log.Info( + fmt.Sprintf( + "Deleted harbor repository %s in project %s, environment %s", + repo.Name, + projectName, + environmentName, + ), + ) + } + } + if len(listRepositories) > 100 { + // h.Log.Info(fmt.Sprintf("more than pagesize repositories returned")) + pageCount = int64(len(listRepositories) / 100) + var page int64 + for page = 2; page <= pageCount; page++ { + listRepositories := h.ListRepositories(ctx, projectName) + for _, repo := range listRepositories { + if strings.Contains(repo.Name, fmt.Sprintf("%s/%s", projectName, environmentName)) { + repoName := strings.Replace(repo.Name, fmt.Sprintf("%s/", projectName), "", 1) + err := h.ClientV5.DeleteRepository(ctx, projectName, repoName) + if err != nil { + h.Log.Info(fmt.Sprintf("Error deleting harbor repository %s", repo.Name)) + } + h.Log.Info( + fmt.Sprintf( + "Deleted harbor repository %s in project %s, environment %s", + repo.Name, + projectName, + environmentName, + ), + ) + } + } + } + } +} + +// ListRepositories . +func (h *Harbor) ListRepositories(ctx context.Context, projectName string) []*harborclientv5model.Repository { + listRepositories, err := h.ClientV5.ListRepositories(ctx, projectName) + if err != nil { + h.Log.Info(fmt.Sprintf("Error listing harbor repositories for project %s", projectName)) + } + return listRepositories +} diff --git a/internal/harbor/harbor22x.go b/internal/harbor/harbor_robot.go similarity index 59% rename from internal/harbor/harbor22x.go rename to internal/harbor/harbor_robot.go index f6416b30..1e9a9905 100644 --- a/internal/harbor/harbor22x.go +++ b/internal/harbor/harbor_robot.go @@ -5,7 +5,6 @@ import ( "encoding/json" "fmt" "math" - "strings" "time" harborclientv5model "github.com/mittwald/goharbor-client/v5/apiv2/model" @@ -15,108 +14,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -// CreateProjectV2 will create a project if one doesn't exist, but will update as required. -func (h *Harbor) CreateProjectV2(ctx context.Context, projectName string) (*harborclientv5model.Project, error) { - exists, err := h.ClientV5.ProjectExists(ctx, projectName) - if err != nil { - h.Log.Info(fmt.Sprintf("Error checking project %s exists, err: %v", projectName, err)) - return nil, err - } - if !exists { - err := h.ClientV5.NewProject(ctx, &harborclientv5model.ProjectReq{ - ProjectName: projectName, - }) - if err != nil { - h.Log.Info(fmt.Sprintf("Error creating project %s, err: %v", projectName, err)) - return nil, err - } - project, err := h.ClientV5.GetProject(ctx, projectName) - if err != nil { - h.Log.Info(fmt.Sprintf("Error getting project %s, err: %v", projectName, err)) - return nil, err - } - stor := int64(-1) - tStr := "true" - project.Metadata = &harborclientv5model.ProjectMetadata{ - AutoScan: &tStr, - ReuseSysCVEAllowlist: &tStr, - Public: "false", - } - err = h.ClientV5.UpdateProject(ctx, project, &stor) - if err != nil { - h.Log.Info(fmt.Sprintf("Error updating project %s, err: %v", projectName, err)) - return nil, err - } - } - project, err := h.ClientV5.GetProject(ctx, projectName) - if err != nil { - h.Log.Info(fmt.Sprintf("Error getting project %s, err: %v", projectName, err)) - return nil, err - } - - // TODO: Repository support not required yet - // this is a place holder - // w, err := h.ClientV5.ListRepositories(ctx, int(project.ProjectID)) - // if err != nil { - // return nil, err - // } - // for _, x := range w { - // fmt.Println(x) - // } - - if h.WebhookAddition { - wps, err := h.ClientV5.ListProjectWebhookPolicies(ctx, int(project.ProjectID)) - if err != nil { - h.Log.Info(fmt.Sprintf("Error listing project %s webhooks", project.Name)) - return nil, err - } - exists := false - for _, wp := range wps { - // if the webhook policy already exists with the name we want - // then update it with any changes that may be required - if wp.Name == "Lagoon Default Webhook" { - exists = true - wp.Targets = []*harborclientv5model.WebhookTargetObject{ - { - Type: "http", - SkipCertVerify: true, - Address: h.WebhookURL, - }, - } - wp.Enabled = true - wp.EventTypes = []string{"SCANNING_FAILED", "SCANNING_COMPLETED"} - err = h.ClientV5.UpdateProjectWebhookPolicy(ctx, int(wp.ProjectID), wp) - if err != nil { - h.Log.Info(fmt.Sprintf("Error updating project %s webhook", project.Name)) - return nil, err - } - } - } - if !exists { - // otherwise create the webhook if it doesn't exist - newPolicy := &harborclientv5model.WebhookPolicy{ - Name: "Lagoon Default Webhook", - ProjectID: int64(project.ProjectID), - Enabled: true, - Targets: []*harborclientv5model.WebhookTargetObject{ - { - Type: "http", - SkipCertVerify: true, - Address: h.WebhookURL, - }, - }, - EventTypes: []string{"SCANNING_FAILED", "SCANNING_COMPLETED"}, - } - err = h.ClientV5.AddProjectWebhookPolicy(ctx, int(project.ProjectID), newPolicy) - if err != nil { - h.Log.Info(fmt.Sprintf("Error adding project %s webhook", project.Name)) - return nil, err - } - } - } - return project, nil -} - // CreateOrRefreshRobotV2 will create or refresh a robot account and return the credentials if needed. func (h *Harbor) CreateOrRefreshRobotV2(ctx context.Context, k8s client.Client, @@ -265,40 +162,6 @@ func (h *Harbor) CreateOrRefreshRobotV2(ctx context.Context, return nil, err } -// DeleteRepository will delete repositories related to an environment -func (h *Harbor) DeleteRepository(ctx context.Context, projectName, branch string) { - environmentName := helpers.ShortenEnvironment(projectName, helpers.MakeSafe(branch)) - h.Config.PageSize = 100 - pageCount := int64(1) - listRepositories := h.ListRepositories(ctx, projectName) - for _, repo := range listRepositories { - if strings.Contains(repo.Name, fmt.Sprintf("%s/%s", projectName, environmentName)) { - repoName := strings.Replace(repo.Name, fmt.Sprintf("%s/", projectName), "", 1) - err := h.ClientV5.DeleteRepository(ctx, projectName, repoName) - if err != nil { - h.Log.Info(fmt.Sprintf("Error deleting harbor repository %s", repo.Name)) - } - } - } - if len(listRepositories) > 100 { - // h.Log.Info(fmt.Sprintf("more than pagesize repositories returned")) - pageCount = int64(len(listRepositories) / 100) - var page int64 - for page = 2; page <= pageCount; page++ { - listRepositories := h.ListRepositories(ctx, projectName) - for _, repo := range listRepositories { - if strings.Contains(repo.Name, fmt.Sprintf("%s/%s", projectName, environmentName)) { - repoName := strings.Replace(repo.Name, fmt.Sprintf("%s/", projectName), "", 1) - err := h.ClientV5.DeleteRepository(ctx, projectName, repoName) - if err != nil { - h.Log.Info(fmt.Sprintf("Error deleting harbor repository %s", repo.Name)) - } - } - } - } - } -} - // DeleteRobotAccount will delete robot account related to an environment func (h *Harbor) DeleteRobotAccount(ctx context.Context, projectName, branch string) { environmentName := helpers.ShortenEnvironment(projectName, helpers.MakeSafe(branch)) @@ -321,19 +184,18 @@ func (h *Harbor) DeleteRobotAccount(ctx context.Context, projectName, branch str h.Log.Info(fmt.Sprintf("Error deleting project %s robot account %s", projectName, robot.Name)) return } + h.Log.Info( + fmt.Sprintf( + "Deleted harbor robot account %s in project %s, environment %s", + robot.Name, + projectName, + environmentName, + ), + ) } } } -// ListRepositories . -func (h *Harbor) ListRepositories(ctx context.Context, projectName string) []*harborclientv5model.Repository { - listRepositories, err := h.ClientV5.ListRepositories(ctx, projectName) - if err != nil { - h.Log.Info(fmt.Sprintf("Error listing harbor repositories for project %s", projectName)) - } - return listRepositories -} - func (h *Harbor) CreateRobotAccountV2(ctx context.Context, robotName, projectName string, expiryDays int64) (*helpers.RegistryCredentials, error) { robotf := harborclientv5model.RobotCreate{ Level: "project", diff --git a/internal/messenger/consumer.go b/internal/messenger/consumer.go index 7a3e0b02..a29907c4 100644 --- a/internal/messenger/consumer.go +++ b/internal/messenger/consumer.go @@ -104,7 +104,7 @@ func (m *Messenger) Consumer(targetName string) { //error { message.Ack(false) // ack to remove from queue }) if err != nil { - log.Fatalf(fmt.Sprintf("Failed to set handler to consumer `%s`: %v", "builddeploy-queue", err)) + log.Fatalf("Failed to set handler to consumer `%s`: %v", "builddeploy-queue", err) } // Handle any tasks that go to the `remove` queue @@ -228,7 +228,7 @@ func (m *Messenger) Consumer(targetName string) { //error { message.Ack(false) // ack to remove from queue }) if err != nil { - log.Fatalf(fmt.Sprintf("Failed to set handler to consumer `%s`: %v", "remove-queue", err)) + log.Fatalf("Failed to set handler to consumer `%s`: %v", "remove-queue", err) } // Handle any tasks that go to the `jobs` queue @@ -286,7 +286,7 @@ func (m *Messenger) Consumer(targetName string) { //error { message.Ack(false) // ack to remove from queue }) if err != nil { - log.Fatalf(fmt.Sprintf("Failed to set handler to consumer `%s`: %v", "jobs-queue", err)) + log.Fatalf("Failed to set handler to consumer `%s`: %v", "jobs-queue", err) } // Handle any tasks that go to the `misc` queue @@ -298,14 +298,6 @@ func (m *Messenger) Consumer(targetName string) { //error { jobSpec := &lagoonv1beta2.LagoonTaskSpec{} json.Unmarshal(message.Body(), jobSpec) // check which key has been received - namespace := helpers.GenerateNamespaceName( - jobSpec.Project.NamespacePattern, // the namespace pattern or `openshiftProjectPattern` from Lagoon is never received by the controller - jobSpec.Environment.Name, - jobSpec.Project.Name, - m.NamespacePrefix, - m.ControllerNamespace, - m.RandomNamespacePrefix, - ) switch jobSpec.Key { case "deploytarget:build:cancel", "kubernetes:build:cancel": opLog.Info( @@ -313,18 +305,18 @@ func (m *Messenger) Consumer(targetName string) { //error { "Received build cancellation for project %s, environment %s - %s", jobSpec.Project.Name, jobSpec.Environment.Name, - namespace, + m.genNamespace(jobSpec), ), ) m.Cache.Add(jobSpec.Misc.Name, jobSpec.Project.Name) // check if there is a v1beta2 task to cancel - hasv1beta2Build, v1beta2Bytes, err := lagoonv1beta2.CancelBuild(ctx, m.Client, namespace, message.Body()) + hasv1beta2Build, v1beta2Bytes, err := lagoonv1beta2.CancelBuild(ctx, m.Client, m.genNamespace(jobSpec), message.Body()) if err != nil { //@TODO: send msg back to lagoon and update task to failed? message.Ack(false) // ack to remove from queue return } - hasv1beta1Build, v1beta1Bytes, err := lagoonv1beta1.CancelBuild(ctx, m.Client, namespace, message.Body()) + hasv1beta1Build, v1beta1Bytes, err := lagoonv1beta1.CancelBuild(ctx, m.Client, m.genNamespace(jobSpec), message.Body()) if err != nil { //@TODO: send msg back to lagoon and update build to failed? message.Ack(false) // ack to remove from queue @@ -362,18 +354,18 @@ func (m *Messenger) Consumer(targetName string) { //error { "Received task cancellation for project %s, environment %s - %s", jobSpec.Project.Name, jobSpec.Environment.Name, - namespace, + m.genNamespace(jobSpec), ), ) m.Cache.Add(jobSpec.Task.TaskName, jobSpec.Project.Name) // check if there is a v1beta2 task to cancel - hasv1beta2Task, v1beta2Bytes, err := lagoonv1beta2.CancelTask(ctx, m.Client, namespace, message.Body()) + hasv1beta2Task, v1beta2Bytes, err := lagoonv1beta2.CancelTask(ctx, m.Client, m.genNamespace(jobSpec), message.Body()) if err != nil { //@TODO: send msg back to lagoon and update task to failed? message.Ack(false) // ack to remove from queue return } - hasv1beta1Task, v1beta1Bytes, err := lagoonv1beta1.CancelTask(ctx, m.Client, namespace, message.Body()) + hasv1beta1Task, v1beta1Bytes, err := lagoonv1beta1.CancelTask(ctx, m.Client, m.genNamespace(jobSpec), message.Body()) if err != nil { //@TODO: send msg back to lagoon and update task to failed? message.Ack(false) // ack to remove from queue @@ -413,7 +405,7 @@ func (m *Messenger) Consumer(targetName string) { //error { jobSpec.Environment.Name, ), ) - err := m.ResticRestore(namespace, jobSpec) + err := m.ResticRestore(m.genNamespace(jobSpec), jobSpec) if err != nil { opLog.Error(err, fmt.Sprintf( @@ -433,7 +425,7 @@ func (m *Messenger) Consumer(targetName string) { //error { jobSpec.Project.Name, ), ) - err := m.IngressRouteMigration(namespace, jobSpec) + err := m.IngressRouteMigration(m.genNamespace(jobSpec), jobSpec) if err != nil { opLog.Error(err, fmt.Sprintf( @@ -453,7 +445,7 @@ func (m *Messenger) Consumer(targetName string) { //error { jobSpec.Project.Name, ), ) - err := m.AdvancedTask(namespace, jobSpec) + err := m.AdvancedTask(m.genNamespace(jobSpec), jobSpec) if err != nil { opLog.Error(err, fmt.Sprintf( @@ -473,7 +465,7 @@ func (m *Messenger) Consumer(targetName string) { //error { jobSpec.Project.Name, ), ) - err := m.ActiveStandbySwitch(namespace, jobSpec) + err := m.ActiveStandbySwitch(m.genNamespace(jobSpec), jobSpec) if err != nil { opLog.Error(err, fmt.Sprintf( @@ -486,6 +478,18 @@ func (m *Messenger) Consumer(targetName string) { //error { message.Ack(false) // ack to remove from queue return } + case "deploytarget:harborpolicy:update": + err := m.HarborPolicy(ctx, jobSpec) + if err != nil { + opLog.Error(err, + fmt.Sprintf( + "Harbor policy update for project %s failed", + jobSpec.Project.Name, + ), + ) + message.Ack(false) // ack to remove from queue + return + } default: // if we get something that we don't know about, spit out the entire message opLog.Info( @@ -499,7 +503,18 @@ func (m *Messenger) Consumer(targetName string) { //error { message.Ack(false) // ack to remove from queue }) if err != nil { - log.Fatalf(fmt.Sprintf("Failed to set handler to consumer `%s`: %v", "misc-queue", err)) + log.Fatalf("Failed to set handler to consumer `%s`: %v", "misc-queue", err) } <-forever } + +func (m *Messenger) genNamespace(jobSpec *lagoonv1beta2.LagoonTaskSpec) string { + return helpers.GenerateNamespaceName( + jobSpec.Project.NamespacePattern, // the namespace pattern or `openshiftProjectPattern` from Lagoon is never received by the controller + jobSpec.Environment.Name, + jobSpec.Project.Name, + m.NamespacePrefix, + m.ControllerNamespace, + m.RandomNamespacePrefix, + ) +} diff --git a/internal/messenger/messenger.go b/internal/messenger/messenger.go index 76fdfcd2..c42a7d40 100644 --- a/internal/messenger/messenger.go +++ b/internal/messenger/messenger.go @@ -3,6 +3,7 @@ package messenger import ( "github.com/cheshir/go-mq/v2" "github.com/hashicorp/golang-lru/v2/expirable" + "github.com/uselagoon/remote-controller/internal/harbor" "github.com/uselagoon/remote-controller/internal/utilities/deletions" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -32,6 +33,7 @@ type Messenger struct { EnableDebug bool SupportK8upV2 bool Cache *expirable.LRU[string, string] + Harbor harbor.Harbor } // New returns a messaging with config and controller-runtime client. @@ -48,6 +50,7 @@ func New(config mq.Config, enableDebug bool, supportK8upV2 bool, cache *expirable.LRU[string, string], + harbor harbor.Harbor, ) *Messenger { return &Messenger{ Config: config, @@ -63,5 +66,6 @@ func New(config mq.Config, EnableDebug: enableDebug, SupportK8upV2: supportK8upV2, Cache: cache, + Harbor: harbor, } } diff --git a/internal/messenger/task_harborpolicy.go b/internal/messenger/task_harborpolicy.go new file mode 100644 index 00000000..80d51f9d --- /dev/null +++ b/internal/messenger/task_harborpolicy.go @@ -0,0 +1,186 @@ +package messenger + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + harborclientv5model "github.com/mittwald/goharbor-client/v5/apiv2/model" + "github.com/uselagoon/machinery/utils/cron" + lagoonv1beta2 "github.com/uselagoon/remote-controller/api/lagoon/v1beta2" + "github.com/uselagoon/remote-controller/internal/harbor" + ctrl "sigs.k8s.io/controller-runtime" +) + +type RetentionEvent struct { + Type string `json:"type"` // defines the action type + EventType string `json:"eventType"` // defines the eventtype field in the event notification + Data Data `json:"data"` // contains the payload for the action, this could be any json so using a map +} + +type Data struct { + Project Project `json:"project"` + Policy HarborRetentionPolicy `json:"policy"` +} + +type Project struct { + ID int `json:"id"` + Name string `json:"name"` +} + +type HarborRetentionPolicy struct { + Enabled bool `json:"enabled"` + Rules []HarborRetentionRule `json:"rules"` + Schedule string `json:"schedule"` +} + +type HarborRetentionRule struct { + Name string `json:"name"` + Pattern string `json:"pattern"` + LatestPulled uint64 `json:"latestPulled"` +} + +// HarborPolicy handles harbor retention policy changes. +func (m *Messenger) HarborPolicy(ctx context.Context, jobSpec *lagoonv1beta2.LagoonTaskSpec) error { + opLog := ctrl.Log.WithName("handlers").WithName("LagoonTasks") + retPol := &RetentionEvent{} + if err := json.Unmarshal(jobSpec.Misc.MiscResource, retPol); err != nil { + return err + } + lagoonHarbor, err := harbor.New(m.Harbor) + if err != nil { + return err + } + projectName := retPol.Data.Project.Name + project, err := lagoonHarbor.ClientV5.GetProject(ctx, projectName) + if err != nil { + opLog.Info(fmt.Sprintf("Error getting project %s, err: %v", projectName, err)) + return err + } + // handle the creation and updating of retention policies as required + // create the retention policy as required + var retentionPolicy *harborclientv5model.RetentionPolicy + switch retPol.EventType { + case "updatePolicy": + opLog.Info( + fmt.Sprintf( + "Received harbor policy update for project %s", + projectName, + ), + ) + // if this is updating or adding a policy, handle that here + retentionPolicy, err = m.generateRetentionPolicy(int64(project.ProjectID), projectName, retPol.Data.Policy) + if err != nil { + opLog.Info(fmt.Sprintf("Error generating retention policy for project %s, err: %v", projectName, err)) + return err + } + case "removePolicy": + opLog.Info( + fmt.Sprintf( + "Received harbor policy removal for project %s", + projectName, + ), + ) + // add an empty retention policy + retentionPolicy = m.generateEmptyRetentionPolicy(int64(project.ProjectID)) + default: + return fmt.Errorf("unable to determine policy type") + } + // get the existing one if one exists + existingPolicy, err := lagoonHarbor.ClientV5.GetRetentionPolicyByProject(ctx, projectName) + if err != nil && !strings.Contains(err.Error(), "project metadata value is empty: retention_id") { + opLog.Info(fmt.Sprintf("Error getting retention policy %s: %v", project.Name, err)) + return err + } + if existingPolicy != nil { + retentionPolicy.ID = existingPolicy.ID + r1, _ := json.Marshal(existingPolicy) + r2, _ := json.Marshal(retentionPolicy) + // if the policy differs, then we need to update it with our new policy + if string(r1) != string(r2) { + err := lagoonHarbor.ClientV5.UpdateRetentionPolicy(ctx, retentionPolicy) + if err != nil { + f, _ := json.Marshal(err) + opLog.Info(fmt.Sprintf("Error updating retention policy %s: %v", project.Name, string(f))) + return err + } + opLog.Info(fmt.Sprintf("retention policy for %s updated", project.Name)) + } + } else { + // create it if it doesn't + if err := lagoonHarbor.ClientV5.NewRetentionPolicy(ctx, retentionPolicy); err != nil { + opLog.Info(fmt.Sprintf("Error creating retention policy %s: %v", project.Name, err)) + return err + } + opLog.Info(fmt.Sprintf("retention policy for %s created", project.Name)) + } + return nil +} + +func (m *Messenger) generateRetentionPolicy(projectID int64, projectName string, policy HarborRetentionPolicy) (*harborclientv5model.RetentionPolicy, error) { + // generate a somewhat random schedule from the retention schedule template, using the harbor projectname as the seed + schedule, err := cron.ConvertCrontab(projectName, policy.Schedule) + if err != nil { + return nil, fmt.Errorf("error generating retention schedule %s: %v", projectName, err) + } + retPol := &harborclientv5model.RetentionPolicy{ + Algorithm: "or", + Scope: &harborclientv5model.RetentionPolicyScope{ + Level: "project", + Ref: projectID, + }, + Rules: []*harborclientv5model.RetentionRule{}, + Trigger: &harborclientv5model.RetentionRuleTrigger{ + Kind: "Schedule", + Settings: map[string]string{ + "cron": fmt.Sprintf("0 %s", schedule), // harbor needs seconds :\ just add a 0 pad to the start of the schedule + }, + }, + } + for _, rule := range policy.Rules { + retPol.Rules = append(retPol.Rules, + &harborclientv5model.RetentionRule{ + Action: "retain", + Params: map[string]interface{}{ + "latestPulledN": rule.LatestPulled, + }, + ScopeSelectors: map[string][]harborclientv5model.RetentionSelector{ + "repository": { + { + Decoration: "repoMatches", + Kind: "doublestar", + Pattern: rule.Pattern, + }, + }, + }, + TagSelectors: []*harborclientv5model.RetentionSelector{ + { + Decoration: "matches", + Extras: "{\"untagged\":true}", + Kind: "doublestar", + Pattern: "**", + }, + }, + Template: "latestPulledN", + }, + ) + } + return retPol, nil +} + +func (m *Messenger) generateEmptyRetentionPolicy(projectID int64) *harborclientv5model.RetentionPolicy { + return &harborclientv5model.RetentionPolicy{ + Algorithm: "or", + Rules: []*harborclientv5model.RetentionRule{}, + Scope: &harborclientv5model.RetentionPolicyScope{ + Level: "project", + Ref: projectID, + }, + Trigger: &harborclientv5model.RetentionRuleTrigger{ + Kind: "Schedule", + Settings: map[string]string{ + "cron": "", + }, + }} +} diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 29dae687..07585ff9 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -17,6 +17,7 @@ limitations under the License. package e2e import ( + "encoding/json" "fmt" "os" "os/exec" @@ -54,6 +55,10 @@ var ( "lagoon_tasks_running_current", "lagoon_tasks_started_total", } + + createPolicyWant = `{"algorithm":"or","rules":[{"action":"retain","params":{"latestPulledN":3},"scope_selectors":{"repository":[{"decoration":"repoMatches","kind":"doublestar","pattern":"[^pr\\-]*/*"}]},"tag_selectors":[{"decoration":"matches","extras":"{\"untagged\":true}","kind":"doublestar","pattern":"**"}],"template":"latestPulledN"},{"action":"retain","params":{"latestPulledN":1},"scope_selectors":{"repository":[{"decoration":"repoMatches","kind":"doublestar","pattern":"pr-*"}]},"tag_selectors":[{"decoration":"matches","extras":"{\"untagged\":true}","kind":"doublestar","pattern":"**"}],"template":"latestPulledN"}],"scope":{"level":"project"},"trigger":{"kind":"Schedule","settings":{"cron":"0 3 3 * * 3"}}}` + deletePolicyWant = `{"algorithm":"or","rules":[],"scope":{"level":"project"},"trigger":{"kind":"Schedule","settings":{"cron":""}}}` + projectRepositoriesWant = `[{"artifact_count":1,"name":"nginx-example/main/nginx","pull_count":1}]` ) func init() { @@ -62,12 +67,20 @@ func init() { } var _ = Describe("controller", Ordered, func() { + // get the ingress lb ip for use later + ip, err := utils.GetIngressLB() + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + BeforeAll(func() { By("start local services") Expect(utils.StartLocalServices()).To(Succeed()) + By("removing manager namespace") + cmd := exec.Command("kubectl", "delete", "ns", namespace) + _, _ = utils.Run(cmd) + By("creating manager namespace") - cmd := exec.Command("kubectl", "create", "ns", namespace) + cmd = exec.Command("kubectl", "create", "ns", namespace) _, _ = utils.Run(cmd) // when running a re-test, it is best to make sure the old namespace doesn't exist @@ -75,6 +88,9 @@ var _ = Describe("controller", Ordered, func() { // remove the old namespace cmd = exec.Command("kubectl", "delete", "ns", "nginx-example-main") _, _ = utils.Run(cmd) + By("delete harbor project") + _ = utils.DeleteHarborProject(ip, "nginx-example") + // clean up the k8up crds utils.UninstallK8upCRDs() }) @@ -84,10 +100,6 @@ var _ = Describe("controller", Ordered, func() { By("stop metrics consumer") utils.StopMetricsConsumer() - By("removing manager namespace") - cmd := exec.Command("kubectl", "delete", "ns", namespace) - _, _ = utils.Run(cmd) - By("stop local services") utils.StopLocalServices() }) @@ -164,22 +176,7 @@ var _ = Describe("controller", Ordered, func() { for _, name := range []string{"7m5zypx", "8m5zypx", "9m5zypx", "1m5zypx"} { if name == "9m5zypx" { By("creating a LagoonBuild resource via rabbitmq") - cmd = exec.Command( - "curl", - "-s", - "-u", - "guest:guest", - "-H", - "'Accept: application/json'", - "-H", - "'Content-Type:application/json'", - "-X", - "POST", - "-d", - fmt.Sprintf("@test/e2e/testdata/lagoon-build-%s.json", name), - "http://172.17.0.1:15672/api/exchanges/%2f/lagoon-tasks/publish", - ) - _, err = utils.Run(cmd) + err = utils.PublishMessage(fmt.Sprintf("@test/e2e/testdata/lagoon-build-%s.json", name)) ExpectWithOffset(1, err).NotTo(HaveOccurred()) } else { By("creating a LagoonBuild resource") @@ -334,22 +331,7 @@ var _ = Describe("controller", Ordered, func() { time.Sleep(5 * time.Second) By(fmt.Sprintf("creating a %s restore task via rabbitmq", name)) - cmd = exec.Command( - "curl", - "-s", - "-u", - "guest:guest", - "-H", - "'Accept: application/json'", - "-H", - "'Content-Type:application/json'", - "-X", - "POST", - "-d", - fmt.Sprintf("@test/e2e/testdata/%s-restore.json", name), - "http://172.17.0.1:15672/api/exchanges/%2f/lagoon-tasks/publish", - ) - _, err = utils.Run(cmd) + err = utils.PublishMessage(fmt.Sprintf("@test/e2e/testdata/%s-restore.json", name)) ExpectWithOffset(1, err).NotTo(HaveOccurred()) time.Sleep(10 * time.Second) @@ -400,23 +382,40 @@ var _ = Describe("controller", Ordered, func() { } EventuallyWithOffset(1, verifyRobotCredentialsRotate, duration, interval).Should(Succeed()) + By("check harbor project policies before creating policy") + _, err = utils.QueryHarborProjectPolicies(ip, "nginx-example") + if err != nil { + if !strings.Contains(err.Error(), "project metadata value is empty: retention_id") { + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + } + } + By("creating harbor policy update via rabbitmq") + err = utils.PublishMessage("@test/e2e/testdata/create-retention-policy.json") + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + time.Sleep(10 * time.Second) + By("check harbor project policies after creating policy") + after, err := utils.QueryHarborProjectPolicies(ip, "nginx-example") + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + err = comparePolicy(createPolicyWant, after) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + By("delete harbor policy via rabbitmq") + err = utils.PublishMessage("@test/e2e/testdata/remove-retention-policy.json") + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + time.Sleep(10 * time.Second) + By("check harbor project policies after deleting policy") + after, err = utils.QueryHarborProjectPolicies(ip, "nginx-example") + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + err = comparePolicy(deletePolicyWant, after) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + By("check harbor project repositories before deleting environment") + before, err := utils.QueryHarborRepositories(ip, "nginx-example") + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + err = compareRepositories(projectRepositoriesWant, before) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) By("delete environment via rabbitmq") - cmd = exec.Command( - "curl", - "-s", - "-u", - "guest:guest", - "-H", - "'Accept: application/json'", - "-H", - "'Content-Type:application/json'", - "-X", - "POST", - "-d", - "@test/e2e/testdata/remove-environment.json", - "http://172.17.0.1:15672/api/exchanges/%2f/lagoon-tasks/publish", - ) - _, err = utils.Run(cmd) + err = utils.PublishMessage("@test/e2e/testdata/remove-environment.json") ExpectWithOffset(1, err).NotTo(HaveOccurred()) By("validating that the namespace deletes") @@ -435,6 +434,12 @@ var _ = Describe("controller", Ordered, func() { } EventuallyWithOffset(1, verifyNamespaceRemoved, duration, interval).Should(Succeed()) + By("check harbor project repositories after deleting environment") + after, err = utils.QueryHarborRepositories(ip, "nginx-example") + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + err = compareRepositories("null", after) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + By("validating that unauthenticated metrics requests fail") runCmd := `curl -s -k https://remote-controller-controller-manager-metrics-service.remote-controller-system.svc.cluster.local:8443/metrics | grep -v "#" | grep "lagoon_"` _, err = utils.RunCommonsCommand(namespace, runCmd) @@ -451,3 +456,45 @@ var _ = Describe("controller", Ordered, func() { }) }) }) + +func comparePolicy(want, got string) error { + // this removes the next scheduled time from the payload as it can vary + var m map[string]interface{} + if err := json.Unmarshal([]byte(got), &m); err != nil { + return err + } + delete(m["scope"].(map[string]interface{}), "ref") + delete(m["trigger"].(map[string]interface{})["settings"].(map[string]interface{}), "next_scheduled_time") + delete(m, "id") + p, err := json.Marshal(m) + if err != nil { + return err + } + if want != string(p) { + return fmt.Errorf("resulting policies don't match:\nwant: %s\ngot: %s", want, string(p)) + } + return nil +} + +func compareRepositories(want, got string) error { + if got == "null" && want == "null" { + return nil + } + // this removes the next scheduled time from the payload as it can vary + var m []interface{} + if err := json.Unmarshal([]byte(got), &m); err != nil { + return err + } + delete(m[0].(map[string]interface{}), "id") + delete(m[0].(map[string]interface{}), "creation_time") + delete(m[0].(map[string]interface{}), "update_time") + delete(m[0].(map[string]interface{}), "project_id") + p, err := json.Marshal(m) + if err != nil { + return err + } + if want != string(p) { + return fmt.Errorf("resulting policies don't match:\nwant: %s\ngot: %s", want, string(p)) + } + return nil +} diff --git a/test/e2e/testdata/create-retention-policy.json b/test/e2e/testdata/create-retention-policy.json new file mode 100644 index 00000000..b0167eff --- /dev/null +++ b/test/e2e/testdata/create-retention-policy.json @@ -0,0 +1,20 @@ +{"properties":{"delivery_mode":2},"routing_key":"ci-local-controller-kubernetes:misc", + "payload":"{ + \"misc\":{ + \"miscResource\":\"eyJ0eXBlIjoiaGFyYm9yUmV0ZW50aW9uUG9saWN5IiwiZXZlbnRUeXBlIjoidXBkYXRlUG9saWN5IiwiZGF0YSI6eyJwcm9qZWN0Ijp7ImlkIjoxLCJuYW1lIjoibmdpbngtZXhhbXBsZSJ9LCJwb2xpY3kiOnsicnVsZXMiOlt7Im5hbWUiOiJhbGwgYnJhbmNoZXMsIGV4Y2x1ZGluZyBwdWxscmVxdWVzdHMiLCJwYXR0ZXJuIjoiW15wclxcLV0qLyoiLCJsYXRlc3RQdWxsZWQiOjN9LHsibmFtZSI6InB1bGxyZXF1ZXN0cyIsInBhdHRlcm4iOiJwci0qIiwibGF0ZXN0UHVsbGVkIjoxfV0sInNjaGVkdWxlIjoiMyAzICogKiAzIn19fQ==\" + }, + \"key\":\"deploytarget:harborpolicy:update\", + \"environment\":{ + \"name\":\"main\", + \"openshiftProjectName\":\"nginx-example-main\" + }, + \"project\":{ + \"name\":\"nginx-example\" + }, + \"advancedTask\":{} + }", +"payload_encoding":"string" +} + + + diff --git a/test/e2e/testdata/lagoon-build-1m5zypx.yaml b/test/e2e/testdata/lagoon-build-1m5zypx.yaml index ba50e261..e96436c2 100644 --- a/test/e2e/testdata/lagoon-build-1m5zypx.yaml +++ b/test/e2e/testdata/lagoon-build-1m5zypx.yaml @@ -13,6 +13,7 @@ spec: project: name: nginx-example environment: main + id: 1 organization: id: 123 name: test-org diff --git a/test/e2e/testdata/lagoon-build-7m5zypx.yaml b/test/e2e/testdata/lagoon-build-7m5zypx.yaml index f1ed59e8..893b69ff 100644 --- a/test/e2e/testdata/lagoon-build-7m5zypx.yaml +++ b/test/e2e/testdata/lagoon-build-7m5zypx.yaml @@ -13,6 +13,7 @@ spec: gitReference: origin/main project: name: nginx-example + id: 1 environment: main uiLink: https://dashboard.amazeeio.cloud/projects/project/project-environment/deployments/lagoon-build-7m5zypx routerPattern: 'main-nginx-example' diff --git a/test/e2e/testdata/lagoon-build-8m5zypx.yaml b/test/e2e/testdata/lagoon-build-8m5zypx.yaml index 1ac0dccf..9ed85fae 100644 --- a/test/e2e/testdata/lagoon-build-8m5zypx.yaml +++ b/test/e2e/testdata/lagoon-build-8m5zypx.yaml @@ -14,6 +14,7 @@ spec: project: name: nginx-example environment: main + id: 1 organization: id: 123 name: test-org diff --git a/test/e2e/testdata/lagoon-build-9m5zypx.json b/test/e2e/testdata/lagoon-build-9m5zypx.json index bcd95b64..e1e95be2 100644 --- a/test/e2e/testdata/lagoon-build-9m5zypx.json +++ b/test/e2e/testdata/lagoon-build-9m5zypx.json @@ -15,6 +15,11 @@ \"gitReference\": \"origin\/main\", \"project\": { \"name\": \"nginx-example\", + \"id\": 1, + \"organization\": { + \"id\": 123, + \"name\": \"test-org\" + }, \"environment\": \"main\", \"uiLink\": \"https:\/\/dashboard.amazeeio.cloud\/projects\/project\/project-environment\/deployments\/lagoon-build-9m5zypx\", \"routerPattern\": \"main-nginx-example\", diff --git a/test/e2e/testdata/remove-retention-policy.json b/test/e2e/testdata/remove-retention-policy.json new file mode 100644 index 00000000..84f75377 --- /dev/null +++ b/test/e2e/testdata/remove-retention-policy.json @@ -0,0 +1,20 @@ +{"properties":{"delivery_mode":2},"routing_key":"ci-local-controller-kubernetes:misc", + "payload":"{ + \"misc\":{ + \"miscResource\":\"eyJ0eXBlIjoiaGFyYm9yUmV0ZW50aW9uUG9saWN5IiwiZXZlbnRUeXBlIjoicmVtb3ZlUG9saWN5IiwiZGF0YSI6eyJwcm9qZWN0Ijp7ImlkIjoxLCJuYW1lIjoibmdpbngtZXhhbXBsZSJ9fX0=\" + }, + \"key\":\"deploytarget:harborpolicy:update\", + \"environment\":{ + \"name\":\"main\", + \"openshiftProjectName\":\"nginx-example-main\" + }, + \"project\":{ + \"name\":\"nginx-example\" + }, + \"advancedTask\":{} + }", +"payload_encoding":"string" +} + + + diff --git a/test/utils/utils.go b/test/utils/utils.go index 92e885ef..3575fb35 100644 --- a/test/utils/utils.go +++ b/test/utils/utils.go @@ -17,11 +17,18 @@ limitations under the License. package utils import ( + "context" + "crypto/tls" + "encoding/json" "fmt" + "net/http" "os" "os/exec" "strings" + harborclientv5 "github.com/mittwald/goharbor-client/v5/apiv2" + "github.com/mittwald/goharbor-client/v5/apiv2/pkg/config" + "github.com/onsi/ginkgo/v2" ) @@ -93,6 +100,85 @@ func RunCommonsCommand(ns, runCmd string) ([]byte, error) { return Run(cmd) } +func GetIngressLB() (string, error) { + cmd := exec.Command("kubectl", "-n", "ingress-nginx", + "get", "services", "ingress-nginx-controller", + "-o", "jsonpath={.status.loadBalancer.ingress[0].ip}", + ) + status, err := Run(cmd) + if err != nil { + return "", err + } + return string(status), nil +} + +// Removes a CRD file but doesn't cause a failure if already deleted +func PublishMessage(file string) error { + cmd := exec.Command( + "curl", + "-s", + "-u", + "guest:guest", + "-H", + "'Accept: application/json'", + "-H", + "'Content-Type:application/json'", + "-X", + "POST", + "-d", + file, + "http://172.17.0.1:15672/api/exchanges/%2f/lagoon-tasks/publish", + ) + _, err := Run(cmd) + return err +} + +func harborClient(ip string) (*harborclientv5.RESTClient, error) { + harborConfig := &config.Options{ + Page: 1, + PageSize: 100, + } + http.DefaultTransport.(*http.Transport).TLSClientConfig = &tls.Config{InsecureSkipVerify: true} + return harborclientv5.NewRESTClientForHost(fmt.Sprintf("https://registry.%s.nip.io/api/", ip), "admin", "Harbor12345", harborConfig) +} + +func QueryHarborRepositories(ip, projectName string) (string, error) { + c2, err := harborClient(ip) + if err != nil { + return "", fmt.Errorf("here1 %v", err) + } + ctx := context.Background() + listRepositories, err := c2.ListRepositories(ctx, projectName) + if err != nil { + return "", fmt.Errorf("here2 %v", err) + } + r1, _ := json.Marshal(listRepositories) + return string(r1), nil +} + +func QueryHarborProjectPolicies(ip, projectName string) (string, error) { + c2, err := harborClient(ip) + if err != nil { + return "", err + } + ctx := context.Background() + existingPolicy, err := c2.GetRetentionPolicyByProject(ctx, projectName) + if err != nil { + return "", err + } + r1, _ := json.Marshal(existingPolicy) + return string(r1), nil +} + +func DeleteHarborProject(ip, projectName string) error { + c2, err := harborClient(ip) + if err != nil { + return err + } + ctx := context.Background() + return c2.DeleteProject(ctx, projectName) +} + // Run executes the provided command within this context func Run(cmd *exec.Cmd) ([]byte, error) { dir, _ := GetProjectDir()