chore: update from v2 to v3 and change metrics from rbac proxy to bui…

…ltin

Dockerfile

@@ -10,13 +10,17 @@ COPY go.sum go.sum
 RUN go mod download
 
 # Copy the go source
-COPY main.go main.go
-COPY apis/ apis/
-COPY controllers/ controllers/
-COPY internal/ internal/
+COPY cmd/ cmd/
+COPY api/ api/
+COPY internal/controllers internal/controllers
+COPY internal/harbor internal/harbor
+COPY internal/helpers internal/helpers
+COPY internal/messenger internal/messenger
+COPY internal/metrics internal/metrics
+COPY internal/utilities internal/utilities
 
 # Build
-RUN CGO_ENABLED=0 GOOS=linux GOARCH=${ARCH} GO111MODULE=on go build -a -o manager main.go
+RUN CGO_ENABLED=0 GOOS=linux GOARCH=${ARCH} GO111MODULE=on go build -a -o manager cmd/main.go
 
 # Use distroless as minimal base image to package the manager binary
 # Refer to https://github.com/GoogleContainerTools/distroless for more details

Makefile

@@ -62,12 +62,12 @@ test: manifests generate fmt vet envtest ## Run tests.
 # Build manager binary
 .PHONY: manager
 manager: generate fmt vet
-	go build -o bin/manager main.go
+	go build -o bin/manager cmd/main.go
 
 # Run against the configured Kubernetes cluster in ~/.kube/config
 .PHONY: run
 run: generate fmt vet manifests
-	go run ./main.go --controller-namespace=${CONTROLLER_NAMESPACE}
+	go run ./cmd/main.go --controller-namespace=${CONTROLLER_NAMESPACE}
 
 # Install CRDs into a cluster
 .PHONY: install
@@ -130,7 +130,7 @@ ifeq (, $(shell which controller-gen))
 	CONTROLLER_GEN_TMP_DIR=$$(mktemp -d) ;\
 	cd $$CONTROLLER_GEN_TMP_DIR ;\
 	go mod init tmp ;\
-	go install sigs.k8s.io/controller-tools/cmd/[email protected].2 ;\
+	go install sigs.k8s.io/controller-tools/cmd/[email protected].5 ;\
 	rm -rf $$CONTROLLER_GEN_TMP_DIR ;\
 	}
 CONTROLLER_GEN=$(GOBIN)/controller-gen

PROJECT

@@ -1,17 +1,43 @@
 domain: lagoon.sh
-multigroup: true
+layout:
+- go.kubebuilder.io/v3
+projectName: remote-controller
 repo: github.com/uselagoon/remote-controller
 resources:
-- group: crd
+- api:
+    crdVersion: v1
+    namespaced: true
+  controller: true
+  domain: lagoon.sh
+  group: crd
   kind: LagoonBuild
+  path: remote-controller/api/lagoon/v1beta2
   version: v1beta2
-- group: crd
+- api:
+    crdVersion: v1
+    namespaced: true
+  controller: true
+  domain: lagoon.sh
+  group: crd
   kind: LagoonTask
+  path: remote-controller/api/lagoon/v1beta2
   version: v1beta2
-- group: crd
+- api:
+    crdVersion: v1
+    namespaced: true
+  controller: false
+  domain: lagoon.sh
+  group: crd
   kind: LagoonBuild
+  path: remote-controller/api/lagoon/v1beta1
   version: v1beta1
-- group: crd
+- api:
+    crdVersion: v1
+    namespaced: true
+  controller: false
+  domain: lagoon.sh
+  group: crd
   kind: LagoonTask
+  path: remote-controller/api/lagoon/v1beta1
   version: v1beta1
-version: "2"
+version: "3"

main.go → cmd/main.go

@@ -35,21 +35,21 @@ import (
 
 	"github.com/uselagoon/remote-controller/internal/harbor"
 	"github.com/uselagoon/remote-controller/internal/helpers"
-	"github.com/uselagoon/remote-controller/internal/metrics"
 	"github.com/uselagoon/remote-controller/internal/utilities/deletions"
 	"github.com/uselagoon/remote-controller/internal/utilities/pruner"
 
 	cron "gopkg.in/robfig/cron.v2"
 
+	"sigs.k8s.io/controller-runtime/pkg/metrics/filters"
 	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
 
 	"github.com/hashicorp/golang-lru/v2/expirable"
 	k8upv1 "github.com/k8up-io/k8up/v2/api/v1"
-	lagoonv1beta1 "github.com/uselagoon/remote-controller/apis/lagoon/v1beta1"
-	lagoonv1beta2 "github.com/uselagoon/remote-controller/apis/lagoon/v1beta2"
-	harborctrl "github.com/uselagoon/remote-controller/controllers/harbor"
-	lagoonv1beta1ctrl "github.com/uselagoon/remote-controller/controllers/v1beta1"
-	lagoonv1beta2ctrl "github.com/uselagoon/remote-controller/controllers/v1beta2"
+	lagoonv1beta1 "github.com/uselagoon/remote-controller/api/lagoon/v1beta1"
+	lagoonv1beta2 "github.com/uselagoon/remote-controller/api/lagoon/v1beta2"
+	harborctrl "github.com/uselagoon/remote-controller/internal/controllers/harbor"
+	lagoonv1beta1ctrl "github.com/uselagoon/remote-controller/internal/controllers/v1beta1"
+	lagoonv1beta2ctrl "github.com/uselagoon/remote-controller/internal/controllers/v1beta2"
 	"github.com/uselagoon/remote-controller/internal/messenger"
 	k8upv1alpha1 "github.com/vshn/k8up/api/v1alpha1"
 	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
@@ -184,10 +184,10 @@ func main() {
 
 	var unauthenticatedRegistry string
 
-	flag.StringVar(&metricsAddr, "metrics-addr", ":8080",
-		"The address the metric endpoint binds to.")
-	flag.BoolVar(&secureMetrics, "metrics-secure", false,
-		"If set the metrics endpoint is served securely")
+	flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metrics endpoint binds to. "+
+		"Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.")
+	flag.BoolVar(&secureMetrics, "metrics-secure", true,
+		"If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.")
 	flag.BoolVar(&enableHTTP2, "enable-http2", false,
 		"If set, HTTP/2 will be enabled for the metrics and webhook servers")
 
@@ -210,7 +210,7 @@ func main() {
 		"The number of startup attempts before exiting.")
 	flag.IntVar(&startupConnectionInterval, "startup-connection-interval-seconds", 30,
 		"The duration between startup attempts.")
-	flag.BoolVar(&enableLeaderElection, "enable-leader-election", false,
+	flag.BoolVar(&enableLeaderElection, "leader-elect", false,
 		"Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.")
 	flag.BoolVar(&enableMQ, "enable-message-queue", true,
 		"Enable message queue to provide updates back to Lagoon.")
@@ -487,13 +487,21 @@ func main() {
 	if !enableHTTP2 {
 		tlsOpts = append(tlsOpts, disableHTTP2)
 	}
+	metricsServerOptions := metricsserver.Options{
+		BindAddress:   metricsAddr,
+		SecureServing: secureMetrics,
+		TLSOpts:       tlsOpts,
+	}
+	if secureMetrics {
+		// FilterProvider is used to protect the metrics endpoint with authn/authz.
+		// These configurations ensure that only authorized users and service accounts
+		// can access the metrics endpoint. The RBAC are configured in 'config/rbac/kustomization.yaml'. More info:
+		// https://pkg.go.dev/sigs.k8s.io/[email protected]/pkg/metrics/filters#WithAuthenticationAndAuthorization
+		metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization
+	}
 	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
-		Scheme: scheme,
-		Metrics: metricsserver.Options{
-			BindAddress:   metricsAddr,
-			SecureServing: secureMetrics,
-			TLSOpts:       tlsOpts,
-		},
+		Scheme:           scheme,
+		Metrics:          metricsServerOptions,
 		LeaderElection:   enableLeaderElection,
 		LeaderElectionID: leaderElectionID,
 	})
@@ -1009,10 +1017,6 @@ func main() {
 	}
 	// +kubebuilder:scaffold:builder
 
-	setupLog.Info("starting lagoon metrics server")
-	m := metrics.NewServer(setupLog, ":9912")
-	defer m.Shutdown(context.Background())
-
 	setupLog.Info("starting manager")
 	if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
 		setupLog.Error(err, "problem running manager")

config/default/kustomization.yaml

@@ -49,6 +49,8 @@ resources:
 - ../crd
 - ../rbac
 - ../manager
+# [METRICS] Expose the controller manager metrics service.
+- metrics_service.yaml
 patches:
 - path: envs.yaml
 - path: manager_auth_proxy_patch.yaml

config/default/manager_auth_proxy_patch.yaml

@@ -9,20 +9,10 @@ spec:
   template:
     spec:
       containers:
-      - name: kube-rbac-proxy
-        image: gcr.io/kubebuilder/kube-rbac-proxy:v0.4.1
-        args:
-        - "--secure-listen-address=0.0.0.0:8443"
-        - "--upstream=http://127.0.0.1:8080/"
-        - "--logtostderr=true"
-        - "--v=10"
-        ports:
-        - containerPort: 8443
-          name: https
       - name: manager
         args:
-        - "--metrics-addr=127.0.0.1:8080"
-        - "--enable-leader-election"
+        - "--metrics-bind-address=:8443"
+        - "--leader-elect"
         - "--build-pod-cleanup-cron=*/1 * * * *"
         - "--task-pod-cleanup-cron=*/1 * * * *"
         - "--harbor-credential-cron=*/1 * * * *"
@@ -34,3 +24,6 @@ spec:
         - "--enable-deprecated-apis"
         - "--lagoon-feature-flag-support-k8upv2"
         - "--skip-tls-verify"
+        ports:
+        - containerPort: 8443
+          name: https

config/rbac/auth_proxy_service.yaml → config/default/metrics_service.yaml

@@ -9,6 +9,7 @@ spec:
   ports:
   - name: https
     port: 8443
-    targetPort: https
+    protocol: TCP
+    targetPort: 8443
   selector:
-    control-plane: controller-manager
+    control-plane: controller-manager

config/manager/manager.yaml

@@ -26,11 +26,12 @@ spec:
       - command:
         - /manager
         args:
-        - --enable-leader-election
+        - --leader-elect
         image: controller:latest
         name: manager
         resources:
           requests:
             cpu: 100m
-            memory: 20Mi
+            memory: 400Mi
+      serviceAccountName: controller-manager
       terminationGracePeriodSeconds: 10

config/rbac/kustomization.yaml

@@ -1,11 +1,21 @@
 resources:
+# All RBAC will be applied under this service account in
+# the deployment namespace. You may comment out this resource
+# if your manager will use a service account that exists at
+# runtime. Be sure to update RoleBinding and ClusterRoleBinding
+# subjects if changing service account names.
+- service_account.yaml
 - role.yaml
 - role_binding.yaml
 - leader_election_role.yaml
 - leader_election_role_binding.yaml
-# Comment the following 3 lines if you want to disable
-# the auth proxy (https://github.com/brancz/kube-rbac-proxy)
-# which protects your /metrics endpoint.
-- auth_proxy_service.yaml
-- auth_proxy_role.yaml
-- auth_proxy_role_binding.yaml
+# The following RBAC configurations are used to protect
+# the metrics endpoint with authn/authz. These configurations
+# ensure that only authorized users and service accounts
+# can access the metrics endpoint. Comment the following
+# permissions if you want to disable this protection.
+# More info: https://book.kubebuilder.io/reference/metrics.html
+- metrics_auth_role.yaml
+- metrics_auth_role_binding.yaml
+- metrics_reader_role.yaml
+- metrics_reader_role_binding.yaml

config/rbac/leader_election_role.yaml

@@ -17,16 +17,21 @@ rules:
   - patch
   - delete
 - apiGroups:
-  - ""
+  - coordination.k8s.io
   resources:
-  - configmaps/status
+  - leases
   verbs:
   - get
+  - list
+  - watch
+  - create
   - update
   - patch
+  - delete
 - apiGroups:
   - ""
   resources:
   - events
   verbs:
   - create
+  - patch

config/rbac/leader_election_role_binding.yaml

@@ -8,5 +8,5 @@ roleRef:
   name: leader-election-role
 subjects:
 - kind: ServiceAccount
-  name: default
-  namespace: system
+  name: controller-manager
+  namespace: system

config/rbac/auth_proxy_role.yaml → config/rbac/metrics_auth_role.yaml

@@ -1,7 +1,7 @@
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
-  name: proxy-role
+  name: metrics-auth-role
 rules:
 - apiGroups: ["authentication.k8s.io"]
   resources:
@@ -11,3 +11,15 @@ rules:
   resources:
   - subjectaccessreviews
   verbs: ["create"]
+- apiGroups:
+  - authentication.k8s.io
+  resources:
+  - tokenreviews
+  verbs:
+  - create
+- apiGroups:
+  - authorization.k8s.io
+  resources:
+  - subjectaccessreviews
+  verbs:
+  - create

config/rbac/metrics_auth_role_binding.yaml

@@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: metrics-auth-rolebinding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: metrics-auth-role
+subjects:
+- kind: ServiceAccount
+  name: controller-manager
+  namespace: system

config/rbac/metrics_reader_role.yaml

@@ -0,0 +1,9 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: metrics-reader
+rules:
+- nonResourceURLs:
+  - "/metrics"
+  verbs:
+  - get

config/rbac/auth_proxy_role_binding.yaml → config/rbac/metrics_reader_role_binding.yaml

@@ -1,12 +1,12 @@
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
-  name: proxy-rolebinding
+  name: metrics-reader-rolebinding
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: ClusterRole
-  name: proxy-role
+  name: metrics-reader
 subjects:
 - kind: ServiceAccount
-  name: default
-  namespace: system
+  name: controller-manager
+  namespace: system

config/rbac/role_binding.yaml

@@ -8,5 +8,5 @@ roleRef:
   name: manager-role
 subjects:
 - kind: ServiceAccount
-  name: default
+  name: controller-manager
   namespace: system