CNF-11234: Enable RTE metrics to be scraped securely by Prometheus

bundle/manifests/numaresources-controller-manager-metrics-service_v1_service.yaml

@@ -0,0 +1,19 @@
+apiVersion: v1
+kind: Service
+metadata:
+  annotations:
+    service.beta.openshift.io/serving-cert-secret-name: secret-kube-rbac-proxy-tls
+  creationTimestamp: null
+  labels:
+    control-plane: controller-manager
+  name: numaresources-controller-manager-metrics-service
+spec:
+  ports:
+  - name: https
+    port: 8443
+    protocol: TCP
+    targetPort: https
+  selector:
+    control-plane: controller-manager
+status:
+  loadBalancer: {}

bundle/manifests/numaresources-controller-manager_monitoring.coreos.com_v1_servicemonitor.yaml

@@ -0,0 +1,18 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: numaresources-controller-manager
+spec:
+  endpoints:
+  - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
+    interval: 30s
+    path: /metrics
+    scheme: https
+    targetPort: 8443
+    tlsConfig:
+      caFile: /etc/prometheus/configmaps/serving-certs-ca-bundle/service-ca.crt
+      insecureSkipVerify: false
+      serverName: numaresources-controller-manager-metrics-service.numaresources.svc
+  selector:
+    matchLabels:
+      control-plane: controller-manager

bundle/manifests/numaresources-metrics-reader_rbac.authorization.k8s.io_v1_clusterrole.yaml

@@ -0,0 +1,10 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  creationTimestamp: null
+  name: numaresources-metrics-reader
+rules:
+- nonResourceURLs:
+  - /metrics
+  verbs:
+  - get

bundle/manifests/numaresources-operator.clusterserviceversion.yaml

@@ -62,7 +62,7 @@ metadata:
         }
       ]
     capabilities: Basic Install
-    createdAt: "2024-09-25T14:08:22Z"
+    createdAt: "2024-10-06T18:15:59Z"
     olm.skipRange: '>=4.17.0 <4.18.0'
     operators.operatorframework.io/builder: operator-sdk-v1.36.1
     operators.operatorframework.io/project_layout: go.kubebuilder.io/v3
@@ -343,6 +343,7 @@ spec:
           resources:
           - configmaps
           - serviceaccounts
+          - services
           verbs:
           - '*'
         - apiGroups:
@@ -465,6 +466,18 @@ spec:
           - get
           - list
           - update
+        - apiGroups:
+          - authentication.k8s.io
+          resources:
+          - tokenreviews
+          verbs:
+          - create
+        - apiGroups:
+          - authorization.k8s.io
+          resources:
+          - subjectaccessreviews
+          verbs:
+          - create
         serviceAccountName: numaresources-controller-manager
       deployments:
       - label:
@@ -528,6 +541,35 @@ spec:
                     memory: 20Mi
                 securityContext:
                   allowPrivilegeEscalation: false
+              - args:
+                - --secure-listen-address=0.0.0.0:8443
+                - --upstream=http://127.0.0.1:8080/
+                - --config-file=/etc/kube-rbac-proxy/config.yaml
+                - --tls-cert-file=/etc/tls/private/tls.crt
+                - --tls-private-key-file=/etc/tls/private/tls.key
+                - --allow-paths=/metrics
+                - --logtostderr=true
+                - -v=10
+                image: gcr.io/kubebuilder/kube-rbac-proxy:v0.15.0
+                name: kube-rbac-proxy
+                ports:
+                - containerPort: 8443
+                  name: https
+                  protocol: TCP
+                resources: {}
+                securityContext:
+                  allowPrivilegeEscalation: false
+                  capabilities:
+                    drop:
+                    - ALL
+                terminationMessagePolicy: FallbackToLogsOnError
+                volumeMounts:
+                - mountPath: /etc/kube-rbac-proxy
+                  name: secret-kube-rbac-proxy-metric
+                  readOnly: true
+                - mountPath: /etc/tls/private
+                  name: secret-kube-rbac-proxy-tls
+                  readOnly: true
               securityContext:
                 runAsNonRoot: true
               serviceAccountName: numaresources-controller-manager
@@ -537,6 +579,13 @@ spec:
                 key: node-role.kubernetes.io/control-plane
               - effect: NoSchedule
                 key: node-role.kubernetes.io/master
+              volumes:
+              - name: secret-kube-rbac-proxy-tls
+                secret:
+                  secretName: secret-kube-rbac-proxy-tls
+              - name: secret-kube-rbac-proxy-metric
+                secret:
+                  secretName: numaresources-secret-kube-rbac-proxy-metric
       permissions:
       - rules:
         - apiGroups:

bundle/manifests/numaresources-prometheus-k8s_rbac.authorization.k8s.io_v1_role.yaml

@@ -0,0 +1,16 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  creationTimestamp: null
+  name: numaresources-prometheus-k8s
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - services
+  - endpoints
+  - pods
+  verbs:
+  - get
+  - list
+  - watch

bundle/manifests/numaresources-prometheus-k8s_rbac.authorization.k8s.io_v1_rolebinding.yaml

@@ -0,0 +1,13 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  creationTimestamp: null
+  name: numaresources-prometheus-k8s
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: numaresources-prometheus-k8s
+subjects:
+- kind: ServiceAccount
+  name: prometheus-k8s
+  namespace: openshift-monitoring

bundle/manifests/numaresources-secret-kube-rbac-proxy-metric_v1_secret.yaml

@@ -0,0 +1,9 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: numaresources-secret-kube-rbac-proxy-metric
+stringData:
+  config.yaml: "\"authorization\":\n  \"static\":\n  - \"path\": \"/metrics\"\n    \"resourceRequest\":
+    false\n    \"user\":\n      \"name\": \"system:serviceaccount:openshift-monitoring:prometheus-k8s\"\n
+    \   \"verb\": \"get\"    "
+type: Opaque

config/default/kustomization.yaml

@@ -22,13 +22,13 @@ resources:
 # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required.
 #- ../certmanager
 # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'.
-#- ../prometheus
+- ../prometheus
 
 patchesStrategicMerge:
 # Protect the /metrics endpoint by putting it behind auth.
 # If you want your controller-manager to expose the /metrics
 # endpoint w/o any authn/z, please comment the following line.
-#- manager_auth_proxy_patch.yaml
+- manager_auth_proxy_patch.yaml
 
 # Mount the controller config file for loading manager configurations
 # through a ComponentConfig type

config/default/manager_auth_proxy_patch.yaml

@@ -9,20 +9,41 @@ spec:
   template:
     spec:
       containers:
+      - name: manager
       - name: kube-rbac-proxy
-        image: gcr.io/kubebuilder/kube-rbac-proxy:v0.8.0
+        image: gcr.io/kubebuilder/kube-rbac-proxy:v0.15.0
         args:
         - "--secure-listen-address=0.0.0.0:8443"
         - "--upstream=http://127.0.0.1:8080/"
+        - "--config-file=/etc/kube-rbac-proxy/config.yaml"
+        - "--tls-cert-file=/etc/tls/private/tls.crt"
+        - "--tls-private-key-file=/etc/tls/private/tls.key"
+        - "--allow-paths=/metrics"
         - "--logtostderr=true"
         - "-v=10"
         ports:
         - containerPort: 8443
           protocol: TCP
           name: https
-      - name: manager
-        args:
-        - "--platform=kubernetes"
-        - "--health-probe-bind-address=:8081"
-        - "--metrics-bind-address=127.0.0.1:8080"
-        - "--leader-elect"
+        securityContext:
+          allowPrivilegeEscalation: false
+          capabilities:
+            drop:
+            - ALL
+        terminationMessagePolicy: FallbackToLogsOnError
+        volumeMounts:
+        - mountPath: /etc/kube-rbac-proxy
+          name: secret-kube-rbac-proxy-metric
+          readOnly: true
+        - mountPath: /etc/tls/private
+          name: secret-kube-rbac-proxy-tls
+          readOnly: true
+      serviceAccountName: controller-manager
+      terminationGracePeriodSeconds: 10
+      volumes:
+      - name: secret-kube-rbac-proxy-tls
+        secret:
+          secretName: secret-kube-rbac-proxy-tls
+      - name: secret-kube-rbac-proxy-metric
+        secret:
+          secretName: secret-kube-rbac-proxy-metric

config/manager/manager.yaml

@@ -5,6 +5,7 @@ metadata:
     workload.openshift.io/allowed: management
   labels:
     control-plane: controller-manager
+    openshift.io/cluster-monitoring: "true"
   name: system
 ---
 apiVersion: apps/v1

config/prometheus/kustomization.yaml

@@ -1,2 +1,5 @@
 resources:
-- monitor.yaml
+- rbac.yaml
+- secret-kube-rbac-proxy.yaml
+# Please uncomment monitor.yaml to enable prometheus pods to scrape the metrics periodically.
+# - monitor.yaml

config/prometheus/monitor.yaml

@@ -1,20 +1,24 @@
 
-# Prometheus Monitor Service (Metrics)
 apiVersion: monitoring.coreos.com/v1
 kind: ServiceMonitor
 metadata:
-  labels:
-    control-plane: controller-manager
-  name: controller-manager-metrics-monitor
+  name: controller-manager
   namespace: system
 spec:
   endpoints:
-    - path: /metrics
-      port: https
-      scheme: https
-      bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
-      tlsConfig:
-        insecureSkipVerify: true
+  - interval: 30s
+    # Matches the name of the service's port.
+    targetPort: 8443
+    path: /metrics
+    scheme: https
+    bearerTokenFile: "/var/run/secrets/kubernetes.io/serviceaccount/token"
+    tlsConfig:
+      # The CA file used by Prometheus to verify the server's certificate.
+      # It's the cluster's CA bundle from the service CA operator.
+      caFile: /etc/prometheus/configmaps/serving-certs-ca-bundle/service-ca.crt
+      # The name of the server (CN) in the server's certificate.
+      serverName: numaresources-controller-manager-metrics-service.numaresources.svc
+      insecureSkipVerify: false
   selector:
     matchLabels:
       control-plane: controller-manager

config/prometheus/rbac.yaml

@@ -0,0 +1,31 @@
+# creates Role and RoleBinding for prometheus-k8s service account to access our namespace
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: prometheus-k8s
+  namespace: system
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - services
+  - endpoints
+  - pods
+  verbs:
+  - get
+  - list
+  - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: prometheus-k8s
+  namespace: system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: prometheus-k8s
+subjects:
+- kind: ServiceAccount
+  name: prometheus-k8s
+  namespace: openshift-monitoring

config/prometheus/secret-kube-rbac-proxy.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: secret-kube-rbac-proxy-metric
+  namespace: system
+stringData:
+  config.yaml: |-
+    "authorization":
+      "static":
+      - "path": "/metrics"
+        "resourceRequest": false
+        "user":
+          "name": "system:serviceaccount:openshift-monitoring:prometheus-k8s"
+        "verb": "get"    
+type: Opaque

config/rbac/auth_proxy_service.yaml

@@ -1,6 +1,8 @@
 apiVersion: v1
 kind: Service
 metadata:
+  annotations:
+    service.beta.openshift.io/serving-cert-secret-name: secret-kube-rbac-proxy-tls
   labels:
     control-plane: controller-manager
   name: controller-manager-metrics-service

config/rbac/kustomization.yaml

@@ -12,7 +12,7 @@ resources:
 # Comment the following 4 lines if you want to disable
 # the auth proxy (https://github.com/brancz/kube-rbac-proxy)
 # which protects your /metrics endpoint.
-#- auth_proxy_service.yaml
-#- auth_proxy_role.yaml
-#- auth_proxy_role_binding.yaml
-#- auth_proxy_client_clusterrole.yaml
+- auth_proxy_service.yaml
+- auth_proxy_role.yaml
+- auth_proxy_role_binding.yaml
+- auth_proxy_client_clusterrole.yaml

config/rbac/role.yaml

@@ -9,6 +9,7 @@ rules:
   resources:
   - configmaps
   - serviceaccounts
+  - services
   verbs:
   - '*'
 - apiGroups: