From 09830bec39b0cdc280755368addcf47a24aa0a60 Mon Sep 17 00:00:00 2001 From: Maxim Fedotov Date: Thu, 5 Dec 2024 10:16:53 +0300 Subject: [PATCH] feat(core): add kube-rbac-proxy to kubevirt & cdi (#532) add kube-rbac proxy to virt-handler, virt-api, virt-controller, cdi-deployment and cdi-operator --------- Signed-off-by: Maksim Fedotov --- .../patches/018-cover-cdi-metrics.patch | 76 +++++ images/cdi-artifact/patches/README.md | 8 + .../patches/024-cover-kubevirt-metrics.patch | 279 ++++++++++++++++++ images/virt-artifact/patches/README.md | 10 + templates/cdi/cdi-deployment/rbac-for-us.yaml | 14 + templates/cdi/cdi-operator/deployment.yaml | 3 + templates/cdi/cdi-operator/rbac-for-us.yaml | 14 + templates/cdi/config.yaml | 12 + templates/kubevirt/kubevirt.yaml | 25 +- .../kubevirt/virt-controller/rbac-for-us.yaml | 14 + .../kubevirt/virt-handler/rbac-for-us.yaml | 14 + templates/rbac-to-us.yaml | 2 +- 12 files changed, 468 insertions(+), 3 deletions(-) create mode 100644 images/cdi-artifact/patches/018-cover-cdi-metrics.patch create mode 100644 images/virt-artifact/patches/024-cover-kubevirt-metrics.patch create mode 100644 templates/cdi/cdi-deployment/rbac-for-us.yaml create mode 100644 templates/kubevirt/virt-controller/rbac-for-us.yaml create mode 100644 templates/kubevirt/virt-handler/rbac-for-us.yaml diff --git a/images/cdi-artifact/patches/018-cover-cdi-metrics.patch b/images/cdi-artifact/patches/018-cover-cdi-metrics.patch new file mode 100644 index 000000000..9e66c0dab --- /dev/null +++ b/images/cdi-artifact/patches/018-cover-cdi-metrics.patch @@ -0,0 +1,76 @@ +diff --git a/cmd/cdi-controller/controller.go b/cmd/cdi-controller/controller.go +index a8fdfb6b5..74fdd5210 100644 +--- a/cmd/cdi-controller/controller.go ++++ b/cmd/cdi-controller/controller.go +@@ -36,6 +36,7 @@ import ( + "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/manager/signals" ++ metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + + cdiv1 "kubevirt.io/containerized-data-importer-api/pkg/apis/core/v1beta1" + forklift "kubevirt.io/containerized-data-importer-api/pkg/apis/forklift/v1beta1" +@@ -58,6 +59,7 @@ const ( + var ( + kubeconfig string + kubeURL string ++ metricsBindAddress string + importerImage string + clonerImage string + uploadServerImage string +@@ -98,6 +100,7 @@ type ControllerEnvs struct { + func init() { + // flags + flag.StringVar(&kubeURL, "server", "", "(Optional) URL address of a remote api server. Do not set for local clusters.") ++ flag.StringVar(&metricsBindAddress, "metrics_address", ":8080", "(Optional) URL address of a metrics server.") + klog.InitFlags(nil) + flag.Parse() + +@@ -199,6 +202,9 @@ func start() { + LeaderElectionResourceLock: "leases", + Cache: getCacheOptions(apiClient, namespace), + Scheme: scheme, ++ Metrics: metricsserver.Options{ ++ BindAddress: metricsBindAddress, ++ }, + } + + cfg = config.GetConfigOrDie() +diff --git a/cmd/cdi-operator/operator.go b/cmd/cdi-operator/operator.go +index 90049f649..a9eab1454 100644 +--- a/cmd/cdi-operator/operator.go ++++ b/cmd/cdi-operator/operator.go +@@ -38,6 +38,7 @@ import ( + "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/manager/signals" ++ metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + + cdiv1 "kubevirt.io/containerized-data-importer-api/pkg/apis/core/v1beta1" + "kubevirt.io/containerized-data-importer/pkg/operator/controller" +@@ -45,6 +46,7 @@ import ( + ) + + var log = logf.Log.WithName("cmd") ++var metricsBindAddress string + + func printVersion() { + log.Info(fmt.Sprintf("Go Version: %s", runtime.Version())) +@@ -52,6 +54,7 @@ func printVersion() { + } + + func main() { ++ flag.StringVar(&metricsBindAddress, "metrics_address", ":8080", "(Optional) URL address of a metrics server.") + flag.Parse() + + defVerbose := fmt.Sprintf("%d", 1) // note flag values are strings +@@ -97,6 +100,9 @@ func main() { + LeaderElectionNamespace: namespace, + LeaderElectionID: "cdi-operator-leader-election-helper", + LeaderElectionResourceLock: "leases", ++ Metrics: metricsserver.Options{ ++ BindAddress: metricsBindAddress, ++ }, + } + + // Create a new Manager to provide shared dependencies and start components diff --git a/images/cdi-artifact/patches/README.md b/images/cdi-artifact/patches/README.md index 0c6b3e70e..2a115ccb4 100644 --- a/images/cdi-artifact/patches/README.md +++ b/images/cdi-artifact/patches/README.md @@ -68,3 +68,11 @@ When cloning via DataVolume from PVC to PVC, there was no format conversion. The target PVC should have a raw type for volume mode Block and a qcow2 type for Filesystem. The patch adds this conversion. +#### `018-cover-cdi-metrics.patch` + +Configure cdi's components metrics web servers to listen on localhost. +This is necessary for ensuring that the metrics can be accessed only by Prometheus via kube-rbac-proxy sidecar. + +Currently covered metrics: +- cdi-controller +- cdi-deployment diff --git a/images/virt-artifact/patches/024-cover-kubevirt-metrics.patch b/images/virt-artifact/patches/024-cover-kubevirt-metrics.patch new file mode 100644 index 000000000..586cd4f50 --- /dev/null +++ b/images/virt-artifact/patches/024-cover-kubevirt-metrics.patch @@ -0,0 +1,279 @@ +diff --git a/cmd/virt-handler/virt-handler.go b/cmd/virt-handler/virt-handler.go +index 09b2b53904..aaeafc76f3 100644 +--- a/cmd/virt-handler/virt-handler.go ++++ b/cmd/virt-handler/virt-handler.go +@@ -93,10 +93,12 @@ const ( + defaultWatchdogTimeout = 30 * time.Second + + // Default port that virt-handler listens on. +- defaultPort = 8185 ++ defaultPort = 8185 ++ defaultMetricsPort = 8080 + + // Default address that virt-handler listens on. +- defaultHost = "0.0.0.0" ++ defaultHost = "0.0.0.0" ++ defaultMetricsHost = defaultHost + + hostOverride = "" + +@@ -366,6 +368,8 @@ func (app *virtHandlerApp) Run() { + + promErrCh := make(chan error) + go app.runPrometheusServer(promErrCh) ++ healErrCh := make(chan error) ++ go app.runHealthzServer(healErrCh) + + lifecycleHandler := rest.NewLifecycleHandler( + recorder, +@@ -535,7 +539,6 @@ func (app *virtHandlerApp) runPrometheusServer(errCh chan error) { + mux := restful.NewContainer() + webService := new(restful.WebService) + webService.Path("/").Consumes(restful.MIME_JSON).Produces(restful.MIME_JSON) +- webService.Route(webService.GET("/healthz").To(healthz.KubeConnectionHealthzFuncFactory(app.clusterConfig, apiHealthVersion)).Doc("Health endpoint")) + + componentProfiler := profiler.NewProfileManager(app.clusterConfig) + webService.Route(webService.GET("/start-profiler").To(componentProfiler.HandleStartProfiler).Doc("start profiler endpoint")) +@@ -546,14 +549,23 @@ func (app *virtHandlerApp) runPrometheusServer(errCh chan error) { + log.Log.V(1).Infof("metrics: max concurrent requests=%d", app.MaxRequestsInFlight) + mux.Handle("/metrics", metricshandler.Handler(app.MaxRequestsInFlight)) + server := http.Server{ +- Addr: app.ServiceListen.Address(), ++ Addr: app.ServiceListen.MetricsAddress(), + Handler: mux, +- TLSConfig: app.promTLSConfig, +- // Disable HTTP/2 +- // See CVE-2023-44487 +- TLSNextProto: map[string]func(*http.Server, *tls.Conn, http.Handler){}, + } +- errCh <- server.ListenAndServeTLS("", "") ++ errCh <- server.ListenAndServe() ++} ++ ++func (app *virtHandlerApp) runHealthzServer(errCh chan error) { ++ mux := restful.NewContainer() ++ webService := new(restful.WebService) ++ webService.Path("/").Consumes(restful.MIME_JSON).Produces(restful.MIME_JSON) ++ webService.Route(webService.GET("/healthz").To(healthz.KubeConnectionHealthzFuncFactory(app.clusterConfig, apiHealthVersion)).Doc("Health endpoint")) ++ mux.Add(webService) ++ server := http.Server{ ++ Addr: app.ServiceListen.Address(), ++ Handler: mux, ++ } ++ errCh <- server.ListenAndServe() + } + + func (app *virtHandlerApp) runServer(errCh chan error, consoleHandler *rest.ConsoleHandler, lifecycleHandler *rest.LifecycleHandler) { +@@ -588,7 +600,9 @@ func (app *virtHandlerApp) AddFlags() { + app.InitFlags() + + app.BindAddress = defaultHost ++ app.MetricsBindAddress = defaultMetricsHost + app.Port = defaultPort ++ app.MetricsPort = defaultMetricsPort + + app.AddCommonFlags() + +diff --git a/pkg/service/service.go b/pkg/service/service.go +index 0368a0762b..473328fb4b 100644 +--- a/pkg/service/service.go ++++ b/pkg/service/service.go +@@ -42,6 +42,8 @@ type ServiceListen struct { + Name string + BindAddress string + Port int ++ MetricsBindAddress string ++ MetricsPort int + } + + type ServiceLibvirt struct { +@@ -52,6 +54,10 @@ func (service *ServiceListen) Address() string { + return fmt.Sprintf("%s:%s", service.BindAddress, strconv.Itoa(service.Port)) + } + ++func (service *ServiceListen) MetricsAddress() string { ++ return fmt.Sprintf("%s:%s", service.MetricsBindAddress, strconv.Itoa(service.MetricsPort)) ++} ++ + func (service *ServiceListen) InitFlags() { + flag.CommandLine.AddGoFlag(goflag.CommandLine.Lookup("v")) + flag.CommandLine.AddGoFlag(goflag.CommandLine.Lookup("kubeconfig")) +@@ -61,6 +67,9 @@ func (service *ServiceListen) InitFlags() { + func (service *ServiceListen) AddCommonFlags() { + flag.StringVar(&service.BindAddress, "listen", service.BindAddress, "Address where to listen on") + flag.IntVar(&service.Port, "port", service.Port, "Port to listen on") ++ // default values are taken from the common server counterparts ++ flag.StringVar(&service.MetricsBindAddress, "metrics-listen", service.MetricsBindAddress, "Address for metrics to listen on") ++ flag.IntVar(&service.MetricsPort, "metrics-port", service.MetricsPort, "Port for metrics to listen on") + } + + func (service *ServiceLibvirt) AddLibvirtFlags() { +diff --git a/pkg/virt-api/api.go b/pkg/virt-api/api.go +index a7ea3f44d6..6d9ccd800f 100644 +--- a/pkg/virt-api/api.go ++++ b/pkg/virt-api/api.go +@@ -82,9 +82,11 @@ import ( + const ( + // Default port that virt-api listens on. + defaultPort = 443 ++ defaultMetricsPort = 8080 + + // Default address that virt-api listens on. + defaultHost = "0.0.0.0" ++ defaultMetricsHost = defaultHost + + DefaultConsoleServerPort = 8186 + +@@ -156,6 +158,8 @@ func NewVirtApi() VirtApi { + app := &virtAPIApp{} + app.BindAddress = defaultHost + app.Port = defaultPort ++ app.MetricsBindAddress = defaultMetricsHost ++ app.MetricsPort = defaultMetricsPort + + return app + } +@@ -968,6 +972,19 @@ func (app *virtAPIApp) setupTLS(k8sCAManager, kubevirtCAManager, virtualizationC + app.handlerTLSConfiguration = kvtls.SetupTLSForVirtHandlerClients(kubevirtCAManager, app.handlerCertManager, app.externallyManaged) + } + ++func (app *virtAPIApp) startPrometheusServer(errCh chan error) { ++ mux := restful.NewContainer() ++ webService := new(restful.WebService) ++ webService.Path("/").Consumes(restful.MIME_JSON).Produces(restful.MIME_JSON) ++ mux.Add(webService) ++ mux.Handle("/metrics", promhttp.Handler()) ++ server := http.Server{ ++ Addr: app.ServiceListen.MetricsAddress(), ++ Handler: mux, ++ } ++ errCh <- server.ListenAndServe() ++ ++} + func (app *virtAPIApp) startTLS(informerFactory controller.KubeInformerFactory) error { + + errors := make(chan error) +@@ -991,7 +1008,6 @@ func (app *virtAPIApp) startTLS(informerFactory controller.KubeInformerFactory) + + app.Compose() + +- http.Handle("/metrics", promhttp.Handler()) + server := &http.Server{ + Addr: fmt.Sprintf("%s:%d", app.BindAddress, app.Port), + TLSConfig: app.tlsConfig, +@@ -1127,6 +1143,8 @@ func (app *virtAPIApp) Run() { + go app.certmanager.Start() + go app.handlerCertManager.Start() + ++ promErrCh := make(chan error) ++ go app.startPrometheusServer(promErrCh) + // start TLS server + // tls server will only accept connections when fetching a certificate and internal configuration passed once + err = app.startTLS(kubeInformerFactory) +diff --git a/pkg/virt-controller/watch/application.go b/pkg/virt-controller/watch/application.go +index f80a0653ad..4a8d20d7be 100644 +--- a/pkg/virt-controller/watch/application.go ++++ b/pkg/virt-controller/watch/application.go +@@ -92,8 +92,10 @@ import ( + + const ( + defaultPort = 8182 ++ defaultMetricsPort = 8080 + + defaultHost = "0.0.0.0" ++ defaultMetricsHost = defaultHost + + launcherImage = "virt-launcher" + exporterImage = "virt-exportserver" +@@ -492,10 +494,12 @@ func (vca *VirtControllerApp) Run() { + go promCertManager.Start() + promTLSConfig := kvtls.SetupPromTLS(promCertManager, vca.clusterConfig) + ++ promErrCh := make(chan error) ++ go vca.startPrometheusServer(promErrCh) ++ + go func() { + httpLogger := logger.With("service", "http") + _ = httpLogger.Level(log.INFO).Log("action", "listening", "interface", vca.BindAddress, "port", vca.Port) +- http.Handle("/metrics", promhttp.Handler()) + server := http.Server{ + Addr: vca.Address(), + Handler: http.DefaultServeMux, +@@ -519,6 +523,20 @@ func (vca *VirtControllerApp) Run() { + panic("unreachable") + } + ++func (app *VirtControllerApp) startPrometheusServer(errCh chan error) { ++ mux := restful.NewContainer() ++ webService := new(restful.WebService) ++ webService.Path("/").Consumes(restful.MIME_JSON).Produces(restful.MIME_JSON) ++ mux.Add(webService) ++ mux.Handle("/metrics", promhttp.Handler()) ++ server := http.Server{ ++ Addr: app.ServiceListen.MetricsAddress(), ++ Handler: mux, ++ } ++ errCh <- server.ListenAndServe() ++ ++} ++ + func (vca *VirtControllerApp) onStartedLeading() func(ctx context.Context) { + return func(ctx context.Context) { + stop := ctx.Done() +@@ -896,6 +914,8 @@ func (vca *VirtControllerApp) AddFlags() { + + vca.BindAddress = defaultHost + vca.Port = defaultPort ++ vca.MetricsBindAddress = defaultMetricsHost ++ vca.MetricsPort = defaultMetricsPort + + vca.AddCommonFlags() + +diff --git a/pkg/virt-operator/resource/generate/components/daemonsets.go b/pkg/virt-operator/resource/generate/components/daemonsets.go +index da6e00c783..8fa14e93b5 100644 +--- a/pkg/virt-operator/resource/generate/components/daemonsets.go ++++ b/pkg/virt-operator/resource/generate/components/daemonsets.go +@@ -174,7 +174,7 @@ func NewHandlerDaemonSet(namespace, repository, imagePrefix, version, launcherVe + } + container.Args = []string{ + "--port", +- "8443", ++ "8090", + "--hostname-override", + "$(NODE_NAME)", + "--pod-ip-address", +@@ -192,7 +192,7 @@ func NewHandlerDaemonSet(namespace, repository, imagePrefix, version, launcherVe + { + Name: "metrics", + Protocol: corev1.ProtocolTCP, +- ContainerPort: 8443, ++ ContainerPort: 8080, + }, + } + container.SecurityContext = &corev1.SecurityContext{ +@@ -226,10 +226,10 @@ func NewHandlerDaemonSet(namespace, repository, imagePrefix, version, launcherVe + FailureThreshold: 3, + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ +- Scheme: corev1.URISchemeHTTPS, ++ Scheme: corev1.URISchemeHTTP, + Port: intstr.IntOrString{ + Type: intstr.Int, +- IntVal: 8443, ++ IntVal: 8090, + }, + Path: "/healthz", + }, +@@ -241,10 +241,10 @@ func NewHandlerDaemonSet(namespace, repository, imagePrefix, version, launcherVe + container.ReadinessProbe = &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ +- Scheme: corev1.URISchemeHTTPS, ++ Scheme: corev1.URISchemeHTTP, + Port: intstr.IntOrString{ + Type: intstr.Int, +- IntVal: 8443, ++ IntVal: 8090, + }, + Path: "/healthz", + }, diff --git a/images/virt-artifact/patches/README.md b/images/virt-artifact/patches/README.md index 4ae99e60c..377dfb135 100644 --- a/images/virt-artifact/patches/README.md +++ b/images/virt-artifact/patches/README.md @@ -83,3 +83,13 @@ Unsuccessful migrations may leave a lot of Pods. These huge lists reduce perform Replace the expressions for the ValidatingAdmissionPolicy kubevirt-node-restriction-policy. This is necessary because of the kube-api-rewriter that changes the labels. + +#### `024-cover-kubevirt-metrics.patch` + +Configure kubevirt's components metrics web servers to listen on localhost. +This is necessary for ensuring that the metrics can be accessed only by Prometheus via kube-rbac-proxy sidecar. + +Currently covered metrics: +- virt-handler +- virt-controller +- virt-api diff --git a/templates/cdi/cdi-deployment/rbac-for-us.yaml b/templates/cdi/cdi-deployment/rbac-for-us.yaml new file mode 100644 index 000000000..18fcb00d1 --- /dev/null +++ b/templates/cdi/cdi-deployment/rbac-for-us.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: d8:{{ .Chart.Name }}:cdi-deployment-rbac-proxy + {{- include "helm_lib_module_labels" (list . (dict "app" "cdi-deployment")) | nindent 2 }} +subjects: + - kind: ServiceAccount + name: cdi-sa + namespace: d8-{{ .Chart.Name }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: d8:rbac-proxy diff --git a/templates/cdi/cdi-operator/deployment.yaml b/templates/cdi/cdi-operator/deployment.yaml index 007c39793..3064d956b 100644 --- a/templates/cdi/cdi-operator/deployment.yaml +++ b/templates/cdi/cdi-operator/deployment.yaml @@ -102,6 +102,9 @@ spec: - name: PULL_POLICY value: IfNotPresent - name: MONITORING_NAMESPACE + args: + - -metrics_address + - 127.0.0.1:8080 image: {{ include "helm_lib_module_image" (list . "cdiOperator") }} imagePullPolicy: IfNotPresent ports: diff --git a/templates/cdi/cdi-operator/rbac-for-us.yaml b/templates/cdi/cdi-operator/rbac-for-us.yaml index 8e4cdfd8f..4040dbc0f 100644 --- a/templates/cdi/cdi-operator/rbac-for-us.yaml +++ b/templates/cdi/cdi-operator/rbac-for-us.yaml @@ -572,3 +572,17 @@ subjects: - kind: ServiceAccount name: cdi-operator namespace: d8-{{ .Chart.Name }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: d8:{{ .Chart.Name }}:cdi-operator-rbac-proxy + {{- include "helm_lib_module_labels" (list . (dict "app" "cdi-operator")) | nindent 2 }} +subjects: + - kind: ServiceAccount + name: cdi-operator + namespace: d8-{{ .Chart.Name }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: d8:rbac-proxy diff --git a/templates/cdi/config.yaml b/templates/cdi/config.yaml index 5c847266e..2ddd168ed 100644 --- a/templates/cdi/config.yaml +++ b/templates/cdi/config.yaml @@ -34,6 +34,9 @@ spec: - HonorWaitForFirstConsumer {{- end }} customizeComponents: + flags: + controller: + metrics_address: 127.0.0.1:8080 patches: - resourceType: Deployment resourceName: cdi-apiserver @@ -93,16 +96,25 @@ spec: {{- $_ := set $kubeRbacProxySettings "runAsUserNobody" true }} {{- $_ := set $kubeRbacProxySettings "upstreams" (list (dict "upstream" "http://127.0.0.1:9090/metrics" "path" "/proxy/metrics" "name" "kube-api-rewriter") + (dict "upstream" "http://127.0.0.1:8080/metrics" "path" "/metrics" "name" "cdi-operator") ) }} - resourceType: Deployment resourceName: cdi-apiserver patch: {{ include "kube_rbac_proxy.pod_spec_strategic_patch_json" (tuple . $kubeRbacProxySettings) }} type: strategic + {{- $_ := set $kubeRbacProxySettings "upstreams" (list + (dict "upstream" "http://127.0.0.1:9090/metrics" "path" "/proxy/metrics" "name" "kube-api-rewriter") + (dict "upstream" "http://127.0.0.1:8080/metrics" "path" "/metrics" "name" "cdi-deployment") + ) }} - resourceType: Deployment resourceName: cdi-deployment patch: {{ include "kube_rbac_proxy.pod_spec_strategic_patch_json" (tuple . $kubeRbacProxySettings) }} type: strategic + - resourceType: Service + resourceName: cdi-prometheus-metrics + patch: '[{"op": "replace", "path": "/spec/ports/0/targetPort", "value": "https-metrics"}]' + type: json # Add rewriter proxy container port to the Service used by webhook configurations. # First need to set name for existing port to make strategic patch works later. diff --git a/templates/kubevirt/kubevirt.yaml b/templates/kubevirt/kubevirt.yaml index e0a60c2cd..228446089 100644 --- a/templates/kubevirt/kubevirt.yaml +++ b/templates/kubevirt/kubevirt.yaml @@ -42,8 +42,15 @@ spec: disableSerialConsoleLog: {} customizeComponents: flags: - api: {} - controller: {} + api: + metrics-listen: 127.0.0.1 + metrics-port: "8080" + controller: + metrics-listen: 127.0.0.1 + metrics-port: "8080" + handler: + metrics-listen: 127.0.0.1 + metrics-port: "8080" patches: - resourceType: Deployment resourceName: virt-api @@ -103,17 +110,26 @@ spec: {{- $_ := set $kubeRbacProxySettings "runAsUserNobody" true }} {{- $_ := set $kubeRbacProxySettings "upstreams" (list (dict "upstream" "http://127.0.0.1:9090/metrics" "path" "/proxy/metrics" "name" "kube-api-rewriter") + (dict "upstream" "http://127.0.0.1:8080/metrics" "path" "/metrics" "name" "virt-controller") ) }} - resourceName: virt-controller resourceType: Deployment patch: {{ include "kube_rbac_proxy.pod_spec_strategic_patch_json" (tuple . $kubeRbacProxySettings) }} type: strategic + {{- $_ := set $kubeRbacProxySettings "upstreams" (list + (dict "upstream" "http://127.0.0.1:9090/metrics" "path" "/proxy/metrics" "name" "kube-api-rewriter") + (dict "upstream" "http://127.0.0.1:8080/metrics" "path" "/metrics" "name" "virt-api") + ) }} - resourceName: virt-api resourceType: Deployment patch: {{ include "kube_rbac_proxy.pod_spec_strategic_patch_json" (tuple . $kubeRbacProxySettings) }} type: strategic + {{- $_ := set $kubeRbacProxySettings "upstreams" (list + (dict "upstream" "http://127.0.0.1:9090/metrics" "path" "/proxy/metrics" "resource" "daemonsets" "name" "kube-api-rewriter") + (dict "upstream" "http://127.0.0.1:8080/metrics" "path" "/metrics" "resource" "daemonsets" "name" "virt-handler") + ) }} - resourceName: virt-handler resourceType: DaemonSet patch: {{ include "kube_rbac_proxy.pod_spec_strategic_patch_json" (tuple . $kubeRbacProxySettings) }} @@ -213,6 +229,11 @@ spec: resourceName: virt-handler patch: {{ include "pod_spec_priotity_class_name_patch" $priorityClassName }} type: strategic + # Patch service for https-metrics + - resourceType: Service + resourceName: kubevirt-prometheus-metrics + patch: '[{"op": "replace", "path": "/spec/ports/0/targetPort", "value": "https-metrics"}]' + type: json imagePullPolicy: IfNotPresent imagePullSecrets: diff --git a/templates/kubevirt/virt-controller/rbac-for-us.yaml b/templates/kubevirt/virt-controller/rbac-for-us.yaml new file mode 100644 index 000000000..59391c0ca --- /dev/null +++ b/templates/kubevirt/virt-controller/rbac-for-us.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: d8:{{ .Chart.Name }}:virt-controller-rbac-proxy + {{- include "helm_lib_module_labels" (list . (dict "app" "virt-controller")) | nindent 2 }} +subjects: + - kind: ServiceAccount + name: kubevirt-internal-virtualization-controller + namespace: d8-{{ .Chart.Name }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: d8:rbac-proxy diff --git a/templates/kubevirt/virt-handler/rbac-for-us.yaml b/templates/kubevirt/virt-handler/rbac-for-us.yaml new file mode 100644 index 000000000..1d6541f60 --- /dev/null +++ b/templates/kubevirt/virt-handler/rbac-for-us.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: d8:{{ .Chart.Name }}:virt-handler-rbac-proxy + {{- include "helm_lib_module_labels" (list . (dict "app" "virt-handler")) | nindent 2 }} +subjects: + - kind: ServiceAccount + name: kubevirt-internal-virtualization-handler + namespace: d8-{{ .Chart.Name }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: d8:rbac-proxy diff --git a/templates/rbac-to-us.yaml b/templates/rbac-to-us.yaml index ebc2d3203..658303b12 100644 --- a/templates/rbac-to-us.yaml +++ b/templates/rbac-to-us.yaml @@ -8,7 +8,7 @@ metadata: rules: - apiGroups: ["apps"] resources: ["daemonsets/prometheus-metrics", "deployments/prometheus-metrics"] - resourceNames: ["dvcr", "virtualization-controller", "kube-api-rewriter"] + resourceNames: ["dvcr", "virtualization-controller", "kube-api-rewriter", "virt-handler", "virt-api", "virt-controller", "cdi-deployment", "cdi-operator"] verbs: ["get"] {{- if (.Values.global.enabledModules | has "prometheus") }}