feat(core): add kube-rbac-proxy to kubevirt & cdi (#532)

add kube-rbac proxy to virt-handler, virt-api, virt-controller, cdi-deployment and cdi-operator --------- Signed-off-by: Maksim Fedotov <[email protected]>
deckhouse · Dec 5, 2024 · 09830be · 09830be
1 parent 62500cb
commit 09830be
Show file tree

Hide file tree

Showing 12 changed files with 468 additions and 3 deletions.
diff --git a/images/cdi-artifact/patches/018-cover-cdi-metrics.patch b/images/cdi-artifact/patches/018-cover-cdi-metrics.patch
@@ -0,0 +1,76 @@
+diff --git a/cmd/cdi-controller/controller.go b/cmd/cdi-controller/controller.go
+index a8fdfb6b5..74fdd5210 100644
+--- a/cmd/cdi-controller/controller.go
++++ b/cmd/cdi-controller/controller.go
+@@ -36,6 +36,7 @@ import (
+ 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
+ 	"sigs.k8s.io/controller-runtime/pkg/manager"
+ 	"sigs.k8s.io/controller-runtime/pkg/manager/signals"
++	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
+
+ 	cdiv1 "kubevirt.io/containerized-data-importer-api/pkg/apis/core/v1beta1"
+ 	forklift "kubevirt.io/containerized-data-importer-api/pkg/apis/forklift/v1beta1"
+@@ -58,6 +59,7 @@ const (
+ var (
+ 	kubeconfig             string
+ 	kubeURL                string
++	metricsBindAddress     string
+ 	importerImage          string
+ 	clonerImage            string
+ 	uploadServerImage      string
+@@ -98,6 +100,7 @@ type ControllerEnvs struct {
+ func init() {
+ 	// flags
+ 	flag.StringVar(&kubeURL, "server", "", "(Optional) URL address of a remote api server.  Do not set for local clusters.")
++	flag.StringVar(&metricsBindAddress, "metrics_address", ":8080", "(Optional) URL address of a metrics server.")
+ 	klog.InitFlags(nil)
+ 	flag.Parse()
+
+@@ -199,6 +202,9 @@ func start() {
+ 		LeaderElectionResourceLock: "leases",
+ 		Cache:                      getCacheOptions(apiClient, namespace),
+ 		Scheme:                     scheme,
++		Metrics:                    metricsserver.Options{
++			BindAddress: metricsBindAddress,
++		},
+ 	}
+
+ 	cfg = config.GetConfigOrDie()
+diff --git a/cmd/cdi-operator/operator.go b/cmd/cdi-operator/operator.go
+index 90049f649..a9eab1454 100644
+--- a/cmd/cdi-operator/operator.go
++++ b/cmd/cdi-operator/operator.go
+@@ -38,6 +38,7 @@ import (
+ 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
+ 	"sigs.k8s.io/controller-runtime/pkg/manager"
+ 	"sigs.k8s.io/controller-runtime/pkg/manager/signals"
++	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
+
+ 	cdiv1 "kubevirt.io/containerized-data-importer-api/pkg/apis/core/v1beta1"
+ 	"kubevirt.io/containerized-data-importer/pkg/operator/controller"
+@@ -45,6 +46,7 @@ import (
+ )
+
+ var log = logf.Log.WithName("cmd")
++var metricsBindAddress string
+
+ func printVersion() {
+ 	log.Info(fmt.Sprintf("Go Version: %s", runtime.Version()))
+@@ -52,6 +54,7 @@ func printVersion() {
+ }
+
+ func main() {
++	flag.StringVar(&metricsBindAddress, "metrics_address", ":8080", "(Optional) URL address of a metrics server.")
+ 	flag.Parse()
+
+ 	defVerbose := fmt.Sprintf("%d", 1) // note flag values are strings
+@@ -97,6 +100,9 @@ func main() {
+ 		LeaderElectionNamespace:    namespace,
+ 		LeaderElectionID:           "cdi-operator-leader-election-helper",
+ 		LeaderElectionResourceLock: "leases",
++		Metrics:                    metricsserver.Options{
++			BindAddress: metricsBindAddress,
++		},
+ 	}
+
+ 	// Create a new Manager to provide shared dependencies and start components
diff --git a/images/cdi-artifact/patches/README.md b/images/cdi-artifact/patches/README.md
@@ -68,3 +68,11 @@ When cloning via DataVolume from PVC to PVC, there was no format conversion.
 The target PVC should have a raw type for volume mode Block and a qcow2 type for Filesystem. 
 The patch adds this conversion.
 
+#### `018-cover-cdi-metrics.patch`
+
+Configure cdi's components metrics web servers to listen on localhost. 
+This is necessary for ensuring that the metrics can be accessed only by Prometheus via kube-rbac-proxy sidecar.
+
+Currently covered metrics:
+- cdi-controller
+- cdi-deployment
diff --git a/images/virt-artifact/patches/024-cover-kubevirt-metrics.patch b/images/virt-artifact/patches/024-cover-kubevirt-metrics.patch
@@ -0,0 +1,279 @@
+diff --git a/cmd/virt-handler/virt-handler.go b/cmd/virt-handler/virt-handler.go
+index 09b2b53904..aaeafc76f3 100644
+--- a/cmd/virt-handler/virt-handler.go
++++ b/cmd/virt-handler/virt-handler.go
+@@ -93,10 +93,12 @@ const (
+ 	defaultWatchdogTimeout = 30 * time.Second
+
+ 	// Default port that virt-handler listens on.
+-	defaultPort = 8185
++	defaultPort        = 8185
++	defaultMetricsPort = 8080
+
+ 	// Default address that virt-handler listens on.
+-	defaultHost = "0.0.0.0"
++	defaultHost        = "0.0.0.0"
++	defaultMetricsHost = defaultHost
+
+ 	hostOverride = ""
+
+@@ -366,6 +368,8 @@ func (app *virtHandlerApp) Run() {
+
+ 	promErrCh := make(chan error)
+ 	go app.runPrometheusServer(promErrCh)
++	healErrCh := make(chan error)
++	go app.runHealthzServer(healErrCh)
+
+ 	lifecycleHandler := rest.NewLifecycleHandler(
+ 		recorder,
+@@ -535,7 +539,6 @@ func (app *virtHandlerApp) runPrometheusServer(errCh chan error) {
+ 	mux := restful.NewContainer()
+ 	webService := new(restful.WebService)
+ 	webService.Path("/").Consumes(restful.MIME_JSON).Produces(restful.MIME_JSON)
+-	webService.Route(webService.GET("/healthz").To(healthz.KubeConnectionHealthzFuncFactory(app.clusterConfig, apiHealthVersion)).Doc("Health endpoint"))
+
+ 	componentProfiler := profiler.NewProfileManager(app.clusterConfig)
+ 	webService.Route(webService.GET("/start-profiler").To(componentProfiler.HandleStartProfiler).Doc("start profiler endpoint"))
+@@ -546,14 +549,23 @@ func (app *virtHandlerApp) runPrometheusServer(errCh chan error) {
+ 	log.Log.V(1).Infof("metrics: max concurrent requests=%d", app.MaxRequestsInFlight)
+ 	mux.Handle("/metrics", metricshandler.Handler(app.MaxRequestsInFlight))
+ 	server := http.Server{
+-		Addr:      app.ServiceListen.Address(),
++		Addr:      app.ServiceListen.MetricsAddress(),
+ 		Handler:   mux,
+-		TLSConfig: app.promTLSConfig,
+-		// Disable HTTP/2
+-		// See CVE-2023-44487
+-		TLSNextProto: map[string]func(*http.Server, *tls.Conn, http.Handler){},
+ 	}
+-	errCh <- server.ListenAndServeTLS("", "")
++	errCh <- server.ListenAndServe()
++}
++
++func (app *virtHandlerApp) runHealthzServer(errCh chan error) {
++	mux := restful.NewContainer()
++	webService := new(restful.WebService)
++	webService.Path("/").Consumes(restful.MIME_JSON).Produces(restful.MIME_JSON)
++	webService.Route(webService.GET("/healthz").To(healthz.KubeConnectionHealthzFuncFactory(app.clusterConfig, apiHealthVersion)).Doc("Health endpoint"))
++	mux.Add(webService)
++	server := http.Server{
++		Addr:    app.ServiceListen.Address(),
++		Handler: mux,
++	}
++	errCh <- server.ListenAndServe()
+ }
+
+ func (app *virtHandlerApp) runServer(errCh chan error, consoleHandler *rest.ConsoleHandler, lifecycleHandler *rest.LifecycleHandler) {
+@@ -588,7 +600,9 @@ func (app *virtHandlerApp) AddFlags() {
+ 	app.InitFlags()
+
+ 	app.BindAddress = defaultHost
++	app.MetricsBindAddress = defaultMetricsHost
+ 	app.Port = defaultPort
++	app.MetricsPort = defaultMetricsPort
+
+ 	app.AddCommonFlags()
+
+diff --git a/pkg/service/service.go b/pkg/service/service.go
+index 0368a0762b..473328fb4b 100644
+--- a/pkg/service/service.go
++++ b/pkg/service/service.go
+@@ -42,6 +42,8 @@ type ServiceListen struct {
+ 	Name        string
+ 	BindAddress string
+ 	Port        int
++	MetricsBindAddress string
++	MetricsPort        int
+ }
+
+ type ServiceLibvirt struct {
+@@ -52,6 +54,10 @@ func (service *ServiceListen) Address() string {
+ 	return fmt.Sprintf("%s:%s", service.BindAddress, strconv.Itoa(service.Port))
+ }
+
++func (service *ServiceListen) MetricsAddress() string {
++	return fmt.Sprintf("%s:%s", service.MetricsBindAddress, strconv.Itoa(service.MetricsPort))
++}
++
+ func (service *ServiceListen) InitFlags() {
+ 	flag.CommandLine.AddGoFlag(goflag.CommandLine.Lookup("v"))
+ 	flag.CommandLine.AddGoFlag(goflag.CommandLine.Lookup("kubeconfig"))
+@@ -61,6 +67,9 @@ func (service *ServiceListen) InitFlags() {
+ func (service *ServiceListen) AddCommonFlags() {
+ 	flag.StringVar(&service.BindAddress, "listen", service.BindAddress, "Address where to listen on")
+ 	flag.IntVar(&service.Port, "port", service.Port, "Port to listen on")
++	// default values are taken from the common server counterparts
++	flag.StringVar(&service.MetricsBindAddress, "metrics-listen", service.MetricsBindAddress, "Address for metrics to listen on")
++	flag.IntVar(&service.MetricsPort, "metrics-port", service.MetricsPort, "Port for metrics to listen on")
+ }
+
+ func (service *ServiceLibvirt) AddLibvirtFlags() {
+diff --git a/pkg/virt-api/api.go b/pkg/virt-api/api.go
+index a7ea3f44d6..6d9ccd800f 100644
+--- a/pkg/virt-api/api.go
++++ b/pkg/virt-api/api.go
+@@ -82,9 +82,11 @@ import (
+ const (
+ 	// Default port that virt-api listens on.
+ 	defaultPort = 443
++	defaultMetricsPort = 8080
+
+ 	// Default address that virt-api listens on.
+ 	defaultHost = "0.0.0.0"
++	defaultMetricsHost = defaultHost
+
+ 	DefaultConsoleServerPort = 8186
+
+@@ -156,6 +158,8 @@ func NewVirtApi() VirtApi {
+ 	app := &virtAPIApp{}
+ 	app.BindAddress = defaultHost
+ 	app.Port = defaultPort
++	app.MetricsBindAddress = defaultMetricsHost
++	app.MetricsPort = defaultMetricsPort
+
+ 	return app
+ }
+@@ -968,6 +972,19 @@ func (app *virtAPIApp) setupTLS(k8sCAManager, kubevirtCAManager, virtualizationC
+ 	app.handlerTLSConfiguration = kvtls.SetupTLSForVirtHandlerClients(kubevirtCAManager, app.handlerCertManager, app.externallyManaged)
+ }
+
++func (app *virtAPIApp) startPrometheusServer(errCh chan error) {
++	mux := restful.NewContainer()
++	webService := new(restful.WebService)
++	webService.Path("/").Consumes(restful.MIME_JSON).Produces(restful.MIME_JSON)
++	mux.Add(webService)
++	mux.Handle("/metrics", promhttp.Handler())
++	server := http.Server{
++		Addr:    app.ServiceListen.MetricsAddress(),
++		Handler: mux,
++	}
++	errCh <- server.ListenAndServe()
++
++}
+ func (app *virtAPIApp) startTLS(informerFactory controller.KubeInformerFactory) error {
+
+ 	errors := make(chan error)
+@@ -991,7 +1008,6 @@ func (app *virtAPIApp) startTLS(informerFactory controller.KubeInformerFactory)
+
+ 	app.Compose()
+
+-	http.Handle("/metrics", promhttp.Handler())
+ 	server := &http.Server{
+ 		Addr:      fmt.Sprintf("%s:%d", app.BindAddress, app.Port),
+ 		TLSConfig: app.tlsConfig,
+@@ -1127,6 +1143,8 @@ func (app *virtAPIApp) Run() {
+ 	go app.certmanager.Start()
+ 	go app.handlerCertManager.Start()
+
++	promErrCh := make(chan error)
++	go app.startPrometheusServer(promErrCh)
+ 	// start TLS server
+ 	// tls server will only accept connections when fetching a certificate and internal configuration passed once
+ 	err = app.startTLS(kubeInformerFactory)
+diff --git a/pkg/virt-controller/watch/application.go b/pkg/virt-controller/watch/application.go
+index f80a0653ad..4a8d20d7be 100644
+--- a/pkg/virt-controller/watch/application.go
++++ b/pkg/virt-controller/watch/application.go
+@@ -92,8 +92,10 @@ import (
+
+ const (
+ 	defaultPort = 8182
++	defaultMetricsPort = 8080
+
+ 	defaultHost = "0.0.0.0"
++	defaultMetricsHost = defaultHost
+
+ 	launcherImage       = "virt-launcher"
+ 	exporterImage       = "virt-exportserver"
+@@ -492,10 +494,12 @@ func (vca *VirtControllerApp) Run() {
+ 	go promCertManager.Start()
+ 	promTLSConfig := kvtls.SetupPromTLS(promCertManager, vca.clusterConfig)
+
++	promErrCh := make(chan error)
++	go vca.startPrometheusServer(promErrCh)
++
+ 	go func() {
+ 		httpLogger := logger.With("service", "http")
+ 		_ = httpLogger.Level(log.INFO).Log("action", "listening", "interface", vca.BindAddress, "port", vca.Port)
+-		http.Handle("/metrics", promhttp.Handler())
+ 		server := http.Server{
+ 			Addr:      vca.Address(),
+ 			Handler:   http.DefaultServeMux,
+@@ -519,6 +523,20 @@ func (vca *VirtControllerApp) Run() {
+ 	panic("unreachable")
+ }
+
++func (app *VirtControllerApp) startPrometheusServer(errCh chan error) {
++	mux := restful.NewContainer()
++	webService := new(restful.WebService)
++	webService.Path("/").Consumes(restful.MIME_JSON).Produces(restful.MIME_JSON)
++	mux.Add(webService)
++	mux.Handle("/metrics", promhttp.Handler())
++	server := http.Server{
++		Addr:    app.ServiceListen.MetricsAddress(),
++		Handler: mux,
++	}
++	errCh <- server.ListenAndServe()
++
++}
++
+ func (vca *VirtControllerApp) onStartedLeading() func(ctx context.Context) {
+ 	return func(ctx context.Context) {
+ 		stop := ctx.Done()
+@@ -896,6 +914,8 @@ func (vca *VirtControllerApp) AddFlags() {
+
+ 	vca.BindAddress = defaultHost
+ 	vca.Port = defaultPort
++	vca.MetricsBindAddress = defaultMetricsHost
++	vca.MetricsPort = defaultMetricsPort
+
+ 	vca.AddCommonFlags()
+
+diff --git a/pkg/virt-operator/resource/generate/components/daemonsets.go b/pkg/virt-operator/resource/generate/components/daemonsets.go
+index da6e00c783..8fa14e93b5 100644
+--- a/pkg/virt-operator/resource/generate/components/daemonsets.go
++++ b/pkg/virt-operator/resource/generate/components/daemonsets.go
+@@ -174,7 +174,7 @@ func NewHandlerDaemonSet(namespace, repository, imagePrefix, version, launcherVe
+ 	}
+ 	container.Args = []string{
+ 		"--port",
+-		"8443",
++		"8090",
+ 		"--hostname-override",
+ 		"$(NODE_NAME)",
+ 		"--pod-ip-address",
+@@ -192,7 +192,7 @@ func NewHandlerDaemonSet(namespace, repository, imagePrefix, version, launcherVe
+ 		{
+ 			Name:          "metrics",
+ 			Protocol:      corev1.ProtocolTCP,
+-			ContainerPort: 8443,
++			ContainerPort: 8080,
+ 		},
+ 	}
+ 	container.SecurityContext = &corev1.SecurityContext{
+@@ -226,10 +226,10 @@ func NewHandlerDaemonSet(namespace, repository, imagePrefix, version, launcherVe
+ 		FailureThreshold: 3,
+ 		ProbeHandler: corev1.ProbeHandler{
+ 			HTTPGet: &corev1.HTTPGetAction{
+-				Scheme: corev1.URISchemeHTTPS,
++				Scheme: corev1.URISchemeHTTP,
+ 				Port: intstr.IntOrString{
+ 					Type:   intstr.Int,
+-					IntVal: 8443,
++					IntVal: 8090,
+ 				},
+ 				Path: "/healthz",
+ 			},
+@@ -241,10 +241,10 @@ func NewHandlerDaemonSet(namespace, repository, imagePrefix, version, launcherVe
+ 	container.ReadinessProbe = &corev1.Probe{
+ 		ProbeHandler: corev1.ProbeHandler{
+ 			HTTPGet: &corev1.HTTPGetAction{
+-				Scheme: corev1.URISchemeHTTPS,
++				Scheme: corev1.URISchemeHTTP,
+ 				Port: intstr.IntOrString{
+ 					Type:   intstr.Int,
+-					IntVal: 8443,
++					IntVal: 8090,
+ 				},
+ 				Path: "/healthz",
+ 			},
diff --git a/images/virt-artifact/patches/README.md b/images/virt-artifact/patches/README.md
@@ -83,3 +83,13 @@ Unsuccessful migrations may leave a lot of Pods. These huge lists reduce perform
 
 Replace the expressions for the ValidatingAdmissionPolicy kubevirt-node-restriction-policy.
 This is necessary because of the kube-api-rewriter that changes the labels.
+
+#### `024-cover-kubevirt-metrics.patch`
+
+Configure kubevirt's components metrics web servers to listen on localhost. 
+This is necessary for ensuring that the metrics can be accessed only by Prometheus via kube-rbac-proxy sidecar.
+
+Currently covered metrics:
+- virt-handler
+- virt-controller
+- virt-api
diff --git a/templates/cdi/cdi-deployment/rbac-for-us.yaml b/templates/cdi/cdi-deployment/rbac-for-us.yaml
@@ -0,0 +1,14 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: d8:{{ .Chart.Name }}:cdi-deployment-rbac-proxy
+  {{- include "helm_lib_module_labels" (list . (dict "app" "cdi-deployment")) | nindent 2 }}
+subjects:
+  - kind: ServiceAccount
+    name: cdi-sa
+    namespace: d8-{{ .Chart.Name }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: d8:rbac-proxy