feat: cache kubeconfig & add metrics

scrapers/kubernetes/events_watch.go

@@ -78,7 +78,7 @@ func WatchResources(ctx api.ScrapeContext, config v1.Kubernetes) error {
 		}
 	}
 
-	clientset, err := kubernetes.NewForConfig(restConfig)
+	clientset, err := kube.ClientSetFromRestConfig(restConfig)
 	if err != nil {
 		return fmt.Errorf("failed to create kubernetes clientset from rest config: %w", err)
 	}
@@ -129,7 +129,7 @@ func WatchResources(ctx api.ScrapeContext, config v1.Kubernetes) error {
 		go informer.Run(stopper)
 	}
 
-	ctx.Logger.V(1).Infof("waiting for informers: %w", err)
+	ctx.Logger.V(1).Infof("waiting for informers")
 	wg.Wait()
 
 	return nil

utils/kube/kube.go

@@ -29,6 +29,8 @@ import (
 	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
 
 	"github.com/flanksource/commons/files"
+	"github.com/patrickmn/go-cache"
+	"github.com/prometheus/client_golang/prometheus"
 	"gopkg.in/flanksource/yaml.v3"
 	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -38,6 +40,36 @@ import (
 	"k8s.io/client-go/util/homedir"
 )
 
+var (
+	kubeClientCreatedCount = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "kube_client_from_rest_count",
+			Help: "The total number of times kubernetes clientset were created from rest configs",
+		},
+		[]string{},
+	)
+
+	kubeClientCacheHitCount = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "kube_client_from_rest_count_cache_hit",
+			Help: "The total number of times kubernetes clientset were created from rest configs",
+		},
+		[]string{},
+	)
+
+	kubeClientCreatErrorCount = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "kube_client_from_rest_error_count",
+			Help: "The total number of times kubernetes clientset were failed to be created from rest configs",
+		},
+		[]string{},
+	)
+)
+
+func init() {
+	prometheus.MustRegister(kubeClientCreatedCount, kubeClientCacheHitCount, kubeClientCreatErrorCount)
+}
+
 func getRestMapper(config *rest.Config) (meta.RESTMapper, error) {
 	// re-use kubectl cache
 	host := config.Host
@@ -88,13 +120,47 @@ func GetClientByGroupVersionKind(cfg *rest.Config, apiVersion, kind string) (dyn
 	return dc.Resource(mapping.Resource), nil
 }
 
+var clientSetCache = cache.New(time.Hour*24, time.Hour*24)
+
+func ClientSetFromRestConfig(restConfig *rest.Config) (*kubernetes.Clientset, error) {
+	client, cached, err := clientSetFromRestConfigCached(restConfig)
+	if err != nil {
+		kubeClientCreatErrorCount.WithLabelValues().Inc()
+		return nil, err
+	}
+
+	if cached {
+		kubeClientCacheHitCount.WithLabelValues().Inc()
+	} else {
+		kubeClientCreatedCount.WithLabelValues().Inc()
+	}
+
+	return client, nil
+}
+
+func clientSetFromRestConfigCached(restConfig *rest.Config) (*kubernetes.Clientset, bool, error) {
+	key := restConfig.String()
+
+	if val, ok := clientSetCache.Get(key); ok {
+		return val.(*kubernetes.Clientset), true, nil
+	}
+
+	client, err := kubernetes.NewForConfig(restConfig)
+	if err != nil {
+		return nil, false, err
+	}
+	clientSetCache.SetDefault(key, client)
+
+	return client, false, nil
+}
+
 func NewKubeClientWithConfigPath(kubeConfigPath string) (kubernetes.Interface, *rest.Config, error) {
 	config, err := clientcmd.BuildConfigFromFlags("", kubeConfigPath)
 	if err != nil {
 		return fake.NewSimpleClientset(), nil, err
 	}
 
-	client, err := kubernetes.NewForConfig(config)
+	client, err := ClientSetFromRestConfig(config)
 	return client, config, err
 }
 
@@ -118,7 +184,7 @@ func NewKubeClientWithConfig(kubeConfig string) (kubernetes.Interface, *rest.Con
 		return fake.NewSimpleClientset(), nil, err
 	}
 
-	client, err := kubernetes.NewForConfig(config)
+	client, err := ClientSetFromRestConfig(config)
 	return client, config, err
 }
 
@@ -149,13 +215,13 @@ func NewK8sClient() (kubernetes.Interface, *rest.Config, error) {
 		}
 	}
 
-	client, err := kubernetes.NewForConfig(restConfig)
+	client, err := ClientSetFromRestConfig(restConfig)
 	return client, restConfig, err
 }
 
 // GetClusterName ...
 func GetClusterName(config *rest.Config) string {
-	clientset, err := kubernetes.NewForConfig(config)
+	clientset, err := ClientSetFromRestConfig(config)
 	if err != nil {
 		return ""
 	}