diff --git a/scrapers/kubernetes/events_watch.go b/scrapers/kubernetes/events_watch.go index 4e9a9dbad..66a1b0574 100644 --- a/scrapers/kubernetes/events_watch.go +++ b/scrapers/kubernetes/events_watch.go @@ -78,7 +78,7 @@ func WatchResources(ctx api.ScrapeContext, config v1.Kubernetes) error { } } - clientset, err := kubernetes.NewForConfig(restConfig) + clientset, err := kube.ClientSetFromRestConfig(restConfig) if err != nil { return fmt.Errorf("failed to create kubernetes clientset from rest config: %w", err) } @@ -129,7 +129,7 @@ func WatchResources(ctx api.ScrapeContext, config v1.Kubernetes) error { go informer.Run(stopper) } - ctx.Logger.V(1).Infof("waiting for informers: %w", err) + ctx.Logger.V(1).Infof("waiting for informers") wg.Wait() return nil diff --git a/utils/kube/kube.go b/utils/kube/kube.go index c41bbfc59..0d085734f 100644 --- a/utils/kube/kube.go +++ b/utils/kube/kube.go @@ -29,6 +29,8 @@ import ( clientcmdapi "k8s.io/client-go/tools/clientcmd/api" "github.com/flanksource/commons/files" + "github.com/patrickmn/go-cache" + "github.com/prometheus/client_golang/prometheus" "gopkg.in/flanksource/yaml.v3" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -38,6 +40,36 @@ import ( "k8s.io/client-go/util/homedir" ) +var ( + kubeClientCreatedCount = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "kube_client_from_rest_count", + Help: "The total number of times kubernetes clientset were created from rest configs", + }, + []string{}, + ) + + kubeClientCacheHitCount = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "kube_client_from_rest_count_cache_hit", + Help: "The total number of times kubernetes clientset were created from rest configs", + }, + []string{}, + ) + + kubeClientCreatErrorCount = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "kube_client_from_rest_error_count", + Help: "The total number of times kubernetes clientset were failed to be created from rest configs", + }, + []string{}, + ) +) + +func init() { + prometheus.MustRegister(kubeClientCreatedCount, kubeClientCacheHitCount, kubeClientCreatErrorCount) +} + func getRestMapper(config *rest.Config) (meta.RESTMapper, error) { // re-use kubectl cache host := config.Host @@ -88,13 +120,47 @@ func GetClientByGroupVersionKind(cfg *rest.Config, apiVersion, kind string) (dyn return dc.Resource(mapping.Resource), nil } +var clientSetCache = cache.New(time.Hour*24, time.Hour*24) + +func ClientSetFromRestConfig(restConfig *rest.Config) (*kubernetes.Clientset, error) { + client, cached, err := clientSetFromRestConfigCached(restConfig) + if err != nil { + kubeClientCreatErrorCount.WithLabelValues().Inc() + return nil, err + } + + if cached { + kubeClientCacheHitCount.WithLabelValues().Inc() + } else { + kubeClientCreatedCount.WithLabelValues().Inc() + } + + return client, nil +} + +func clientSetFromRestConfigCached(restConfig *rest.Config) (*kubernetes.Clientset, bool, error) { + key := restConfig.String() + + if val, ok := clientSetCache.Get(key); ok { + return val.(*kubernetes.Clientset), true, nil + } + + client, err := kubernetes.NewForConfig(restConfig) + if err != nil { + return nil, false, err + } + clientSetCache.SetDefault(key, client) + + return client, false, nil +} + func NewKubeClientWithConfigPath(kubeConfigPath string) (kubernetes.Interface, *rest.Config, error) { config, err := clientcmd.BuildConfigFromFlags("", kubeConfigPath) if err != nil { return fake.NewSimpleClientset(), nil, err } - client, err := kubernetes.NewForConfig(config) + client, err := ClientSetFromRestConfig(config) return client, config, err } @@ -118,7 +184,7 @@ func NewKubeClientWithConfig(kubeConfig string) (kubernetes.Interface, *rest.Con return fake.NewSimpleClientset(), nil, err } - client, err := kubernetes.NewForConfig(config) + client, err := ClientSetFromRestConfig(config) return client, config, err } @@ -149,13 +215,13 @@ func NewK8sClient() (kubernetes.Interface, *rest.Config, error) { } } - client, err := kubernetes.NewForConfig(restConfig) + client, err := ClientSetFromRestConfig(restConfig) return client, restConfig, err } // GetClusterName ... func GetClusterName(config *rest.Config) string { - clientset, err := kubernetes.NewForConfig(config) + clientset, err := ClientSetFromRestConfig(config) if err != nil { return "" }