From 330309a187a8f441b4e61290312818a2f2997f6a Mon Sep 17 00:00:00 2001 From: Quyc <94251049@qq.com> Date: Wed, 19 Jul 2023 00:21:41 +0800 Subject: [PATCH] perf: API support for k8s version 1.27 (#1711) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: 曲源成 --- api/handler/nodes.go | 22 +++- node/kubecache/kube_cache.go | 22 +++- worker/appm/controller/export.go | 23 +++- worker/appm/controller/refresh_xpa.go | 35 ++++-- worker/appm/controller/start.go | 30 +++-- worker/appm/controller/stop.go | 23 +++- worker/appm/conversion/autoscaler.go | 128 +++++++++++++++++++++- worker/appm/conversion/autoscaler_test.go | 2 +- worker/appm/f/function.go | 24 +++- worker/appm/store/lister.go | 46 ++++---- worker/appm/store/store.go | 93 ++++++++++++---- worker/appm/types/v1/v1.go | 58 +++++++++- worker/gc/gc.go | 3 + 13 files changed, 429 insertions(+), 80 deletions(-) diff --git a/api/handler/nodes.go b/api/handler/nodes.go index bd2717f553..9948e5ce44 100644 --- a/api/handler/nodes.go +++ b/api/handler/nodes.go @@ -4,15 +4,18 @@ import ( "context" "fmt" "github.com/goodrain/rainbond/api/client/prometheus" + k8sutil "github.com/goodrain/rainbond/util/k8s" "github.com/pquerna/ffjson/ffjson" "github.com/shirou/gopsutil/disk" "github.com/sirupsen/logrus" v1 "k8s.io/api/core/v1" - "k8s.io/api/policy/v1beta1" + policyv1 "k8s.io/api/policy/v1" + policyv1beta1 "k8s.io/api/policy/v1beta1" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/types" + utilversion "k8s.io/apimachinery/pkg/util/version" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "runtime" @@ -336,7 +339,22 @@ func (n *nodesHandle) GetNodeScheduler(ctx context.Context, nodeName string) (st //evictPod - func (n *nodesHandle) evictPod(pod v1.Pod, policyGroupVersion string) error { deleteOptions := &metav1.DeleteOptions{} - eviction := &v1beta1.Eviction{ + if k8sutil.GetKubeVersion().AtLeast(utilversion.MustParseSemantic("v1.21.0")) { + eviction := &policyv1.Eviction{ + TypeMeta: metav1.TypeMeta{ + APIVersion: policyGroupVersion, + Kind: EvictionKind, + }, + ObjectMeta: metav1.ObjectMeta{ + Name: pod.Name, + Namespace: pod.Namespace, + }, + DeleteOptions: deleteOptions, + } + // Remember to change change the URL manipulation func when Evction's version change + return n.clientset.PolicyV1().Evictions(eviction.Namespace).Evict(context.Background(), eviction) + } + eviction := &policyv1beta1.Eviction{ TypeMeta: metav1.TypeMeta{ APIVersion: policyGroupVersion, Kind: EvictionKind, diff --git a/node/kubecache/kube_cache.go b/node/kubecache/kube_cache.go index 9b4c398748..f203752f45 100644 --- a/node/kubecache/kube_cache.go +++ b/node/kubecache/kube_cache.go @@ -21,6 +21,8 @@ package kubecache import ( "context" "fmt" + k8sutil "github.com/goodrain/rainbond/util/k8s" + utilversion "k8s.io/apimachinery/pkg/util/version" "math" "strings" "time" @@ -35,7 +37,8 @@ import ( "github.com/sirupsen/logrus" v1 "k8s.io/api/core/v1" - "k8s.io/api/policy/v1beta1" + policyv1 "k8s.io/api/policy/v1" + policyv1beta1 "k8s.io/api/policy/v1beta1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -195,7 +198,22 @@ func (k *kubeClient) GetPodsByNodes(nodeName string) (pods []v1.Pod, err error) //evictPod 驱离POD func (k *kubeClient) evictPod(pod v1.Pod, policyGroupVersion string) error { deleteOptions := &metav1.DeleteOptions{} - eviction := &v1beta1.Eviction{ + if k8sutil.GetKubeVersion().AtLeast(utilversion.MustParseSemantic("v1.21.0")) { + eviction := &policyv1.Eviction{ + TypeMeta: metav1.TypeMeta{ + APIVersion: policyGroupVersion, + Kind: EvictionKind, + }, + ObjectMeta: metav1.ObjectMeta{ + Name: pod.Name, + Namespace: pod.Namespace, + }, + DeleteOptions: deleteOptions, + } + // Remember to change change the URL manipulation func when Evction's version change + return k.kubeclient.PolicyV1().Evictions(eviction.Namespace).Evict(context.Background(), eviction) + } + eviction := &policyv1beta1.Eviction{ TypeMeta: metav1.TypeMeta{ APIVersion: policyGroupVersion, Kind: EvictionKind, diff --git a/worker/appm/controller/export.go b/worker/appm/controller/export.go index 8c46e04b62..1f2b949fcc 100644 --- a/worker/appm/controller/export.go +++ b/worker/appm/controller/export.go @@ -8,7 +8,8 @@ import ( v1 "github.com/goodrain/rainbond/worker/appm/types/v1" "github.com/sirupsen/logrus" appv1 "k8s.io/api/apps/v1" - "k8s.io/api/autoscaling/v2beta2" + autoscalv2beta2 "k8s.io/api/autoscaling/v2beta2" + autoscalv2 "k8s.io/api/autoscaling/v2" batchv1 "k8s.io/api/batch/v1" "k8s.io/api/batch/v1beta1" corev1 "k8s.io/api/core/v1" @@ -356,7 +357,25 @@ func (s *exportController) exportOne(app v1.AppService, r *RainbondExport) error hpa.Kind = "HorizontalPodAutoscaler" hpa.Namespace = "" hpa.APIVersion = APIVersionHorizontalPodAutoscaler - hpa.Status = v2beta2.HorizontalPodAutoscalerStatus{} + hpa.Status = autoscalv2.HorizontalPodAutoscalerStatus{} + if len(hpa.ResourceVersion) == 0 { + hpaBytes, err := yaml.Marshal(hpa) + if err != nil { + return fmt.Errorf("hpa to yaml failure %v", err) + } + err = s.write(path.Join(exportTemplatePath, "HorizontalPodAutoscaler.yaml"), hpaBytes, "\n---\n") + if err != nil { + return fmt.Errorf("write hpa yaml failure %v", err) + } + } + } + } + if hpas := app.GetHPABeta2s(); len(hpas) != 0 { + for _, hpa := range hpas { + hpa.Kind = "HorizontalPodAutoscaler" + hpa.Namespace = "" + hpa.APIVersion = APIVersionHorizontalPodAutoscaler + hpa.Status = autoscalv2beta2.HorizontalPodAutoscalerStatus{} if len(hpa.ResourceVersion) == 0 { hpaBytes, err := yaml.Marshal(hpa) if err != nil { diff --git a/worker/appm/controller/refresh_xpa.go b/worker/appm/controller/refresh_xpa.go index fe410096f7..09572f1fd9 100644 --- a/worker/appm/controller/refresh_xpa.go +++ b/worker/appm/controller/refresh_xpa.go @@ -20,6 +20,8 @@ package controller import ( "context" + k8sutil "github.com/goodrain/rainbond/util/k8s" + utilversion "k8s.io/apimachinery/pkg/util/version" "sync" "github.com/sirupsen/logrus" @@ -55,16 +57,31 @@ func (a *refreshXPAController) Begin() { } func (a *refreshXPAController) applyOne(clientset kubernetes.Interface, app *v1.AppService) error { - for _, hpa := range app.GetHPAs() { - f.EnsureHPA(hpa, clientset) - } + if k8sutil.GetKubeVersion().AtLeast(utilversion.MustParseSemantic("v1.23.0")) { + for _, hpa := range app.GetHPAs() { + f.EnsureHPA(hpa, clientset) + } - for _, hpa := range app.GetDelHPAs() { - logrus.Debugf("hpa name: %s; start deleting hpa.", hpa.GetName()) - err := clientset.AutoscalingV2beta2().HorizontalPodAutoscalers(hpa.GetNamespace()).Delete(context.Background(), hpa.GetName(), metav1.DeleteOptions{}) - if err != nil { - // don't return error, hope it is ok next time - logrus.Warningf("error deleting secret(%#v): %v", hpa, err) + for _, hpa := range app.GetDelHPAs() { + logrus.Debugf("hpa name: %s; start deleting hpa.", hpa.GetName()) + err := clientset.AutoscalingV2().HorizontalPodAutoscalers(hpa.GetNamespace()).Delete(context.Background(), hpa.GetName(), metav1.DeleteOptions{}) + if err != nil { + // don't return error, hope it is ok next time + logrus.Warningf("error deleting secret(%#v): %v", hpa, err) + } + } + } else { + for _, hpa := range app.GetHPABeta2s() { + f.EnsureHPABetav2(hpa, clientset) + } + + for _, hpa := range app.GetDelHPABeta2s() { + logrus.Debugf("hpa name: %s; start deleting hpa.", hpa.GetName()) + err := clientset.AutoscalingV2beta2().HorizontalPodAutoscalers(hpa.GetNamespace()).Delete(context.Background(), hpa.GetName(), metav1.DeleteOptions{}) + if err != nil { + // don't return error, hope it is ok next time + logrus.Warningf("error deleting secret(%#v): %v", hpa, err) + } } } diff --git a/worker/appm/controller/start.go b/worker/appm/controller/start.go index c9a8f3602f..a35a20961b 100644 --- a/worker/appm/controller/start.go +++ b/worker/appm/controller/start.go @@ -21,6 +21,8 @@ package controller import ( "context" "fmt" + k8sutil "github.com/goodrain/rainbond/util/k8s" + utilversion "k8s.io/apimachinery/pkg/util/version" "sync" "time" @@ -198,13 +200,27 @@ func (s *startController) startOne(app v1.AppService) error { } //step 6: create hpa - if hpas := app.GetHPAs(); len(hpas) != 0 { - for _, hpa := range hpas { - if len(hpa.ResourceVersion) == 0 { - _, err := s.manager.client.AutoscalingV2beta2().HorizontalPodAutoscalers(hpa.GetNamespace()).Create(s.ctx, hpa, metav1.CreateOptions{}) - if err != nil && !errors.IsAlreadyExists(err) { - logrus.Debugf("hpa: %#v", hpa) - return fmt.Errorf("create hpa: %v", err) + if k8sutil.GetKubeVersion().AtLeast(utilversion.MustParseSemantic("v1.23.0")) { + if hpas := app.GetHPAs(); len(hpas) != 0 { + for _, hpa := range hpas { + if len(hpa.ResourceVersion) == 0 { + _, err := s.manager.client.AutoscalingV2().HorizontalPodAutoscalers(hpa.GetNamespace()).Create(s.ctx, hpa, metav1.CreateOptions{}) + if err != nil && !errors.IsAlreadyExists(err) { + logrus.Debugf("hpa: %#v", hpa) + return fmt.Errorf("create hpa: %v", err) + } + } + } + } + } else { + if hpas := app.GetHPABeta2s(); len(hpas) != 0 { + for _, hpa := range hpas { + if len(hpa.ResourceVersion) == 0 { + _, err := s.manager.client.AutoscalingV2beta2().HorizontalPodAutoscalers(hpa.GetNamespace()).Create(s.ctx, hpa, metav1.CreateOptions{}) + if err != nil && !errors.IsAlreadyExists(err) { + logrus.Debugf("hpa: %#v", hpa) + return fmt.Errorf("create hpa: %v", err) + } } } } diff --git a/worker/appm/controller/stop.go b/worker/appm/controller/stop.go index 3984a67ae2..4f7c73cc8b 100644 --- a/worker/appm/controller/stop.go +++ b/worker/appm/controller/stop.go @@ -22,7 +22,9 @@ import ( "context" "fmt" "github.com/goodrain/rainbond/db" + k8sutil "github.com/goodrain/rainbond/util/k8s" "github.com/jinzhu/gorm" + utilversion "k8s.io/apimachinery/pkg/util/version" "sync" "time" @@ -206,11 +208,22 @@ func (s *stopController) stopOne(app v1.AppService) error { } } //step 7: deleta all hpa - if hpas := app.GetHPAs(); len(hpas) != 0 { - for _, hpa := range hpas { - err := s.manager.client.AutoscalingV2beta2().HorizontalPodAutoscalers(hpa.GetNamespace()).Delete(s.ctx, hpa.GetName(), metav1.DeleteOptions{}) - if err != nil && !errors.IsNotFound(err) { - return fmt.Errorf("delete hpa: %v", err) + if k8sutil.GetKubeVersion().AtLeast(utilversion.MustParseSemantic("v1.23.0")) { + if hpas := app.GetHPAs(); len(hpas) != 0 { + for _, hpa := range hpas { + err := s.manager.client.AutoscalingV2().HorizontalPodAutoscalers(hpa.GetNamespace()).Delete(s.ctx, hpa.GetName(), metav1.DeleteOptions{}) + if err != nil && !errors.IsNotFound(err) { + return fmt.Errorf("delete hpa: %v", err) + } + } + } + } else { + if hpas := app.GetHPABeta2s(); len(hpas) != 0 { + for _, hpa := range hpas { + err := s.manager.client.AutoscalingV2beta2().HorizontalPodAutoscalers(hpa.GetNamespace()).Delete(s.ctx, hpa.GetName(), metav1.DeleteOptions{}) + if err != nil && !errors.IsNotFound(err) { + return fmt.Errorf("delete hpa: %v", err) + } } } } diff --git a/worker/appm/conversion/autoscaler.go b/worker/appm/conversion/autoscaler.go index 57f362f4c5..8fbe50dd77 100644 --- a/worker/appm/conversion/autoscaler.go +++ b/worker/appm/conversion/autoscaler.go @@ -20,10 +20,13 @@ package conversion import ( "fmt" + k8sutil "github.com/goodrain/rainbond/util/k8s" + utilversion "k8s.io/apimachinery/pkg/util/version" "github.com/sirupsen/logrus" - autoscalingv2 "k8s.io/api/autoscaling/v2beta2" + autoscalingv2beta2 "k8s.io/api/autoscaling/v2beta2" + autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -41,15 +44,128 @@ var str2ResourceName = map[string]corev1.ResourceName{ // TenantServiceAutoscaler - func TenantServiceAutoscaler(as *v1.AppService, dbmanager db.Manager) error { - hpas, err := newHPAs(as, dbmanager) + if k8sutil.GetKubeVersion().AtLeast(utilversion.MustParseSemantic("v1.23.0")) { + hpas, err := newHPAs(as, dbmanager) + if err != nil { + return fmt.Errorf("create HPAs: %v", err) + } + logrus.Debugf("the numbers of HPAs: %d", len(hpas)) + + as.SetHPAs(hpas) + + } else { + hpas, err := newHPABeta2s(as, dbmanager) + if err != nil { + return fmt.Errorf("create Beta2 HPAs: %v", err) + } + logrus.Debugf("the numbers of Beta2 HPAs: %d", len(hpas)) + + as.SetHPAbeta2s(hpas) + + } + return nil +} + +func newHPABeta2s(as *v1.AppService, dbmanager db.Manager) ([]*autoscalingv2beta2.HorizontalPodAutoscaler, error) { + xpaRules, err := dbmanager.TenantServceAutoscalerRulesDao().ListEnableOnesByServiceID(as.ServiceID) if err != nil { - return fmt.Errorf("create HPAs: %v", err) + return nil, err } - logrus.Debugf("the numbers of HPAs: %d", len(hpas)) - as.SetHPAs(hpas) + var hpas []*autoscalingv2beta2.HorizontalPodAutoscaler + for _, rule := range xpaRules { + metrics, err := dbmanager.TenantServceAutoscalerRuleMetricsDao().ListByRuleID(rule.RuleID) + if err != nil { + return nil, err + } - return nil + var kind, name string + if as.GetStatefulSet() != nil { + kind, name = "StatefulSet", as.GetStatefulSet().GetName() + } else { + kind, name = "Deployment", as.GetDeployment().GetName() + } + + labels := as.GetCommonLabels(map[string]string{ + "rule_id": rule.RuleID, + "version": as.DeployVersion, + }) + + hpa := newHPABeta2(as.GetNamespace(), kind, name, labels, rule, metrics) + + hpas = append(hpas, hpa) + } + + return hpas, nil +} + +func createResourceMetricsBeta2(metric *model.TenantServiceAutoscalerRuleMetrics) autoscalingv2beta2.MetricSpec { + ms := autoscalingv2beta2.MetricSpec{ + Type: autoscalingv2beta2.ResourceMetricSourceType, + Resource: &autoscalingv2beta2.ResourceMetricSource{ + Name: str2ResourceName[metric.MetricsName], + }, + } + + if metric.MetricTargetType == "utilization" { + value := int32(metric.MetricTargetValue) + ms.Resource.Target = autoscalingv2beta2.MetricTarget{ + Type: autoscalingv2beta2.UtilizationMetricType, + AverageUtilization: &value, + } + } + if metric.MetricTargetType == "average_value" { + ms.Resource.Target.Type = autoscalingv2beta2.AverageValueMetricType + if metric.MetricsName == "cpu" { + ms.Resource.Target.AverageValue = resource.NewMilliQuantity(int64(metric.MetricTargetValue), resource.DecimalSI) + } + if metric.MetricsName == "memory" { + ms.Resource.Target.AverageValue = resource.NewQuantity(int64(metric.MetricTargetValue*1024*1024), resource.BinarySI) + } + } + + return ms +} + +func newHPABeta2(namespace, kind, name string, labels map[string]string, rule *model.TenantServiceAutoscalerRules, metrics []*model.TenantServiceAutoscalerRuleMetrics) *autoscalingv2beta2.HorizontalPodAutoscaler { + hpa := &autoscalingv2beta2.HorizontalPodAutoscaler{ + ObjectMeta: metav1.ObjectMeta{ + Name: rule.RuleID, + Namespace: namespace, + Labels: labels, + }, + } + + spec := autoscalingv2beta2.HorizontalPodAutoscalerSpec{ + MinReplicas: util.Int32(int32(rule.MinReplicas)), + MaxReplicas: int32(rule.MaxReplicas), + ScaleTargetRef: autoscalingv2beta2.CrossVersionObjectReference{ + Kind: kind, + Name: name, + APIVersion: "apps/v1", + }, + } + + for _, metric := range metrics { + if metric.MetricsType != "resource_metrics" { + logrus.Warningf("rule id: %s; unsupported metric type: %s", rule.RuleID, metric.MetricsType) + continue + } + if metric.MetricTargetValue <= 0 { + // TODO: If the target value of cpu and memory is 0, it will not take effect. + // TODO: The target value of the custom indicator can be 0. + continue + } + + ms := createResourceMetricsBeta2(metric) + spec.Metrics = append(spec.Metrics, ms) + } + if len(spec.Metrics) == 0 { + return nil + } + hpa.Spec = spec + + return hpa } func newHPAs(as *v1.AppService, dbmanager db.Manager) ([]*autoscalingv2.HorizontalPodAutoscaler, error) { diff --git a/worker/appm/conversion/autoscaler_test.go b/worker/appm/conversion/autoscaler_test.go index 536e443e16..60f6502328 100644 --- a/worker/appm/conversion/autoscaler_test.go +++ b/worker/appm/conversion/autoscaler_test.go @@ -53,7 +53,7 @@ func TestNewHPA(t *testing.T) { t.Fatalf("error creating k8s clientset: %s", err.Error()) } - _, err = clientset.AutoscalingV2beta2().HorizontalPodAutoscalers(hpa.GetNamespace()).Create(context.Background(), hpa, metav1.CreateOptions{}) + _, err = clientset.AutoscalingV2().HorizontalPodAutoscalers(hpa.GetNamespace()).Create(context.Background(), hpa, metav1.CreateOptions{}) if err != nil { t.Fatalf("create hpa: %v", err) } diff --git a/worker/appm/f/function.go b/worker/appm/f/function.go index 1aa4733ec9..a6cdfa8696 100644 --- a/worker/appm/f/function.go +++ b/worker/appm/f/function.go @@ -29,7 +29,8 @@ import ( monitorv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" "github.com/prometheus-operator/prometheus-operator/pkg/client/versioned" "github.com/sirupsen/logrus" - autoscalingv2 "k8s.io/api/autoscaling/v2beta2" + autoscalingv2 "k8s.io/api/autoscaling/v2" + autoscalingv2beta2 "k8s.io/api/autoscaling/v2beta2" corev1 "k8s.io/api/core/v1" networkingv1 "k8s.io/api/networking/v1" betav1 "k8s.io/api/networking/v1beta1" @@ -332,6 +333,27 @@ func EnsureService(new *corev1.Service, clientSet kubernetes.Interface) error { // EnsureHPA - func EnsureHPA(new *autoscalingv2.HorizontalPodAutoscaler, clientSet kubernetes.Interface) { + _, err := clientSet.AutoscalingV2().HorizontalPodAutoscalers(new.Namespace).Get(context.Background(), new.Name, metav1.GetOptions{}) + if err != nil { + if k8sErrors.IsNotFound(err) { + _, err = clientSet.AutoscalingV2().HorizontalPodAutoscalers(new.Namespace).Create(context.Background(), new, metav1.CreateOptions{}) + if err != nil { + logrus.Warningf("error creating hpa %+v: %v", new, err) + } + return + } + logrus.Errorf("error getting hpa(%s): %v", fmt.Sprintf("%s/%s", new.Namespace, new.Name), err) + return + } + _, err = clientSet.AutoscalingV2().HorizontalPodAutoscalers(new.Namespace).Update(context.Background(), new, metav1.UpdateOptions{}) + if err != nil { + logrus.Warningf("error updating hpa %+v: %v", new, err) + return + } +} + +// EnsureHPABetav2 - +func EnsureHPABetav2(new *autoscalingv2beta2.HorizontalPodAutoscaler, clientSet kubernetes.Interface) { _, err := clientSet.AutoscalingV2beta2().HorizontalPodAutoscalers(new.Namespace).Get(context.Background(), new.Name, metav1.GetOptions{}) if err != nil { if k8sErrors.IsNotFound(err) { diff --git a/worker/appm/store/lister.go b/worker/appm/store/lister.go index 90ffa5e4f2..b6d518518e 100644 --- a/worker/appm/store/lister.go +++ b/worker/appm/store/lister.go @@ -22,7 +22,8 @@ import ( "github.com/goodrain/rainbond/pkg/generated/listers/rainbond/v1alpha1" crdlisters "k8s.io/apiextensions-apiserver/pkg/client/listers/apiextensions/v1" appsv1 "k8s.io/client-go/listers/apps/v1" - autoscalingv2 "k8s.io/client-go/listers/autoscaling/v2beta2" + autoscalingv2 "k8s.io/client-go/listers/autoscaling/v2" + autoscalingv2beta2 "k8s.io/client-go/listers/autoscaling/v2beta2" v1 "k8s.io/client-go/listers/batch/v1" batchv1beta1 "k8s.io/client-go/listers/batch/v1beta1" corev1 "k8s.io/client-go/listers/core/v1" @@ -33,25 +34,26 @@ import ( //Lister kube-api client cache type Lister struct { - Ingress networkingv1.IngressLister - BetaIngress betav1.IngressLister - Service corev1.ServiceLister - Secret corev1.SecretLister - StatefulSet appsv1.StatefulSetLister - Deployment appsv1.DeploymentLister - Pod corev1.PodLister - ReplicaSets appsv1.ReplicaSetLister - ConfigMap corev1.ConfigMapLister - Endpoints corev1.EndpointsLister - Nodes corev1.NodeLister - StorageClass storagev1.StorageClassLister - Claims corev1.PersistentVolumeClaimLister - HorizontalPodAutoscaler autoscalingv2.HorizontalPodAutoscalerLister - CRD crdlisters.CustomResourceDefinitionLister - HelmApp v1alpha1.HelmAppLister - ComponentDefinition v1alpha1.ComponentDefinitionLister - ThirdComponent v1alpha1.ThirdComponentLister - Job v1.JobLister - CronJob v1.CronJobLister - BetaCronJob batchv1beta1.CronJobLister + Ingress networkingv1.IngressLister + BetaIngress betav1.IngressLister + Service corev1.ServiceLister + Secret corev1.SecretLister + StatefulSet appsv1.StatefulSetLister + Deployment appsv1.DeploymentLister + Pod corev1.PodLister + ReplicaSets appsv1.ReplicaSetLister + ConfigMap corev1.ConfigMapLister + Endpoints corev1.EndpointsLister + Nodes corev1.NodeLister + StorageClass storagev1.StorageClassLister + Claims corev1.PersistentVolumeClaimLister + HorizontalPodAutoscaler autoscalingv2.HorizontalPodAutoscalerLister + HorizontalPodAutoscalerbeta2 autoscalingv2beta2.HorizontalPodAutoscalerLister + CRD crdlisters.CustomResourceDefinitionLister + HelmApp v1alpha1.HelmAppLister + ComponentDefinition v1alpha1.ComponentDefinitionLister + ThirdComponent v1alpha1.ThirdComponentLister + Job v1.JobLister + CronJob v1.CronJobLister + BetaCronJob batchv1beta1.CronJobLister } diff --git a/worker/appm/store/store.go b/worker/appm/store/store.go index eabe31f8df..5b3dd75f1d 100644 --- a/worker/appm/store/store.go +++ b/worker/appm/store/store.go @@ -52,7 +52,8 @@ import ( "github.com/prometheus-operator/prometheus-operator/pkg/client/versioned" "github.com/sirupsen/logrus" appsv1 "k8s.io/api/apps/v1" - autoscalingv2 "k8s.io/api/autoscaling/v2beta2" + autoscalingv2 "k8s.io/api/autoscaling/v2" + autoscalingv2beta2 "k8s.io/api/autoscaling/v2beta2" corev1 "k8s.io/api/core/v1" networkingv1 "k8s.io/api/networking/v1" storagev1 "k8s.io/api/storage/v1" @@ -243,8 +244,13 @@ func NewStore( store.informers.Events = infFactory.Core().V1().Events().Informer() - store.informers.HorizontalPodAutoscaler = infFactory.Autoscaling().V2beta2().HorizontalPodAutoscalers().Informer() - store.listers.HorizontalPodAutoscaler = infFactory.Autoscaling().V2beta2().HorizontalPodAutoscalers().Lister() + if k8sutil.GetKubeVersion().AtLeast(utilversion.MustParseSemantic("v1.23.0")) { + store.informers.HorizontalPodAutoscaler = infFactory.Autoscaling().V2().HorizontalPodAutoscalers().Informer() + store.listers.HorizontalPodAutoscaler = infFactory.Autoscaling().V2().HorizontalPodAutoscalers().Lister() + } else { + store.informers.HorizontalPodAutoscaler = infFactory.Autoscaling().V2beta2().HorizontalPodAutoscalers().Informer() + store.listers.HorizontalPodAutoscalerbeta2 = infFactory.Autoscaling().V2beta2().HorizontalPodAutoscalers().Lister() + } // rainbond custom resource rainbondInformer := externalversions.NewSharedInformerFactoryWithOptions(rainbondClient, 10*time.Second, @@ -671,23 +677,40 @@ func (a *appRuntimeStore) OnAdd(obj interface{}) { } } } - - if hpa, ok := obj.(*autoscalingv2.HorizontalPodAutoscaler); ok { - serviceID := hpa.Labels["service_id"] - version := hpa.Labels["version"] - createrID := hpa.Labels["creater_id"] - if serviceID != "" && version != "" && createrID != "" { - appservice, err := a.getAppService(serviceID, version, createrID, true) - if err == conversion.ErrServiceNotFound { - a.conf.KubeClient.AutoscalingV2beta2().HorizontalPodAutoscalers(hpa.GetNamespace()).Delete(context.Background(), hpa.GetName(), metav1.DeleteOptions{}) + if k8sutil.GetKubeVersion().AtLeast(utilversion.MustParseSemantic("v1.23.0")) { + if hpa, ok := obj.(*autoscalingv2.HorizontalPodAutoscaler); ok { + serviceID := hpa.Labels["service_id"] + version := hpa.Labels["version"] + createrID := hpa.Labels["creater_id"] + if serviceID != "" && version != "" && createrID != "" { + appservice, err := a.getAppService(serviceID, version, createrID, true) + if err == conversion.ErrServiceNotFound { + a.conf.KubeClient.AutoscalingV2().HorizontalPodAutoscalers(hpa.GetNamespace()).Delete(context.Background(), hpa.GetName(), metav1.DeleteOptions{}) + } + if appservice != nil { + appservice.SetHPA(hpa) + } + return } - if appservice != nil { - appservice.SetHPA(hpa) + } + } else { + if hpa, ok := obj.(*autoscalingv2beta2.HorizontalPodAutoscaler); ok { + serviceID := hpa.Labels["service_id"] + version := hpa.Labels["version"] + createrID := hpa.Labels["creater_id"] + if serviceID != "" && version != "" && createrID != "" { + appservice, err := a.getAppService(serviceID, version, createrID, true) + if err == conversion.ErrServiceNotFound { + a.conf.KubeClient.AutoscalingV2beta2().HorizontalPodAutoscalers(hpa.GetNamespace()).Delete(context.Background(), hpa.GetName(), metav1.DeleteOptions{}) + } + if appservice != nil { + appservice.SetHPAbeta2(hpa) + } + return } - - return } } + if sc, ok := obj.(*storagev1.StorageClass); ok { vt := workerutil.TransStorageClass2RBDVolumeType(sc) for _, ch := range a.volumeTypeListeners { @@ -955,6 +978,21 @@ func (a *appRuntimeStore) OnDeletes(objs ...interface{}) { } } } + if hpa, ok := obj.(*autoscalingv2beta2.HorizontalPodAutoscaler); ok { + serviceID := hpa.Labels["service_id"] + version := hpa.Labels["version"] + createrID := hpa.Labels["creater_id"] + if serviceID != "" && version != "" && createrID != "" { + appservice, _ := a.getAppService(serviceID, version, createrID, false) + if appservice != nil { + appservice.DelHPABeta2(hpa) + if appservice.IsClosed() { + a.DeleteAppService(appservice) + } + return + } + } + } if sc, ok := obj.(*storagev1.StorageClass); ok { if err := a.dbmanager.VolumeTypeDao().DeleteModelByVolumeTypes(sc.GetName()); err != nil { logrus.Errorf("delete volumeType from db error: %s", err.Error()) @@ -1755,13 +1793,24 @@ func (a *appRuntimeStore) scalingRecordServiceAndRuleID(evt *corev1.Event) (stri serviceID = statefulset.GetLabels()["service_id"] ruleID = statefulset.GetLabels()["rule_id"] case "HorizontalPodAutoscaler": - hpa, err := a.listers.HorizontalPodAutoscaler.HorizontalPodAutoscalers(evt.InvolvedObject.Namespace).Get(evt.InvolvedObject.Name) - if err != nil { - logrus.Warningf("retrieve statefulset: %v", err) - return "", "" + if k8sutil.GetKubeVersion().AtLeast(utilversion.MustParseSemantic("v1.23.0")) { + hpa, err := a.listers.HorizontalPodAutoscaler.HorizontalPodAutoscalers(evt.InvolvedObject.Namespace).Get(evt.InvolvedObject.Name) + if err != nil { + logrus.Warningf("retrieve statefulset: %v", err) + return "", "" + } + serviceID = hpa.GetLabels()["service_id"] + ruleID = hpa.GetLabels()["rule_id"] + } else { + hpa, err := a.listers.HorizontalPodAutoscalerbeta2.HorizontalPodAutoscalers(evt.InvolvedObject.Namespace).Get(evt.InvolvedObject.Name) + if err != nil { + logrus.Warningf("retrieve statefulset: %v", err) + return "", "" + } + serviceID = hpa.GetLabels()["service_id"] + ruleID = hpa.GetLabels()["rule_id"] } - serviceID = hpa.GetLabels()["service_id"] - ruleID = hpa.GetLabels()["rule_id"] + default: logrus.Warningf("unsupported object kind: %s", evt.InvolvedObject.Kind) } diff --git a/worker/appm/types/v1/v1.go b/worker/appm/types/v1/v1.go index 76c13f9837..1b5ec00d82 100644 --- a/worker/appm/types/v1/v1.go +++ b/worker/appm/types/v1/v1.go @@ -35,7 +35,8 @@ import ( monitorv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" "github.com/sirupsen/logrus" v1 "k8s.io/api/apps/v1" - autoscalingv2 "k8s.io/api/autoscaling/v2beta2" + autoscalingv2 "k8s.io/api/autoscaling/v2" + autoscalingv2beta2 "k8s.io/api/autoscaling/v2beta2" batchv1 "k8s.io/api/batch/v1" batchv1beta1 "k8s.io/api/batch/v1beta1" corev1 "k8s.io/api/core/v1" @@ -177,7 +178,9 @@ type AppService struct { betaCronJob *batchv1beta1.CronJob workload client.Object hpas []*autoscalingv2.HorizontalPodAutoscaler + hpasbeta2s []*autoscalingv2beta2.HorizontalPodAutoscaler delHPAs []*autoscalingv2.HorizontalPodAutoscaler + delHPAbeta2s []*autoscalingv2beta2.HorizontalPodAutoscaler replicasets []*v1.ReplicaSet services []*corev1.Service delServices []*corev1.Service @@ -829,6 +832,18 @@ func (a *AppService) SetDeletedResources(old *AppService) { a.delHPAs = append(a.delHPAs, o) } } + for _, o := range old.GetHPABeta2s() { + del := true + for _, n := range a.GetHPABeta2s() { + if o.Name == n.Name { + del = false + break + } + } + if del { + a.delHPAbeta2s = append(a.delHPAbeta2s, o) + } + } } // DistinguishPod uses replica set to distinguish between old and new pods @@ -923,6 +938,24 @@ func (a *AppService) SetHPA(hpa *autoscalingv2.HorizontalPodAutoscaler) { a.hpas = append(a.hpas, hpa) } +// SetHPAbeta2s - +func (a *AppService) SetHPAbeta2s(hpasbeta2s []*autoscalingv2beta2.HorizontalPodAutoscaler) { + a.hpasbeta2s = hpasbeta2s +} + +// SetHPAbeta2 - +func (a *AppService) SetHPAbeta2(hpasbeta2 *autoscalingv2beta2.HorizontalPodAutoscaler) { + if len(a.hpasbeta2s) > 0 { + for i, old := range a.hpasbeta2s { + if old.GetName() == hpasbeta2.GetName() { + a.hpasbeta2s[i] = hpasbeta2 + return + } + } + } + a.hpasbeta2s = append(a.hpasbeta2s, hpasbeta2) +} + // SetServiceMonitor - func (a *AppService) SetServiceMonitor(sm *monitorv1.ServiceMonitor) { for i, s := range a.serviceMonitor { @@ -965,6 +998,16 @@ func (a *AppService) GetDelHPAs() []*autoscalingv2.HorizontalPodAutoscaler { return a.delHPAs } +// GetHPABeta2s - +func (a *AppService) GetHPABeta2s() []*autoscalingv2beta2.HorizontalPodAutoscaler { + return a.hpasbeta2s +} + +// GetDelHPABeta2s - +func (a *AppService) GetDelHPABeta2s() []*autoscalingv2beta2.HorizontalPodAutoscaler { + return a.delHPAbeta2s +} + // DelHPA - func (a *AppService) DelHPA(hpa *autoscalingv2.HorizontalPodAutoscaler) { if len(a.hpas) == 0 { @@ -978,6 +1021,19 @@ func (a *AppService) DelHPA(hpa *autoscalingv2.HorizontalPodAutoscaler) { } } +// DelHPABeta2 - +func (a *AppService) DelHPABeta2(hpa *autoscalingv2beta2.HorizontalPodAutoscaler) { + if len(a.hpasbeta2s) == 0 { + return + } + for i, old := range a.hpasbeta2s { + if old.GetName() == hpa.GetName() { + a.hpasbeta2s = append(a.hpasbeta2s[0:i], a.hpasbeta2s[i+1:]...) + return + } + } +} + // SetStorageClass set storageclass func (a *AppService) SetStorageClass(sc *storagev1.StorageClass) { if len(a.storageClasses) > 0 { diff --git a/worker/gc/gc.go b/worker/gc/gc.go index 24c3ea4059..87a7a70e5c 100644 --- a/worker/gc/gc.go +++ b/worker/gc/gc.go @@ -143,6 +143,9 @@ func (g *GarbageCollector) DelKubernetesObjects(serviceGCReq model.ServiceGCTask if err := g.clientset.AutoscalingV2beta2().HorizontalPodAutoscalers(namespace).DeleteCollection(context.Background(), deleteOpts, listOpts); err != nil { logrus.Warningf("[DelKubernetesObjects] delete hpas(%s): %v", serviceGCReq.ServiceID, err) } + if err := g.clientset.AutoscalingV2().HorizontalPodAutoscalers(namespace).DeleteCollection(context.Background(), deleteOpts, listOpts); err != nil { + logrus.Warningf("[DelKubernetesObjects] delete hpas(%s): %v", serviceGCReq.ServiceID, err) + } // kubernetes does not support api for deleting collection of service // read: https://github.com/kubernetes/kubernetes/issues/68468#issuecomment-419981870 serviceList, err := g.clientset.CoreV1().Services(namespace).List(context.Background(), listOpts)