perf: API support for k8s version 1.27 (#1711)

Co-authored-by: 曲源成 <[email protected]>

api/handler/nodes.go

@@ -4,15 +4,18 @@ import (
  "context"
  "fmt"
  "github.com/goodrain/rainbond/api/client/prometheus"
+ k8sutil "github.com/goodrain/rainbond/util/k8s"
  "github.com/pquerna/ffjson/ffjson"
  "github.com/shirou/gopsutil/disk"
  "github.com/sirupsen/logrus"
  v1 "k8s.io/api/core/v1"
- "k8s.io/api/policy/v1beta1"
+ policyv1 "k8s.io/api/policy/v1"
+ policyv1beta1 "k8s.io/api/policy/v1beta1"
  "k8s.io/apimachinery/pkg/api/meta"
  metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
  "k8s.io/apimachinery/pkg/fields"
  "k8s.io/apimachinery/pkg/types"
+ utilversion "k8s.io/apimachinery/pkg/util/version"
  "k8s.io/client-go/kubernetes"
  "k8s.io/client-go/rest"
  "runtime"
@@ -336,7 +339,22 @@ func (n *nodesHandle) GetNodeScheduler(ctx context.Context, nodeName string) (st
 //evictPod -
 func (n *nodesHandle) evictPod(pod v1.Pod, policyGroupVersion string) error {
  deleteOptions := &metav1.DeleteOptions{}
- eviction := &v1beta1.Eviction{
+ if k8sutil.GetKubeVersion().AtLeast(utilversion.MustParseSemantic("v1.21.0")) {
+ eviction := &policyv1.Eviction{
+ TypeMeta: metav1.TypeMeta{
+ APIVersion: policyGroupVersion,
+ Kind: EvictionKind,
+ },
+ ObjectMeta: metav1.ObjectMeta{
+ Name: pod.Name,
+ Namespace: pod.Namespace,
+ },
+ DeleteOptions: deleteOptions,
+ }
+ // Remember to change change the URL manipulation func when Evction's version change
+ return n.clientset.PolicyV1().Evictions(eviction.Namespace).Evict(context.Background(), eviction)
+ }
+ eviction := &policyv1beta1.Eviction{
  TypeMeta: metav1.TypeMeta{
  APIVersion: policyGroupVersion,
  Kind: EvictionKind,

node/kubecache/kube_cache.go

@@ -21,6 +21,8 @@ package kubecache
 import (
  "context"
  "fmt"
+ k8sutil "github.com/goodrain/rainbond/util/k8s"
+ utilversion "k8s.io/apimachinery/pkg/util/version"
  "math"
  "strings"
  "time"
@@ -35,7 +37,8 @@ import (
 
  "github.com/sirupsen/logrus"
  v1 "k8s.io/api/core/v1"
- "k8s.io/api/policy/v1beta1"
+ policyv1 "k8s.io/api/policy/v1"
+ policyv1beta1 "k8s.io/api/policy/v1beta1"
  apierrors "k8s.io/apimachinery/pkg/api/errors"
  "k8s.io/apimachinery/pkg/api/resource"
  metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -195,7 +198,22 @@ func (k *kubeClient) GetPodsByNodes(nodeName string) (pods []v1.Pod, err error)
 //evictPod 驱离POD
 func (k *kubeClient) evictPod(pod v1.Pod, policyGroupVersion string) error {
  deleteOptions := &metav1.DeleteOptions{}
- eviction := &v1beta1.Eviction{
+ if k8sutil.GetKubeVersion().AtLeast(utilversion.MustParseSemantic("v1.21.0")) {
+ eviction := &policyv1.Eviction{
+ TypeMeta: metav1.TypeMeta{
+ APIVersion: policyGroupVersion,
+ Kind: EvictionKind,
+ },
+ ObjectMeta: metav1.ObjectMeta{
+ Name: pod.Name,
+ Namespace: pod.Namespace,
+ },
+ DeleteOptions: deleteOptions,
+ }
+ // Remember to change change the URL manipulation func when Evction's version change
+ return k.kubeclient.PolicyV1().Evictions(eviction.Namespace).Evict(context.Background(), eviction)
+ }
+ eviction := &policyv1beta1.Eviction{
  TypeMeta: metav1.TypeMeta{
  APIVersion: policyGroupVersion,
  Kind: EvictionKind,

worker/appm/controller/export.go

@@ -8,7 +8,8 @@ import (
  v1 "github.com/goodrain/rainbond/worker/appm/types/v1"
  "github.com/sirupsen/logrus"
  appv1 "k8s.io/api/apps/v1"
- "k8s.io/api/autoscaling/v2beta2"
+ autoscalv2beta2 "k8s.io/api/autoscaling/v2beta2"
+ autoscalv2 "k8s.io/api/autoscaling/v2"
  batchv1 "k8s.io/api/batch/v1"
  "k8s.io/api/batch/v1beta1"
  corev1 "k8s.io/api/core/v1"
@@ -356,7 +357,25 @@ func (s *exportController) exportOne(app v1.AppService, r *RainbondExport) error
  hpa.Kind = "HorizontalPodAutoscaler"
  hpa.Namespace = ""
  hpa.APIVersion = APIVersionHorizontalPodAutoscaler
- hpa.Status = v2beta2.HorizontalPodAutoscalerStatus{}
+ hpa.Status = autoscalv2.HorizontalPodAutoscalerStatus{}
+ if len(hpa.ResourceVersion) == 0 {
+ hpaBytes, err := yaml.Marshal(hpa)
+ if err != nil {
+ return fmt.Errorf("hpa to yaml failure %v", err)
+ }
+ err = s.write(path.Join(exportTemplatePath, "HorizontalPodAutoscaler.yaml"), hpaBytes, "\n---\n")
+ if err != nil {
+ return fmt.Errorf("write hpa yaml failure %v", err)
+ }
+ }
+ }
+ }
+ if hpas := app.GetHPABeta2s(); len(hpas) != 0 {
+ for _, hpa := range hpas {
+ hpa.Kind = "HorizontalPodAutoscaler"
+ hpa.Namespace = ""
+ hpa.APIVersion = APIVersionHorizontalPodAutoscaler
+ hpa.Status = autoscalv2beta2.HorizontalPodAutoscalerStatus{}
  if len(hpa.ResourceVersion) == 0 {
  hpaBytes, err := yaml.Marshal(hpa)
  if err != nil {

worker/appm/controller/refresh_xpa.go

@@ -20,6 +20,8 @@ package controller
 
 import (
  "context"
+ k8sutil "github.com/goodrain/rainbond/util/k8s"
+ utilversion "k8s.io/apimachinery/pkg/util/version"
  "sync"
 
  "github.com/sirupsen/logrus"
@@ -55,16 +57,31 @@ func (a *refreshXPAController) Begin() {
 }
 
 func (a *refreshXPAController) applyOne(clientset kubernetes.Interface, app *v1.AppService) error {
- for _, hpa := range app.GetHPAs() {
- f.EnsureHPA(hpa, clientset)
- }
+ if k8sutil.GetKubeVersion().AtLeast(utilversion.MustParseSemantic("v1.23.0")) {
+ for _, hpa := range app.GetHPAs() {
+ f.EnsureHPA(hpa, clientset)
+ }
 
- for _, hpa := range app.GetDelHPAs() {
- logrus.Debugf("hpa name: %s; start deleting hpa.", hpa.GetName())
- err := clientset.AutoscalingV2beta2().HorizontalPodAutoscalers(hpa.GetNamespace()).Delete(context.Background(), hpa.GetName(), metav1.DeleteOptions{})
- if err != nil {
- // don't return error, hope it is ok next time
- logrus.Warningf("error deleting secret(%#v): %v", hpa, err)
+ for _, hpa := range app.GetDelHPAs() {
+ logrus.Debugf("hpa name: %s; start deleting hpa.", hpa.GetName())
+ err := clientset.AutoscalingV2().HorizontalPodAutoscalers(hpa.GetNamespace()).Delete(context.Background(), hpa.GetName(), metav1.DeleteOptions{})
+ if err != nil {
+ // don't return error, hope it is ok next time
+ logrus.Warningf("error deleting secret(%#v): %v", hpa, err)
+ }
+ }
+ } else {
+ for _, hpa := range app.GetHPABeta2s() {
+ f.EnsureHPABetav2(hpa, clientset)
+ }
+
+ for _, hpa := range app.GetDelHPABeta2s() {
+ logrus.Debugf("hpa name: %s; start deleting hpa.", hpa.GetName())
+ err := clientset.AutoscalingV2beta2().HorizontalPodAutoscalers(hpa.GetNamespace()).Delete(context.Background(), hpa.GetName(), metav1.DeleteOptions{})
+ if err != nil {
+ // don't return error, hope it is ok next time
+ logrus.Warningf("error deleting secret(%#v): %v", hpa, err)
+ }
  }
  }
 

worker/appm/controller/start.go

@@ -21,6 +21,8 @@ package controller
 import (
  "context"
  "fmt"
+ k8sutil "github.com/goodrain/rainbond/util/k8s"
+ utilversion "k8s.io/apimachinery/pkg/util/version"
  "sync"
  "time"
 
@@ -198,13 +200,27 @@ func (s *startController) startOne(app v1.AppService) error {
  }
 
  //step 6: create hpa
- if hpas := app.GetHPAs(); len(hpas) != 0 {
- for _, hpa := range hpas {
- if len(hpa.ResourceVersion) == 0 {
- _, err := s.manager.client.AutoscalingV2beta2().HorizontalPodAutoscalers(hpa.GetNamespace()).Create(s.ctx, hpa, metav1.CreateOptions{})
- if err != nil && !errors.IsAlreadyExists(err) {
- logrus.Debugf("hpa: %#v", hpa)
- return fmt.Errorf("create hpa: %v", err)
+ if k8sutil.GetKubeVersion().AtLeast(utilversion.MustParseSemantic("v1.23.0")) {
+ if hpas := app.GetHPAs(); len(hpas) != 0 {
+ for _, hpa := range hpas {
+ if len(hpa.ResourceVersion) == 0 {
+ _, err := s.manager.client.AutoscalingV2().HorizontalPodAutoscalers(hpa.GetNamespace()).Create(s.ctx, hpa, metav1.CreateOptions{})
+ if err != nil && !errors.IsAlreadyExists(err) {
+ logrus.Debugf("hpa: %#v", hpa)
+ return fmt.Errorf("create hpa: %v", err)
+ }
+ }
+ }
+ }
+ } else {
+ if hpas := app.GetHPABeta2s(); len(hpas) != 0 {
+ for _, hpa := range hpas {
+ if len(hpa.ResourceVersion) == 0 {
+ _, err := s.manager.client.AutoscalingV2beta2().HorizontalPodAutoscalers(hpa.GetNamespace()).Create(s.ctx, hpa, metav1.CreateOptions{})
+ if err != nil && !errors.IsAlreadyExists(err) {
+ logrus.Debugf("hpa: %#v", hpa)
+ return fmt.Errorf("create hpa: %v", err)
+ }
  }
  }
  }

worker/appm/controller/stop.go

@@ -22,7 +22,9 @@ import (
  "context"
  "fmt"
  "github.com/goodrain/rainbond/db"
+ k8sutil "github.com/goodrain/rainbond/util/k8s"
  "github.com/jinzhu/gorm"
+ utilversion "k8s.io/apimachinery/pkg/util/version"
  "sync"
  "time"
 
@@ -206,11 +208,22 @@ func (s *stopController) stopOne(app v1.AppService) error {
  }
  }
  //step 7: deleta all hpa
- if hpas := app.GetHPAs(); len(hpas) != 0 {
- for _, hpa := range hpas {
- err := s.manager.client.AutoscalingV2beta2().HorizontalPodAutoscalers(hpa.GetNamespace()).Delete(s.ctx, hpa.GetName(), metav1.DeleteOptions{})
- if err != nil && !errors.IsNotFound(err) {
- return fmt.Errorf("delete hpa: %v", err)
+ if k8sutil.GetKubeVersion().AtLeast(utilversion.MustParseSemantic("v1.23.0")) {
+ if hpas := app.GetHPAs(); len(hpas) != 0 {
+ for _, hpa := range hpas {
+ err := s.manager.client.AutoscalingV2().HorizontalPodAutoscalers(hpa.GetNamespace()).Delete(s.ctx, hpa.GetName(), metav1.DeleteOptions{})
+ if err != nil && !errors.IsNotFound(err) {
+ return fmt.Errorf("delete hpa: %v", err)
+ }
+ }
+ }
+ } else {
+ if hpas := app.GetHPABeta2s(); len(hpas) != 0 {
+ for _, hpa := range hpas {
+ err := s.manager.client.AutoscalingV2beta2().HorizontalPodAutoscalers(hpa.GetNamespace()).Delete(s.ctx, hpa.GetName(), metav1.DeleteOptions{})
+ if err != nil && !errors.IsNotFound(err) {
+ return fmt.Errorf("delete hpa: %v", err)
+ }
  }
  }
  }