From 05ebc64fc454c331fec7aa49d2a044816954dcee Mon Sep 17 00:00:00 2001 From: "jicheng.sk" Date: Mon, 18 Mar 2024 14:30:06 +0800 Subject: [PATCH] add enhanced livenessprobe controller Signed-off-by: jicheng.sk --- pkg/controller/controllers.go | 9 +- .../livenessprobemapnodeprobe.go | 139 ++ .../livenessprobemapnodeprobe_test.go | 922 ++++++++++ .../probemapnodeprobe_pod_event_handler.go | 64 + .../probemapnodeprobe_utils.go | 223 +++ .../probemapnodeprobe_utils_test.go | 1624 +++++++++++++++++ .../nodepodprobe/node_pod_probe_controller.go | 17 +- pkg/util/livenessprobe/livenessprobe_utils.go | 10 + .../mutating/enhancedlivenessprobe_handler.go | 10 +- test/e2e/apps/imagelistpulljobs.go | 2 +- test/e2e/apps/livenessprobemapnodeprobe.go | 89 + test/e2e/apps/pullimages.go | 4 +- .../livenessprobemapnodeprobe_utils.go | 98 + 13 files changed, 3190 insertions(+), 21 deletions(-) create mode 100644 pkg/controller/livenessprobemapnodeprobe/livenessprobemapnodeprobe.go create mode 100644 pkg/controller/livenessprobemapnodeprobe/livenessprobemapnodeprobe_test.go create mode 100644 pkg/controller/livenessprobemapnodeprobe/probemapnodeprobe_pod_event_handler.go create mode 100644 pkg/controller/livenessprobemapnodeprobe/probemapnodeprobe_utils.go create mode 100644 pkg/controller/livenessprobemapnodeprobe/probemapnodeprobe_utils_test.go create mode 100644 pkg/util/livenessprobe/livenessprobe_utils.go create mode 100644 test/e2e/apps/livenessprobemapnodeprobe.go create mode 100644 test/e2e/framework/livenessprobemapnodeprobe_utils.go diff --git a/pkg/controller/controllers.go b/pkg/controller/controllers.go index bef3d95ccd..1959ff8bbe 100644 --- a/pkg/controller/controllers.go +++ b/pkg/controller/controllers.go @@ -17,6 +17,10 @@ limitations under the License. package controller import ( + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/manager" + "github.com/openkruise/kruise/pkg/controller/advancedcronjob" "github.com/openkruise/kruise/pkg/controller/broadcastjob" "github.com/openkruise/kruise/pkg/controller/cloneset" @@ -26,6 +30,7 @@ import ( "github.com/openkruise/kruise/pkg/controller/ephemeraljob" "github.com/openkruise/kruise/pkg/controller/imagelistpulljob" "github.com/openkruise/kruise/pkg/controller/imagepulljob" + enhancedlivenessprobe2nodeprobe "github.com/openkruise/kruise/pkg/controller/livenessprobemapnodeprobe" "github.com/openkruise/kruise/pkg/controller/nodeimage" "github.com/openkruise/kruise/pkg/controller/nodepodprobe" "github.com/openkruise/kruise/pkg/controller/persistentpodstate" @@ -38,9 +43,6 @@ import ( "github.com/openkruise/kruise/pkg/controller/statefulset" "github.com/openkruise/kruise/pkg/controller/uniteddeployment" "github.com/openkruise/kruise/pkg/controller/workloadspread" - "k8s.io/apimachinery/pkg/api/meta" - "k8s.io/klog/v2" - "sigs.k8s.io/controller-runtime/pkg/manager" ) var controllerAddFuncs []func(manager.Manager) error @@ -67,6 +69,7 @@ func init() { controllerAddFuncs = append(controllerAddFuncs, podprobemarker.Add) controllerAddFuncs = append(controllerAddFuncs, nodepodprobe.Add) controllerAddFuncs = append(controllerAddFuncs, imagelistpulljob.Add) + controllerAddFuncs = append(controllerAddFuncs, enhancedlivenessprobe2nodeprobe.Add) } func SetupWithManager(m manager.Manager) error { diff --git a/pkg/controller/livenessprobemapnodeprobe/livenessprobemapnodeprobe.go b/pkg/controller/livenessprobemapnodeprobe/livenessprobemapnodeprobe.go new file mode 100644 index 0000000000..cbe4a91d4a --- /dev/null +++ b/pkg/controller/livenessprobemapnodeprobe/livenessprobemapnodeprobe.go @@ -0,0 +1,139 @@ +package enhancedlivenessprobe2nodeprobe + +import ( + "context" + "fmt" + "time" + + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/record" + "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" + + "github.com/openkruise/kruise/pkg/util" + utilclient "github.com/openkruise/kruise/pkg/util/client" +) + +const ( + concurrentReconciles = 10 + controllerName = "livenessprobemapnodeprobe-controller" + + FinalizerPodEnhancedLivenessProbe = "pod.kruise.io/enhanced-liveness-probe-cleanup" + + AddNodeProbeConfigOpType = "addNodeProbe" + DelNodeProbeConfigOpType = "delNodeProbe" +) + +func Add(mgr manager.Manager) error { + return add(mgr, newReconciler(mgr)) +} + +var _ reconcile.Reconciler = &ReconcileEnhancedLivenessProbe2NodeProbe{} + +// ReconcileEnhancedLivenessProbe reconciles a Pod object +type ReconcileEnhancedLivenessProbe2NodeProbe struct { + client.Client + recorder record.EventRecorder +} + +// newReconciler returns a new reconcile.Reconciler +func newReconciler(mgr manager.Manager) *ReconcileEnhancedLivenessProbe2NodeProbe { + return &ReconcileEnhancedLivenessProbe2NodeProbe{ + Client: utilclient.NewClientFromManager(mgr, controllerName), + recorder: mgr.GetEventRecorderFor(controllerName), + } +} + +// add adds a new Controller to mgr with r as the reconcile.Reconciler +func add(mgr manager.Manager, r *ReconcileEnhancedLivenessProbe2NodeProbe) error { + // Create a new controller + c, err := controller.New(controllerName, mgr, controller.Options{Reconciler: r, MaxConcurrentReconciles: concurrentReconciles}) + if err != nil { + return err + } + + // watch events of pod + if err = c.Watch(&source.Kind{Type: &v1.Pod{}}, &enqueueRequestForPod{reader: mgr.GetClient()}); err != nil { + return err + } + return nil +} + +// +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch;update;patch +// +kubebuilder:rbac:groups=apps.kruise.io,resources=nodepodprobes,verbs=get;list;watch;create;update;patch:delete +func (r *ReconcileEnhancedLivenessProbe2NodeProbe) Reconcile(_ context.Context, request reconcile.Request) (res reconcile.Result, err error) { + start := time.Now() + klog.V(3).Infof("Starting to process Pod %v", request.NamespacedName) + defer func() { + if err != nil { + klog.Warningf("Failed to process Pod %v, elapsedTime %v, error: %v", request.NamespacedName, time.Since(start), err) + } else { + klog.Infof("Finish to process Pod %v, elapsedTime %v", request.NamespacedName, time.Since(start)) + } + }() + + err = r.syncPodContainersLivenessProbe(request.Namespace, request.Name) + if err != nil { + return reconcile.Result{}, err + } + return reconcile.Result{}, nil +} + +func (r *ReconcileEnhancedLivenessProbe2NodeProbe) syncPodContainersLivenessProbe(namespace, name string) error { + getPod := &v1.Pod{} + var err error + if err = r.Get(context.TODO(), types.NamespacedName{Namespace: namespace, Name: name}, getPod); err != nil { + if errors.IsNotFound(err) { + return nil + } + return fmt.Errorf("failed to get pod %s/%s: %v", namespace, name, err) + } + + if getPod.DeletionTimestamp.IsZero() { + if !controllerutil.ContainsFinalizer(getPod, FinalizerPodEnhancedLivenessProbe) { + err = util.UpdateFinalizer(r.Client, getPod, util.AddFinalizerOpType, FinalizerPodEnhancedLivenessProbe) + if err != nil { + klog.Errorf("Failed to update pod %s/%s finalizer %v, err: %v", getPod.Namespace, getPod.Name, FinalizerPodEnhancedLivenessProbe, err) + return err + } + } + err = r.addOrRemoveNodePodProbeConfig(getPod, AddNodeProbeConfigOpType) + if err != nil { + klog.Errorf("Failed to add or remove node pod probe config in %s process for pod: %s/%s, err: %v", AddNodeProbeConfigOpType, getPod.Namespace, getPod.Name, err) + return err + } + } else { + // pod in deleting process + err = r.addOrRemoveNodePodProbeConfig(getPod, DelNodeProbeConfigOpType) + if err != nil { + klog.Errorf("Failed to add or remove node pod probe config in %s process for pod: %s/%s, "+ + "err: %v", DelNodeProbeConfigOpType, getPod.Namespace, getPod.Name, err) + return err + } + if controllerutil.ContainsFinalizer(getPod, FinalizerPodEnhancedLivenessProbe) { + err = util.UpdateFinalizer(r.Client, getPod, util.RemoveFinalizerOpType, FinalizerPodEnhancedLivenessProbe) + if err != nil { + klog.Errorf("Failed to update pod %s/%s finalizer %v, err: %v", getPod.Namespace, getPod.Name, FinalizerPodEnhancedLivenessProbe, err) + return err + } + } + } + return nil +} + +func (r *ReconcileEnhancedLivenessProbe2NodeProbe) addOrRemoveNodePodProbeConfig(pod *v1.Pod, op string) error { + if op == DelNodeProbeConfigOpType { + return r.delNodeProbeConfig(pod) + } + if op == AddNodeProbeConfigOpType { + return r.addNodeProbeConfig(pod) + } + return fmt.Errorf("No found op %s(just support %s and %s)", op, AddNodeProbeConfigOpType, DelNodeProbeConfigOpType) +} diff --git a/pkg/controller/livenessprobemapnodeprobe/livenessprobemapnodeprobe_test.go b/pkg/controller/livenessprobemapnodeprobe/livenessprobemapnodeprobe_test.go new file mode 100644 index 0000000000..396418c9e9 --- /dev/null +++ b/pkg/controller/livenessprobemapnodeprobe/livenessprobemapnodeprobe_test.go @@ -0,0 +1,922 @@ +package enhancedlivenessprobe2nodeprobe + +import ( + "context" + "fmt" + "reflect" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" + appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" + + "github.com/openkruise/kruise/pkg/util" +) + +var ( + scheme *runtime.Scheme +) + +func init() { + scheme = runtime.NewScheme() + utilruntime.Must(corev1.AddToScheme(scheme)) + utilruntime.Must(appsv1alpha1.AddToScheme(scheme)) +} + +func TestAddOrRemoveNodePodProbeConfig(t *testing.T) { + testCase := []struct { + name string + pod *corev1.Pod + op string + podNodeProbe *appsv1alpha1.NodePodProbe + expectNodeProbe *appsv1alpha1.NodePodProbe + }{ + { + name: "test: add node probe config", + pod: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + Namespace: "sp1", + Annotations: map[string]string{ + alpha1.AnnotationNativeContainerProbeContext: `[{"name":"c1","livenessProbe":{"httpGet":{"path":"/health","port":7001},"initialDelaySeconds":1000,"timeoutSeconds":5,"periodSeconds":100,"successThreshold":2,"failureThreshold":3}},{"name":"c2","livenessProbe":{"tcpSocket":{"port":7001},"initialDelaySeconds":1000,"timeoutSeconds":5,"periodSeconds":100,"successThreshold":2,"failureThreshold":3}}]`, + }, + UID: types.UID("111-222"), + }, + Spec: corev1.PodSpec{ + NodeName: "node1", + }, + }, + op: AddNodeProbeConfigOpType, + podNodeProbe: &appsv1alpha1.NodePodProbe{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + ResourceVersion: "111", + }, + Spec: appsv1alpha1.NodePodProbeSpec{ + PodProbes: []appsv1alpha1.PodProbe{ + { + Name: "pod2", + Namespace: "sp2", + UID: fmt.Sprintf("%s", "222-111"), + Probes: []appsv1alpha1.ContainerProbe{ + { + Name: "pod2-c2", + ContainerName: "c2", + Probe: appsv1alpha1.ContainerProbeSpec{ + Probe: corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + Exec: &corev1.ExecAction{ + Command: []string{"/bin/sh", "-c", "/healthy.sh"}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + expectNodeProbe: &appsv1alpha1.NodePodProbe{ + TypeMeta: metav1.TypeMeta{ + Kind: "NodePodProbe", + APIVersion: "apps.kruise.io/v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + ResourceVersion: "112", + }, + Spec: appsv1alpha1.NodePodProbeSpec{ + PodProbes: []appsv1alpha1.PodProbe{ + { + Name: "pod2", + Namespace: "sp2", + UID: fmt.Sprintf("%s", "222-111"), + Probes: []appsv1alpha1.ContainerProbe{ + { + Name: "pod2-c2", + ContainerName: "c2", + Probe: appsv1alpha1.ContainerProbeSpec{ + Probe: corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + Exec: &corev1.ExecAction{ + Command: []string{"/bin/sh", "-c", "/healthy.sh"}, + }, + }, + }, + }, + }, + }, + }, + { + Name: "pod1", + Namespace: "sp1", + UID: fmt.Sprintf("%s", "111-222"), + Probes: []appsv1alpha1.ContainerProbe{ + { + Name: "pod1-c1", + ContainerName: "c1", + Probe: appsv1alpha1.ContainerProbeSpec{ + Probe: corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/health", + Port: intstr.IntOrString{Type: intstr.Int, IntVal: 7001}, + }, + }, + InitialDelaySeconds: 1000, + TimeoutSeconds: 5, + PeriodSeconds: 100, + SuccessThreshold: 2, + FailureThreshold: 3, + }, + }, + }, + { + Name: "pod1-c2", + ContainerName: "c2", + Probe: appsv1alpha1.ContainerProbeSpec{ + Probe: corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + TCPSocket: &corev1.TCPSocketAction{ + Port: intstr.IntOrString{Type: intstr.Int, IntVal: 7001}, + }, + }, + InitialDelaySeconds: 1000, + TimeoutSeconds: 5, + PeriodSeconds: 100, + SuccessThreshold: 2, + FailureThreshold: 3, + }, + }, + }, + }, + }, + }, + }, + }, + }, + { + name: "test: remove node probe config", + pod: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + Namespace: "sp1", + Annotations: map[string]string{ + alpha1.AnnotationNativeContainerProbeContext: `[{"name":"c1","livenessProbe":{"httpGet":{"path":"/health","port":7001},"initialDelaySeconds":1000,"timeoutSeconds":5,"periodSeconds":100,"successThreshold":2,"failureThreshold":3}},{"name":"c2","livenessProbe":{"tcpSocket":{"port":7001},"initialDelaySeconds":1000,"timeoutSeconds":5,"periodSeconds":100,"successThreshold":2,"failureThreshold":3}}]`, + }, + UID: types.UID("111-222"), + }, + Spec: corev1.PodSpec{ + NodeName: "node1", + }, + }, + op: DelNodeProbeConfigOpType, + podNodeProbe: &appsv1alpha1.NodePodProbe{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + ResourceVersion: "111", + }, + Spec: appsv1alpha1.NodePodProbeSpec{ + PodProbes: []appsv1alpha1.PodProbe{ + { + Name: "pod2", + Namespace: "sp2", + UID: fmt.Sprintf("%s", "222-111"), + Probes: []appsv1alpha1.ContainerProbe{ + { + Name: "pod2-c2", + ContainerName: "c2", + Probe: appsv1alpha1.ContainerProbeSpec{ + Probe: corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + Exec: &corev1.ExecAction{ + Command: []string{"/bin/sh", "-c", "/healthy.sh"}, + }, + }, + }, + }, + }, + }, + }, + { + Name: "pod1", + Namespace: "sp1", + UID: fmt.Sprintf("%s", "111-222"), + Probes: []appsv1alpha1.ContainerProbe{ + { + Name: "pod1-c1", + ContainerName: "c1", + Probe: appsv1alpha1.ContainerProbeSpec{ + Probe: corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/health", + Port: intstr.IntOrString{Type: intstr.Int, IntVal: 7001}, + }, + }, + InitialDelaySeconds: 1000, + TimeoutSeconds: 5, + PeriodSeconds: 100, + SuccessThreshold: 2, + FailureThreshold: 3, + }, + }, + }, + { + Name: "pod1-c2", + ContainerName: "c2", + Probe: appsv1alpha1.ContainerProbeSpec{ + Probe: corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + TCPSocket: &corev1.TCPSocketAction{ + Port: intstr.IntOrString{Type: intstr.Int, IntVal: 7001}, + }, + }, + InitialDelaySeconds: 1000, + TimeoutSeconds: 5, + PeriodSeconds: 100, + SuccessThreshold: 2, + FailureThreshold: 3, + }, + }, + }, + }, + }, + }, + }, + }, + expectNodeProbe: &appsv1alpha1.NodePodProbe{ + TypeMeta: metav1.TypeMeta{ + Kind: "NodePodProbe", + APIVersion: "apps.kruise.io/v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + ResourceVersion: "112", + }, + Spec: appsv1alpha1.NodePodProbeSpec{ + PodProbes: []appsv1alpha1.PodProbe{ + { + Name: "pod2", + Namespace: "sp2", + UID: fmt.Sprintf("%s", "222-111"), + Probes: []appsv1alpha1.ContainerProbe{ + { + Name: "pod2-c2", + ContainerName: "c2", + Probe: appsv1alpha1.ContainerProbeSpec{ + Probe: corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + Exec: &corev1.ExecAction{ + Command: []string{"/bin/sh", "-c", "/healthy.sh"}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + { + name: "test: add node probe config(change different config)", + pod: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + Namespace: "sp1", + Annotations: map[string]string{ + alpha1.AnnotationNativeContainerProbeContext: `[{"name":"c1","livenessProbe":{"httpGet":{"path":"/health","port":7001},"initialDelaySeconds":1000,"timeoutSeconds":5,"periodSeconds":100,"successThreshold":2,"failureThreshold":3}}]`, + }, + UID: types.UID("111-222"), + }, + Spec: corev1.PodSpec{ + NodeName: "node1", + }, + }, + op: AddNodeProbeConfigOpType, + podNodeProbe: &appsv1alpha1.NodePodProbe{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + ResourceVersion: "111", + }, + Spec: appsv1alpha1.NodePodProbeSpec{ + PodProbes: []appsv1alpha1.PodProbe{ + { + Name: "pod2", + Namespace: "sp2", + UID: fmt.Sprintf("%s", "222-111"), + Probes: []appsv1alpha1.ContainerProbe{ + { + Name: "pod2-c2", + ContainerName: "c2", + Probe: appsv1alpha1.ContainerProbeSpec{ + Probe: corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + Exec: &corev1.ExecAction{ + Command: []string{"/bin/sh", "-c", "/healthy.sh"}, + }, + }, + }, + }, + }, + }, + }, + { + Name: "pod1", + Namespace: "sp1", + UID: fmt.Sprintf("%s", "111-222"), + Probes: []appsv1alpha1.ContainerProbe{ + { + Name: "pod1-c1", + ContainerName: "c1", + Probe: appsv1alpha1.ContainerProbeSpec{ + Probe: corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/health", + Port: intstr.IntOrString{Type: intstr.Int, IntVal: 7001}, + }, + }, + InitialDelaySeconds: 1000, + TimeoutSeconds: 5, + PeriodSeconds: 100, + SuccessThreshold: 2, + FailureThreshold: 3, + }, + }, + }, + { + Name: "pod1-c2", + ContainerName: "c2", + Probe: appsv1alpha1.ContainerProbeSpec{ + Probe: corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + TCPSocket: &corev1.TCPSocketAction{ + Port: intstr.IntOrString{Type: intstr.Int, IntVal: 7001}, + }, + }, + InitialDelaySeconds: 1000, + TimeoutSeconds: 5, + PeriodSeconds: 100, + SuccessThreshold: 2, + FailureThreshold: 3, + }, + }, + }, + }, + }, + }, + }, + }, + expectNodeProbe: &appsv1alpha1.NodePodProbe{ + TypeMeta: metav1.TypeMeta{ + Kind: "NodePodProbe", + APIVersion: "apps.kruise.io/v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + ResourceVersion: "112", + }, + Spec: appsv1alpha1.NodePodProbeSpec{ + PodProbes: []appsv1alpha1.PodProbe{ + { + Name: "pod2", + Namespace: "sp2", + UID: fmt.Sprintf("%s", "222-111"), + Probes: []appsv1alpha1.ContainerProbe{ + { + Name: "pod2-c2", + ContainerName: "c2", + Probe: appsv1alpha1.ContainerProbeSpec{ + Probe: corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + Exec: &corev1.ExecAction{ + Command: []string{"/bin/sh", "-c", "/healthy.sh"}, + }, + }, + }, + }, + }, + }, + }, + { + Name: "pod1", + Namespace: "sp1", + UID: fmt.Sprintf("%s", "111-222"), + Probes: []appsv1alpha1.ContainerProbe{ + { + Name: "pod1-c1", + ContainerName: "c1", + Probe: appsv1alpha1.ContainerProbeSpec{ + Probe: corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/health", + Port: intstr.IntOrString{Type: intstr.Int, IntVal: 7001}, + }, + }, + InitialDelaySeconds: 1000, + TimeoutSeconds: 5, + PeriodSeconds: 100, + SuccessThreshold: 2, + FailureThreshold: 3, + }, + }, + }, + }, + }, + }, + }, + }, + }, + { + name: "test: add node probe config(no found config)", + pod: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + Namespace: "sp1", + UID: types.UID("111-222"), + }, + Spec: corev1.PodSpec{ + NodeName: "node1", + }, + }, + op: AddNodeProbeConfigOpType, + podNodeProbe: &appsv1alpha1.NodePodProbe{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + ResourceVersion: "111", + }, + Spec: appsv1alpha1.NodePodProbeSpec{ + PodProbes: []appsv1alpha1.PodProbe{ + { + Name: "pod2", + Namespace: "sp2", + UID: fmt.Sprintf("%s", "222-111"), + Probes: []appsv1alpha1.ContainerProbe{ + { + Name: "pod2-c2", + ContainerName: "c2", + Probe: appsv1alpha1.ContainerProbeSpec{ + Probe: corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + Exec: &corev1.ExecAction{ + Command: []string{"/bin/sh", "-c", "/healthy.sh"}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + expectNodeProbe: &appsv1alpha1.NodePodProbe{ + TypeMeta: metav1.TypeMeta{ + Kind: "NodePodProbe", + APIVersion: "apps.kruise.io/v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + ResourceVersion: "112", + }, + Spec: appsv1alpha1.NodePodProbeSpec{ + PodProbes: []appsv1alpha1.PodProbe{ + { + Name: "pod2", + Namespace: "sp2", + UID: fmt.Sprintf("%s", "222-111"), + Probes: []appsv1alpha1.ContainerProbe{ + { + Name: "pod2-c2", + ContainerName: "c2", + Probe: appsv1alpha1.ContainerProbeSpec{ + Probe: corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + Exec: &corev1.ExecAction{ + Command: []string{"/bin/sh", "-c", "/healthy.sh"}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + } + + for _, tc := range testCase { + t.Run(tc.name, func(t *testing.T) { + builder := fake.NewClientBuilder().WithScheme(scheme) + fakeClient := builder.WithObjects(tc.pod).WithObjects(tc.podNodeProbe).Build() + r := ReconcileEnhancedLivenessProbe2NodeProbe{Client: fakeClient} + err := r.addOrRemoveNodePodProbeConfig(tc.pod, tc.op) + if err != nil { + t.Errorf("Test case: %v failed, err: %v", tc.name, err) + } + // get NodeProbe + getNodeProbe := appsv1alpha1.NodePodProbe{} + err = r.Get(context.TODO(), types.NamespacedName{Name: r.GetPodNodeName(tc.pod)}, &getNodeProbe) + if err != nil { + t.Errorf("Failed to get node probe object, err: %v", err) + } + t.Logf("GetNodeProbe: %v", util.DumpJSON(getNodeProbe)) + if !reflect.DeepEqual(util.DumpJSON(tc.expectNodeProbe), util.DumpJSON(getNodeProbe)) { + t.Errorf("Expect: %v, but: %v", util.DumpJSON(tc.expectNodeProbe), util.DumpJSON(getNodeProbe)) + } + + }) + } +} + +func TestSyncPodContainersLivenessProbe(t *testing.T) { + testCase := []struct { + name string + request reconcile.Request + pod *corev1.Pod + podNodeProbe *appsv1alpha1.NodePodProbe + expectPod *corev1.Pod + expectPodNodeProbe *appsv1alpha1.NodePodProbe + }{ + { + name: "test reconcile: add node probe config", + request: reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: "pod1", + Namespace: "sp1", + }, + }, + pod: &corev1.Pod{ + TypeMeta: metav1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + Namespace: "sp1", + Annotations: map[string]string{ + alpha1.AnnotationNativeContainerProbeContext: `[{"name":"c1","livenessProbe":{"httpGet":{"path":"/health","port":7001},"initialDelaySeconds":1000,"timeoutSeconds":5,"periodSeconds":100,"successThreshold":2,"failureThreshold":3}},{"name":"c2","livenessProbe":{"tcpSocket":{"port":7001},"initialDelaySeconds":1000,"timeoutSeconds":5,"periodSeconds":100,"successThreshold":2,"failureThreshold":3}}]`, + }, + UID: types.UID("111-222"), + }, + Spec: corev1.PodSpec{ + NodeName: "node1", + }, + }, + podNodeProbe: &appsv1alpha1.NodePodProbe{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + }, + Spec: appsv1alpha1.NodePodProbeSpec{ + PodProbes: []appsv1alpha1.PodProbe{ + { + Name: "pod2", + Namespace: "sp2", + UID: fmt.Sprintf("%s", "222-111"), + Probes: []appsv1alpha1.ContainerProbe{ + { + Name: "pod2-c2", + ContainerName: "c2", + Probe: appsv1alpha1.ContainerProbeSpec{ + Probe: corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + Exec: &corev1.ExecAction{ + Command: []string{"/bin/sh", "-c", "/healthy.sh"}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + expectPod: &corev1.Pod{ + TypeMeta: metav1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + Namespace: "sp1", + Annotations: map[string]string{ + alpha1.AnnotationNativeContainerProbeContext: `[{"name":"c1","livenessProbe":{"httpGet":{"path":"/health","port":7001},"initialDelaySeconds":1000,"timeoutSeconds":5,"periodSeconds":100,"successThreshold":2,"failureThreshold":3}},{"name":"c2","livenessProbe":{"tcpSocket":{"port":7001},"initialDelaySeconds":1000,"timeoutSeconds":5,"periodSeconds":100,"successThreshold":2,"failureThreshold":3}}]`, + }, + UID: types.UID("111-222"), + Finalizers: []string{ + FinalizerPodEnhancedLivenessProbe, + }, + }, + Spec: corev1.PodSpec{ + NodeName: "node1", + }, + }, + expectPodNodeProbe: &appsv1alpha1.NodePodProbe{ + TypeMeta: metav1.TypeMeta{ + Kind: "NodePodProbe", + APIVersion: "apps.kruise.io/v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + }, + Spec: appsv1alpha1.NodePodProbeSpec{ + PodProbes: []appsv1alpha1.PodProbe{ + { + Name: "pod2", + Namespace: "sp2", + UID: fmt.Sprintf("%s", "222-111"), + Probes: []appsv1alpha1.ContainerProbe{ + { + Name: "pod2-c2", + ContainerName: "c2", + Probe: appsv1alpha1.ContainerProbeSpec{ + Probe: corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + Exec: &corev1.ExecAction{ + Command: []string{"/bin/sh", "-c", "/healthy.sh"}, + }, + }, + }, + }, + }, + }, + }, + { + Name: "pod1", + Namespace: "sp1", + UID: fmt.Sprintf("%s", "111-222"), + Probes: []appsv1alpha1.ContainerProbe{ + { + Name: "pod1-c1", + ContainerName: "c1", + Probe: appsv1alpha1.ContainerProbeSpec{ + Probe: corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/health", + Port: intstr.IntOrString{Type: intstr.Int, IntVal: 7001}, + }, + }, + InitialDelaySeconds: 1000, + TimeoutSeconds: 5, + PeriodSeconds: 100, + SuccessThreshold: 2, + FailureThreshold: 3, + }, + }, + }, + { + Name: "pod1-c2", + ContainerName: "c2", + Probe: appsv1alpha1.ContainerProbeSpec{ + Probe: corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + TCPSocket: &corev1.TCPSocketAction{ + Port: intstr.IntOrString{Type: intstr.Int, IntVal: 7001}, + }, + }, + InitialDelaySeconds: 1000, + TimeoutSeconds: 5, + PeriodSeconds: 100, + SuccessThreshold: 2, + FailureThreshold: 3, + }, + }, + }, + }, + }, + }, + }, + }, + }, + { + name: "test reconcile: remove node probe config", + request: reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: "pod1", + Namespace: "sp1", + }, + }, + pod: &corev1.Pod{ + TypeMeta: metav1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + Namespace: "sp1", + Annotations: map[string]string{ + alpha1.AnnotationNativeContainerProbeContext: `[{"name":"c1","livenessProbe":{"httpGet":{"path":"/health","port":7001},"initialDelaySeconds":1000,"timeoutSeconds":5,"periodSeconds":100,"successThreshold":2,"failureThreshold":3}},{"name":"c2","livenessProbe":{"tcpSocket":{"port":7001},"initialDelaySeconds":1000,"timeoutSeconds":5,"periodSeconds":100,"successThreshold":2,"failureThreshold":3}}]`, + }, + UID: types.UID("111-222"), + DeletionTimestamp: &metav1.Time{Time: time.Now()}, + Finalizers: []string{ + FinalizerPodEnhancedLivenessProbe, + "no-delete-protection", + }, + }, + Spec: corev1.PodSpec{ + NodeName: "node1", + }, + }, + podNodeProbe: &appsv1alpha1.NodePodProbe{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + }, + Spec: appsv1alpha1.NodePodProbeSpec{ + PodProbes: []appsv1alpha1.PodProbe{ + { + Name: "pod2", + Namespace: "sp2", + UID: fmt.Sprintf("%s", "222-111"), + Probes: []appsv1alpha1.ContainerProbe{ + { + Name: "pod2-c2", + ContainerName: "c2", + Probe: appsv1alpha1.ContainerProbeSpec{ + Probe: corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + Exec: &corev1.ExecAction{ + Command: []string{"/bin/sh", "-c", "/healthy.sh"}, + }, + }, + }, + }, + }, + }, + }, + { + Name: "pod1", + Namespace: "sp1", + UID: fmt.Sprintf("%s", "111-222"), + Probes: []appsv1alpha1.ContainerProbe{ + { + Name: "pod1-c1", + ContainerName: "c1", + Probe: appsv1alpha1.ContainerProbeSpec{ + Probe: corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/health", + Port: intstr.IntOrString{Type: intstr.Int, IntVal: 7001}, + }, + }, + InitialDelaySeconds: 1000, + TimeoutSeconds: 5, + PeriodSeconds: 100, + SuccessThreshold: 2, + FailureThreshold: 3, + }, + }, + }, + { + Name: "pod1-c2", + ContainerName: "c2", + Probe: appsv1alpha1.ContainerProbeSpec{ + Probe: corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + TCPSocket: &corev1.TCPSocketAction{ + Port: intstr.IntOrString{Type: intstr.Int, IntVal: 7001}, + }, + }, + InitialDelaySeconds: 1000, + TimeoutSeconds: 5, + PeriodSeconds: 100, + SuccessThreshold: 2, + FailureThreshold: 3, + }, + }, + }, + }, + }, + }, + }, + }, + expectPod: &corev1.Pod{ + TypeMeta: metav1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + Namespace: "sp1", + Annotations: map[string]string{ + alpha1.AnnotationNativeContainerProbeContext: `[{"name":"c1","livenessProbe":{"httpGet":{"path":"/health","port":7001},"initialDelaySeconds":1000,"timeoutSeconds":5,"periodSeconds":100,"successThreshold":2,"failureThreshold":3}},{"name":"c2","livenessProbe":{"tcpSocket":{"port":7001},"initialDelaySeconds":1000,"timeoutSeconds":5,"periodSeconds":100,"successThreshold":2,"failureThreshold":3}}]`, + }, + UID: types.UID("111-222"), + Finalizers: []string{ + "no-delete-protection", + }, + DeletionTimestamp: &metav1.Time{Time: time.Now()}, + }, + Spec: corev1.PodSpec{ + NodeName: "node1", + }, + }, + expectPodNodeProbe: &appsv1alpha1.NodePodProbe{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + }, + Spec: appsv1alpha1.NodePodProbeSpec{ + PodProbes: []appsv1alpha1.PodProbe{ + { + Name: "pod2", + Namespace: "sp2", + UID: fmt.Sprintf("%s", "222-111"), + Probes: []appsv1alpha1.ContainerProbe{ + { + Name: "pod2-c2", + ContainerName: "c2", + Probe: appsv1alpha1.ContainerProbeSpec{ + Probe: corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + Exec: &corev1.ExecAction{ + Command: []string{"/bin/sh", "-c", "/healthy.sh"}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + } + + for _, tc := range testCase { + t.Run(tc.name, func(t *testing.T) { + fakeClient := fake.NewClientBuilder().WithScheme(scheme).Build() + // create pod + err := fakeClient.Create(context.TODO(), tc.pod) + if err != nil { + t.Errorf("Failed to create pod: %s/%s, err: %v", tc.pod.Namespace, tc.pod.Name, err) + } + // create nodeProbe + err = fakeClient.Create(context.TODO(), tc.podNodeProbe) + if err != nil { + t.Errorf("Failed to create pod node probe: %s, err: %v", tc.podNodeProbe.Name, err) + } + recon := ReconcileEnhancedLivenessProbe2NodeProbe{Client: fakeClient} + _, err = recon.Reconcile(context.TODO(), tc.request) + if err != nil { + t.Errorf("Reconcile failed: %s", err.Error()) + } + // get pod and check + getPod := corev1.Pod{} + err = fakeClient.Get(context.TODO(), tc.request.NamespacedName, &getPod) + if err != nil { + t.Errorf("Failed to get pod: %s/%s, err: %v", tc.request.Namespace, tc.request.Name, err) + } + t.Logf("getPod: %v", util.DumpJSON(getPod)) + // check pod + if !checkPodFinalizerEqual(tc.expectPod, &getPod) { + t.Errorf("No match, expect: %v, get: %v", util.DumpJSON(tc.expectPod), util.DumpJSON(getPod)) + } + // get nodeProbe and check + getPodNodeProbe := appsv1alpha1.NodePodProbe{} + err = fakeClient.Get(context.TODO(), types.NamespacedName{Name: recon.GetPodNodeName(tc.pod)}, &getPodNodeProbe) + if err != nil { + t.Errorf("Failed to get pod node probe: %s, err: %v", recon.GetPodNodeName(tc.pod), err) + } + t.Logf("getPodNodeProbe: %v", util.DumpJSON(getPodNodeProbe)) + // check modePodProbe + if !checkNodePodProbeSpecEqual(tc.expectPodNodeProbe, &getPodNodeProbe) { + t.Errorf("No match, expect: %v, get: %v", util.DumpJSON(tc.expectPodNodeProbe), util.DumpJSON(getPodNodeProbe)) + } + }) + } + +} + +func checkNodePodProbeSpecEqual(expectNodePodProbe, getNodePodProbe *appsv1alpha1.NodePodProbe) bool { + if expectNodePodProbe == nil || getNodePodProbe == nil { + return false + } + return reflect.DeepEqual(expectNodePodProbe.Spec, getNodePodProbe.Spec) +} + +func checkPodFinalizerEqual(expectPod, getPod *corev1.Pod) bool { + if expectPod == nil || getPod == nil { + return false + } + return reflect.DeepEqual(expectPod.Finalizers, getPod.Finalizers) +} diff --git a/pkg/controller/livenessprobemapnodeprobe/probemapnodeprobe_pod_event_handler.go b/pkg/controller/livenessprobemapnodeprobe/probemapnodeprobe_pod_event_handler.go new file mode 100644 index 0000000000..5c46f9492c --- /dev/null +++ b/pkg/controller/livenessprobemapnodeprobe/probemapnodeprobe_pod_event_handler.go @@ -0,0 +1,64 @@ +package enhancedlivenessprobe2nodeprobe + +import ( + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/workqueue" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/reconcile" +) + +var _ handler.EventHandler = &enqueueRequestForPod{} + +type enqueueRequestForPod struct { + reader client.Reader +} + +func (p *enqueueRequestForPod) Create(evt event.CreateEvent, q workqueue.RateLimitingInterface) { + obj, ok := evt.Object.(*v1.Pod) + if !ok { + return + } + p.queue(q, obj) + return +} + +func (p *enqueueRequestForPod) Delete(evt event.DeleteEvent, q workqueue.RateLimitingInterface) { + obj, ok := evt.Object.(*v1.Pod) + if !ok { + return + } + p.queue(q, obj) + return +} + +func (p *enqueueRequestForPod) Generic(evt event.GenericEvent, q workqueue.RateLimitingInterface) { + obj, ok := evt.Object.(*v1.Pod) + if !ok { + return + } + p.queue(q, obj) + return +} + +func (p *enqueueRequestForPod) Update(evt event.UpdateEvent, q workqueue.RateLimitingInterface) { + new, ok := evt.ObjectNew.(*v1.Pod) + if !ok { + return + } + p.queue(q, new) + return +} + +func (p *enqueueRequestForPod) queue(q workqueue.RateLimitingInterface, pod *v1.Pod) { + if usingEnhancedLivenessProbe(pod) && getRawEnhancedLivenessProbeConfig(pod) != "" { + q.Add(reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: pod.Name, + Namespace: pod.Namespace, + }, + }) + } +} diff --git a/pkg/controller/livenessprobemapnodeprobe/probemapnodeprobe_utils.go b/pkg/controller/livenessprobemapnodeprobe/probemapnodeprobe_utils.go new file mode 100644 index 0000000000..58625eda5d --- /dev/null +++ b/pkg/controller/livenessprobemapnodeprobe/probemapnodeprobe_utils.go @@ -0,0 +1,223 @@ +package enhancedlivenessprobe2nodeprobe + +import ( + "context" + "encoding/json" + "fmt" + "reflect" + + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/retry" + "k8s.io/klog/v2" + + alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" + appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" + livenessprobeUtils "github.com/openkruise/kruise/pkg/util/livenessprobe" +) + +func (r *ReconcileEnhancedLivenessProbe2NodeProbe) delNodeProbeConfig(pod *v1.Pod) error { + return r.retryOnConflictDelNodeProbeConfig(pod) +} + +func (r *ReconcileEnhancedLivenessProbe2NodeProbe) addNodeProbeConfig(pod *v1.Pod) error { + plivenessProbeConfig, err := parseEnhancedLivenessProbeConfig(pod) + if err != nil { + return err + } + if r.GetPodNodeName(pod) == "" { + klog.Warningf("No found pod node name, pod: %v/%v", pod.Namespace, pod.Name) + return nil + } + + npp := appsv1alpha1.NodePodProbe{} + err = r.Get(context.TODO(), types.NamespacedName{Name: r.GetPodNodeName(pod)}, &npp) + if err == nil { + return r.retryOnConflictAddNodeProbeConfig(pod, plivenessProbeConfig) + } + if !errors.IsNotFound(err) { + klog.Errorf("Failed to get node pod probe for pod: %s/%s, err: %v", pod.Namespace, pod.Name, err) + return err + } + newNppGenerated, err := r.generateNewNodePodProbe(pod) + if err != nil { + klog.Errorf("Failed to generate new node pod probe for pod: %s/%s, err: %v", pod.Namespace, pod.Name, err) + return err + } + if err = r.Create(context.TODO(), &newNppGenerated); err != nil { + klog.Errorf("Failed to create node pod probe for pod: %s/%s, err: %v", pod.Namespace, pod.Name, err) + return err + } + return nil +} + +func (r *ReconcileEnhancedLivenessProbe2NodeProbe) retryOnConflictDelNodeProbeConfig(pod *v1.Pod) error { + nppClone := &appsv1alpha1.NodePodProbe{} + err := retry.RetryOnConflict(retry.DefaultBackoff, func() error { + if err := r.Client.Get(context.TODO(), types.NamespacedName{Name: r.GetPodNodeName(pod)}, nppClone); err != nil { + if errors.IsNotFound(err) { + klog.Warningf("No found getting updated npp %s from client", r.GetPodNodeName(pod)) + return nil + } + klog.Errorf("Error getting updated npp %s from client", r.GetPodNodeName(pod)) + return err + } + + podNewNppClone := nppClone.DeepCopy() + podNewNppCloneSpec := podNewNppClone.Spec.DeepCopy() + + newPodProbeTmp := []appsv1alpha1.PodProbe{} + for index := range podNewNppCloneSpec.PodProbes { + podContainerProbe := podNewNppCloneSpec.PodProbes[index] + if podContainerProbe.Name == pod.Name && podContainerProbe.Namespace == pod.Namespace && + podContainerProbe.UID == fmt.Sprintf("%v", pod.UID) { + continue + } + newPodProbeTmp = append(newPodProbeTmp, podContainerProbe) + } + podNewNppCloneSpec.PodProbes = newPodProbeTmp + + // delete node pod probe + if len(podNewNppCloneSpec.PodProbes) == 0 { + if err := r.Delete(context.TODO(), nppClone); err != nil { + klog.Errorf("Failed to delete node pod probe for pod: %s/%s, err: %v", pod.Namespace, pod.Name, err) + return err + } + return nil + } + if reflect.DeepEqual(podNewNppCloneSpec, nppClone.Spec) { + return nil + } + nppClone.Spec = *podNewNppCloneSpec + return r.Client.Update(context.TODO(), nppClone) + }) + if err != nil { + klog.Errorf("NodePodProbe update NodePodProbe(%s) failed:%s", r.GetPodNodeName(pod), err.Error()) + return err + } + return nil +} + +func (r *ReconcileEnhancedLivenessProbe2NodeProbe) retryOnConflictAddNodeProbeConfig(pod *v1.Pod, + containersLivenessProbeConfig []livenessprobeUtils.ContainerLivenessProbe) error { + nppClone := &appsv1alpha1.NodePodProbe{} + err := retry.RetryOnConflict(retry.DefaultBackoff, func() error { + if err := r.Client.Get(context.TODO(), types.NamespacedName{Name: r.GetPodNodeName(pod)}, nppClone); err != nil { + if errors.IsNotFound(err) { + klog.Warningf("No found getting updated npp %s from client", r.GetPodNodeName(pod)) + return nil + } + klog.Errorf("Error getting updated npp %s from client", r.GetPodNodeName(pod)) + return err + } + + podNewNppClone := nppClone.DeepCopy() + podNewNppCloneSpec := podNewNppClone.Spec.DeepCopy() + + isHit := false + for index := range podNewNppCloneSpec.PodProbes { + podContainerProbe := &podNewNppCloneSpec.PodProbes[index] + if podContainerProbe.Name == pod.Name && podContainerProbe.Namespace == pod.Namespace && + podContainerProbe.UID == fmt.Sprintf("%v", pod.UID) { + isHit = true + // diff the current pod container probes vs the npp container probes + newPodContainerProbes := generatePodContainersProbe(pod, containersLivenessProbeConfig) + podContainerProbe.Probes = newPodContainerProbes + } + } + if !isHit { + if len(podNewNppCloneSpec.PodProbes) == 0 { + podNewNppCloneSpec.PodProbes = []appsv1alpha1.PodProbe{} + } + newPodProbe := generatePodProbeByContainersProbeConfig(pod, containersLivenessProbeConfig) + if len(newPodProbe.Probes) != 0 { + podNewNppCloneSpec.PodProbes = append(podNewNppCloneSpec.PodProbes, newPodProbe) + } + } + if reflect.DeepEqual(podNewNppCloneSpec, nppClone.Spec) { + return nil + } + nppClone.Spec = *podNewNppCloneSpec + return r.Client.Update(context.TODO(), nppClone) + }) + if err != nil { + klog.Errorf("NodePodProbe update NodePodProbe(%s) failed:%s", r.GetPodNodeName(pod), err.Error()) + return err + } + return nil +} + +func (r *ReconcileEnhancedLivenessProbe2NodeProbe) generateNewNodePodProbe(pod *v1.Pod) (appsv1alpha1.NodePodProbe, error) { + podLivenessProbeConfig, err := parseEnhancedLivenessProbeConfig(pod) + if err != nil { + return appsv1alpha1.NodePodProbe{}, err + } + npp_tmp := appsv1alpha1.NodePodProbe{ + ObjectMeta: metav1.ObjectMeta{ + Name: r.GetPodNodeName(pod), + }, + } + npp_tmp.Spec = appsv1alpha1.NodePodProbeSpec{} + npp_tmp.Spec.PodProbes = []appsv1alpha1.PodProbe{} + newPodProbe := generatePodProbeByContainersProbeConfig(pod, podLivenessProbeConfig) + if len(newPodProbe.Probes) != 0 { + npp_tmp.Spec.PodProbes = append(npp_tmp.Spec.PodProbes, newPodProbe) + } else { + return appsv1alpha1.NodePodProbe{}, fmt.Errorf("Failed to generate pod probe object by containers probe config for pod: %s/%s", pod.Namespace, pod.Name) + } + return npp_tmp, nil +} + +func (r *ReconcileEnhancedLivenessProbe2NodeProbe) GetPodNodeName(pod *v1.Pod) string { + return pod.Spec.NodeName +} + +func generatePodProbeByContainersProbeConfig(pod *v1.Pod, containersLivenessProbeConfig []livenessprobeUtils.ContainerLivenessProbe) appsv1alpha1.PodProbe { + newPodProbe := appsv1alpha1.PodProbe{ + Name: pod.Name, + Namespace: pod.Namespace, + UID: fmt.Sprintf("%v", pod.UID), + IP: pod.Status.PodIP, + } + newPodContainerProbes := generatePodContainersProbe(pod, containersLivenessProbeConfig) + newPodProbe.Probes = newPodContainerProbes + return newPodProbe +} + +func generatePodContainersProbe(pod *v1.Pod, containersLivenessProbeConfig []livenessprobeUtils.ContainerLivenessProbe) []appsv1alpha1.ContainerProbe { + newPodContainerProbes := []appsv1alpha1.ContainerProbe{} + for _, cp := range containersLivenessProbeConfig { + cProbe := appsv1alpha1.ContainerProbe{} + cProbe.Name = fmt.Sprintf("%s-%s", pod.Name, cp.Name) + cProbe.ContainerName = cp.Name + cProbe.Probe.Probe = cp.LivenessProbe + newPodContainerProbes = append(newPodContainerProbes, cProbe) + } + return newPodContainerProbes +} + +func getRawEnhancedLivenessProbeConfig(pod *v1.Pod) string { + return pod.Annotations[alpha1.AnnotationNativeContainerProbeContext] +} + +func parseEnhancedLivenessProbeConfig(pod *v1.Pod) ([]livenessprobeUtils.ContainerLivenessProbe, error) { + podRawLivenessProbeConfig := getRawEnhancedLivenessProbeConfig(pod) + if podRawLivenessProbeConfig == "" { + return []livenessprobeUtils.ContainerLivenessProbe{}, nil + } + podLivenessProbeConfig := []livenessprobeUtils.ContainerLivenessProbe{} + if err := json.Unmarshal([]byte(podRawLivenessProbeConfig), &podLivenessProbeConfig); err != nil { + klog.Errorf("Failed to json unmarshal pod raw livensss probe configuration, pod: %s/%s, error: %v", pod.Namespace, pod.Name, err) + return []livenessprobeUtils.ContainerLivenessProbe{}, err + } + return podLivenessProbeConfig, nil +} + +func usingEnhancedLivenessProbe(pod *v1.Pod) bool { + if pod.Annotations[alpha1.AnnotationUsingEnhancedLiveness] == "true" { + return true + } + return false +} diff --git a/pkg/controller/livenessprobemapnodeprobe/probemapnodeprobe_utils_test.go b/pkg/controller/livenessprobemapnodeprobe/probemapnodeprobe_utils_test.go new file mode 100644 index 0000000000..ae7a9d33ab --- /dev/null +++ b/pkg/controller/livenessprobemapnodeprobe/probemapnodeprobe_utils_test.go @@ -0,0 +1,1624 @@ +package enhancedlivenessprobe2nodeprobe + +import ( + "context" + "fmt" + "reflect" + "testing" + + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" + appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" + "github.com/openkruise/kruise/pkg/util" + livenessprobeUtils "github.com/openkruise/kruise/pkg/util/livenessprobe" +) + +func TestGeneratePodProbeObjByContainersProbeConfig(t *testing.T) { + testCase := []struct { + name string + pod *v1.Pod + containersLivenessProbeConfig []livenessprobeUtils.ContainerLivenessProbe + expect appsv1alpha1.PodProbe + }{ + { + name: "exist container probe config", + pod: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + Namespace: "sp1", + UID: "11-22-33", + }, + Status: v1.PodStatus{ + PodIP: "1.1.1.1", + }, + }, + containersLivenessProbeConfig: []livenessprobeUtils.ContainerLivenessProbe{ + { + Name: "c1", + LivenessProbe: v1.Probe{ + ProbeHandler: v1.ProbeHandler{ + Exec: &v1.ExecAction{ + Command: []string{"/bin/sh", "-c", "/healthy.sh"}, + }, + }, + }, + }, + { + Name: "c2", + LivenessProbe: v1.Probe{ + ProbeHandler: v1.ProbeHandler{ + TCPSocket: &v1.TCPSocketAction{ + Port: intstr.IntOrString{Type: intstr.String, StrVal: "main-port"}, + }, + }, + }, + }, + }, + expect: appsv1alpha1.PodProbe{ + Name: "pod1", + Namespace: "sp1", + UID: "11-22-33", + IP: "1.1.1.1", + Probes: []appsv1alpha1.ContainerProbe{ + { + Name: "pod1-c1", + ContainerName: "c1", + Probe: appsv1alpha1.ContainerProbeSpec{ + Probe: v1.Probe{ + ProbeHandler: v1.ProbeHandler{ + Exec: &v1.ExecAction{ + Command: []string{"/bin/sh", "-c", "/healthy.sh"}, + }, + }, + }, + }, + }, + { + Name: "pod1-c2", + ContainerName: "c2", + Probe: appsv1alpha1.ContainerProbeSpec{ + Probe: v1.Probe{ + ProbeHandler: v1.ProbeHandler{ + TCPSocket: &v1.TCPSocketAction{ + Port: intstr.IntOrString{Type: intstr.String, StrVal: "main-port"}, + }, + }, + }, + }, + }, + }, + }, + }, + { + name: "no container probe config", + pod: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + Namespace: "sp1", + UID: "11-22-33", + }, + Status: v1.PodStatus{ + PodIP: "1.1.1.1", + }, + }, + containersLivenessProbeConfig: []livenessprobeUtils.ContainerLivenessProbe{}, + expect: appsv1alpha1.PodProbe{ + Name: "pod1", + Namespace: "sp1", + UID: "11-22-33", + IP: "1.1.1.1", + Probes: []appsv1alpha1.ContainerProbe{}, + }, + }, + } + for _, tc := range testCase { + t.Run(tc.name, func(t *testing.T) { + got := generatePodProbeByContainersProbeConfig(tc.pod, tc.containersLivenessProbeConfig) + if !reflect.DeepEqual(got, tc.expect) { + t.Errorf("Test case: %v failed, expect: %v, but: %v", tc.name, tc.expect, got) + } + }) + } +} + +func TestGeneratePodContainersProbe(t *testing.T) { + testCase := []struct { + name string + pod *v1.Pod + containersLivenessProbeConfig []livenessprobeUtils.ContainerLivenessProbe + expect []appsv1alpha1.ContainerProbe + }{ + { + name: "exist container probe config", + pod: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + Namespace: "sp1", + UID: "11-22-33", + }, + Status: v1.PodStatus{ + PodIP: "1.1.1.1", + }, + }, + containersLivenessProbeConfig: []livenessprobeUtils.ContainerLivenessProbe{ + { + Name: "c1", + LivenessProbe: v1.Probe{ + ProbeHandler: v1.ProbeHandler{ + Exec: &v1.ExecAction{ + Command: []string{"/bin/sh", "-c", "/healthy.sh"}, + }, + }, + }, + }, + { + Name: "c2", + LivenessProbe: v1.Probe{ + ProbeHandler: v1.ProbeHandler{ + TCPSocket: &v1.TCPSocketAction{ + Port: intstr.IntOrString{Type: intstr.String, StrVal: "main-port"}, + }, + }, + }, + }, + }, + expect: []appsv1alpha1.ContainerProbe{ + { + Name: "pod1-c1", + ContainerName: "c1", + Probe: appsv1alpha1.ContainerProbeSpec{ + Probe: v1.Probe{ + ProbeHandler: v1.ProbeHandler{ + Exec: &v1.ExecAction{ + Command: []string{"/bin/sh", "-c", "/healthy.sh"}, + }, + }, + }, + }, + }, + { + Name: "pod1-c2", + ContainerName: "c2", + Probe: appsv1alpha1.ContainerProbeSpec{ + Probe: v1.Probe{ + ProbeHandler: v1.ProbeHandler{ + TCPSocket: &v1.TCPSocketAction{ + Port: intstr.IntOrString{Type: intstr.String, StrVal: "main-port"}, + }, + }, + }, + }, + }, + }, + }, + { + name: "no container probe config", + pod: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + Namespace: "sp1", + UID: "11-22-33", + }, + Status: v1.PodStatus{ + PodIP: "1.1.1.1", + }, + }, + containersLivenessProbeConfig: []livenessprobeUtils.ContainerLivenessProbe{}, + expect: []appsv1alpha1.ContainerProbe{}, + }, + } + for _, tc := range testCase { + t.Run(tc.name, func(t *testing.T) { + got := generatePodContainersProbe(tc.pod, tc.containersLivenessProbeConfig) + if !reflect.DeepEqual(got, tc.expect) { + t.Errorf("Test case: %v failed, expect: %v, but: %v", tc.name, tc.expect, got) + } + }) + } +} + +func TestGetRawEnhancedLivenessProbeConfig(t *testing.T) { + testCase := []struct { + name string + pod *v1.Pod + expect string + }{ + { + name: "exist annotation for pod liveness probe config", + pod: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + Namespace: "sp1", + Annotations: map[string]string{ + alpha1.AnnotationNativeContainerProbeContext: `[{"name":"c1","livenessProbe":{"httpGet":{"path":"/health","port":7001},"initialDelaySeconds":1000,"timeoutSeconds":5,"periodSeconds":100,"successThreshold":2,"failureThreshold":3}},{"name":"c2","livenessProbe":{"tcpSocket":{"port":7001},"initialDelaySeconds":1000,"timeoutSeconds":5,"periodSeconds":100,"successThreshold":2,"failureThreshold":3}}]`, + }, + }, + }, + expect: `[{"name":"c1","livenessProbe":{"httpGet":{"path":"/health","port":7001},"initialDelaySeconds":1000,"timeoutSeconds":5,"periodSeconds":100,"successThreshold":2,"failureThreshold":3}},{"name":"c2","livenessProbe":{"tcpSocket":{"port":7001},"initialDelaySeconds":1000,"timeoutSeconds":5,"periodSeconds":100,"successThreshold":2,"failureThreshold":3}}]`, + }, + { + name: "no found annotation for pod liveness probe config", + pod: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + Namespace: "sp1", + }, + }, + expect: "", + }, + } + for _, tc := range testCase { + t.Run(tc.name, func(t *testing.T) { + got := getRawEnhancedLivenessProbeConfig(tc.pod) + if !reflect.DeepEqual(got, tc.expect) { + t.Errorf("Test case: %v failed, expect: %v, but: %v", tc.name, tc.expect, got) + } + }) + } +} + +func TestGetEnhancedLivenessProbeConfig(t *testing.T) { + testCase := []struct { + name string + pod *v1.Pod + expect []livenessprobeUtils.ContainerLivenessProbe + }{ + { + name: "exist annotation for pod liveness probe config", + pod: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + Namespace: "sp1", + Annotations: map[string]string{ + alpha1.AnnotationNativeContainerProbeContext: `[{"name":"c1","livenessProbe":{"httpGet":{"path":"/health","port":7001},"initialDelaySeconds":1000,"timeoutSeconds":5,"periodSeconds":100,"successThreshold":2,"failureThreshold":3}},{"name":"c2","livenessProbe":{"tcpSocket":{"port":7001},"initialDelaySeconds":1000,"timeoutSeconds":5,"periodSeconds":100,"successThreshold":2,"failureThreshold":3}}]`, + }, + }, + }, + expect: []livenessprobeUtils.ContainerLivenessProbe{ + { + Name: "c1", + LivenessProbe: v1.Probe{ + ProbeHandler: v1.ProbeHandler{ + HTTPGet: &v1.HTTPGetAction{ + Path: "/health", + Port: intstr.FromInt(7001), + }, + }, + InitialDelaySeconds: 1000, + TimeoutSeconds: 5, + PeriodSeconds: 100, + SuccessThreshold: 2, + FailureThreshold: 3, + }, + }, + { + Name: "c2", + LivenessProbe: v1.Probe{ + ProbeHandler: v1.ProbeHandler{ + TCPSocket: &v1.TCPSocketAction{ + Port: intstr.FromInt(7001), + }, + }, + InitialDelaySeconds: 1000, + TimeoutSeconds: 5, + PeriodSeconds: 100, + SuccessThreshold: 2, + FailureThreshold: 3, + }, + }, + }, + }, + { + name: "no found annotation for pod liveness probe config", + pod: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + Namespace: "sp1", + }, + }, + expect: []livenessprobeUtils.ContainerLivenessProbe{}, + }, + } + for _, tc := range testCase { + t.Run(tc.name, func(t *testing.T) { + got, err := parseEnhancedLivenessProbeConfig(tc.pod) + if err != nil { + t.Errorf("Test case: %v faield, err: %v", tc.name, err) + } + if !reflect.DeepEqual(got, tc.expect) { + t.Errorf("Test case: %v failed, expect: %v, but: %v", tc.name, tc.expect, got) + } + }) + } +} + +func TestUsingEnhancedLivenessProbe(t *testing.T) { + testCase := []struct { + name string + pod *v1.Pod + expect bool + }{ + { + name: "using enhanced liveness probe", + pod: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + Namespace: "sp1", + Annotations: map[string]string{ + alpha1.AnnotationUsingEnhancedLiveness: "true", + }, + }, + }, + expect: true, + }, + { + name: "no using enhanced liveness probe", + pod: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + Namespace: "sp1", + }, + }, + expect: false, + }, + { + name: "no using enhanced liveness probe v2", + pod: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + Namespace: "sp1", + Annotations: map[string]string{ + alpha1.AnnotationUsingEnhancedLiveness: "false", + }, + }, + }, + expect: false, + }, + } + for _, tc := range testCase { + t.Run(tc.name, func(t *testing.T) { + got := usingEnhancedLivenessProbe(tc.pod) + if !reflect.DeepEqual(got, tc.expect) { + t.Errorf("Test case: %v failed, expect: %v, but: %v", tc.name, tc.expect, got) + } + }) + } +} + +func TestGenerateNewNodePodProbe(t *testing.T) { + testCase := []struct { + name string + pod *v1.Pod + expectNodePodProbe *appsv1alpha1.NodePodProbe + expectErr error + }{ + { + name: "correct to generate new node pod probe", + pod: &v1.Pod{ + TypeMeta: metav1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + Namespace: "sp1", + Annotations: map[string]string{ + alpha1.AnnotationNativeContainerProbeContext: `[{"name":"c1","livenessProbe":{"httpGet":{"path":"/health","port":7001},"initialDelaySeconds":1000,"timeoutSeconds":5,"periodSeconds":100,"successThreshold":2,"failureThreshold":3}},{"name":"c2","livenessProbe":{"tcpSocket":{"port":7001},"initialDelaySeconds":1000,"timeoutSeconds":5,"periodSeconds":100,"successThreshold":2,"failureThreshold":3}}]`, + }, + UID: types.UID("111-222"), + }, + Spec: v1.PodSpec{ + NodeName: "node1", + }, + }, + expectNodePodProbe: &appsv1alpha1.NodePodProbe{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + }, + Spec: appsv1alpha1.NodePodProbeSpec{ + PodProbes: []appsv1alpha1.PodProbe{ + { + Name: "pod1", + Namespace: "sp1", + UID: fmt.Sprintf("%s", "111-222"), + Probes: []appsv1alpha1.ContainerProbe{ + { + Name: "pod1-c1", + ContainerName: "c1", + Probe: appsv1alpha1.ContainerProbeSpec{ + Probe: v1.Probe{ + ProbeHandler: v1.ProbeHandler{ + HTTPGet: &v1.HTTPGetAction{ + Path: "/health", + Port: intstr.IntOrString{Type: intstr.Int, IntVal: 7001}, + }, + }, + InitialDelaySeconds: 1000, + TimeoutSeconds: 5, + PeriodSeconds: 100, + SuccessThreshold: 2, + FailureThreshold: 3, + }, + }, + }, + { + Name: "pod1-c2", + ContainerName: "c2", + Probe: appsv1alpha1.ContainerProbeSpec{ + Probe: v1.Probe{ + ProbeHandler: v1.ProbeHandler{ + TCPSocket: &v1.TCPSocketAction{ + Port: intstr.IntOrString{Type: intstr.Int, IntVal: 7001}, + }, + }, + InitialDelaySeconds: 1000, + TimeoutSeconds: 5, + PeriodSeconds: 100, + SuccessThreshold: 2, + FailureThreshold: 3, + }, + }, + }, + }, + }, + }, + }, + }, + expectErr: nil, + }, + { + name: "no found annotation for pod to generate new node pod probe", + pod: &v1.Pod{ + TypeMeta: metav1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + Namespace: "sp1", + UID: types.UID("111-222"), + }, + Spec: v1.PodSpec{ + NodeName: "node1", + }, + }, + expectNodePodProbe: &appsv1alpha1.NodePodProbe{}, + expectErr: fmt.Errorf("Failed to generate pod probe object by containers probe config for pod: %s/%s", "sp1", "pod1"), + }, + } + + for _, tc := range testCase { + t.Run(tc.name, func(t *testing.T) { + fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(tc.pod).Build() + recon := ReconcileEnhancedLivenessProbe2NodeProbe{Client: fakeClient} + got, err := recon.generateNewNodePodProbe(tc.pod) + if !reflect.DeepEqual(err, tc.expectErr) { + t.Errorf("Failed to generate new node pod probe, err: %v, expect: %v", err, tc.expectErr) + } + if !reflect.DeepEqual(util.DumpJSON(tc.expectNodePodProbe), util.DumpJSON(got)) { + t.Errorf("No match, expect: %v, but: %v", util.DumpJSON(tc.expectNodePodProbe), util.DumpJSON(got)) + } + }) + } +} + +func TestRetryOnConflictAddNodeProbeConfig(t *testing.T) { + testCase := []struct { + name string + pod *v1.Pod + podNodeProbe *appsv1alpha1.NodePodProbe + expectPodNodeProbe *appsv1alpha1.NodePodProbe + containersLivenessProbeConfig []livenessprobeUtils.ContainerLivenessProbe + }{ + { + name: "case add node probe config", + pod: &v1.Pod{ + TypeMeta: metav1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + Namespace: "sp1", + UID: types.UID("111-222"), + }, + Spec: v1.PodSpec{ + NodeName: "node1", + }, + }, + podNodeProbe: &appsv1alpha1.NodePodProbe{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + }, + Spec: appsv1alpha1.NodePodProbeSpec{ + PodProbes: []appsv1alpha1.PodProbe{ + { + Name: "pod2", + Namespace: "sp2", + UID: fmt.Sprintf("%s", "222-111"), + Probes: []appsv1alpha1.ContainerProbe{ + { + Name: "pod2-c2", + ContainerName: "c2", + Probe: appsv1alpha1.ContainerProbeSpec{ + Probe: v1.Probe{ + ProbeHandler: v1.ProbeHandler{ + Exec: &v1.ExecAction{ + Command: []string{"/bin/sh", "-c", "/healthy.sh"}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + expectPodNodeProbe: &appsv1alpha1.NodePodProbe{ + TypeMeta: metav1.TypeMeta{ + Kind: "NodePodProbe", + APIVersion: "apps.kruise.io/v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + }, + Spec: appsv1alpha1.NodePodProbeSpec{ + PodProbes: []appsv1alpha1.PodProbe{ + { + Name: "pod2", + Namespace: "sp2", + UID: fmt.Sprintf("%s", "222-111"), + Probes: []appsv1alpha1.ContainerProbe{ + { + Name: "pod2-c2", + ContainerName: "c2", + Probe: appsv1alpha1.ContainerProbeSpec{ + Probe: v1.Probe{ + ProbeHandler: v1.ProbeHandler{ + Exec: &v1.ExecAction{ + Command: []string{"/bin/sh", "-c", "/healthy.sh"}, + }, + }, + }, + }, + }, + }, + }, + { + Name: "pod1", + Namespace: "sp1", + UID: fmt.Sprintf("%s", "111-222"), + Probes: []appsv1alpha1.ContainerProbe{ + { + Name: "pod1-c1", + ContainerName: "c1", + Probe: appsv1alpha1.ContainerProbeSpec{ + Probe: v1.Probe{ + ProbeHandler: v1.ProbeHandler{ + HTTPGet: &v1.HTTPGetAction{ + Path: "/health", + Port: intstr.IntOrString{Type: intstr.Int, IntVal: 7001}, + }, + }, + InitialDelaySeconds: 1000, + TimeoutSeconds: 5, + PeriodSeconds: 100, + SuccessThreshold: 2, + FailureThreshold: 3, + }, + }, + }, + { + Name: "pod1-c2", + ContainerName: "c2", + Probe: appsv1alpha1.ContainerProbeSpec{ + Probe: v1.Probe{ + ProbeHandler: v1.ProbeHandler{ + TCPSocket: &v1.TCPSocketAction{ + Port: intstr.IntOrString{Type: intstr.Int, IntVal: 7001}, + }, + }, + InitialDelaySeconds: 1000, + TimeoutSeconds: 5, + PeriodSeconds: 100, + SuccessThreshold: 2, + FailureThreshold: 3, + }, + }, + }, + }, + }, + }, + }, + }, + containersLivenessProbeConfig: []livenessprobeUtils.ContainerLivenessProbe{ + { + Name: "c1", + LivenessProbe: v1.Probe{ + ProbeHandler: v1.ProbeHandler{ + HTTPGet: &v1.HTTPGetAction{ + Path: "/health", + Port: intstr.FromInt(7001), + }, + }, + InitialDelaySeconds: 1000, + TimeoutSeconds: 5, + PeriodSeconds: 100, + SuccessThreshold: 2, + FailureThreshold: 3, + }, + }, + { + Name: "c2", + LivenessProbe: v1.Probe{ + ProbeHandler: v1.ProbeHandler{ + TCPSocket: &v1.TCPSocketAction{ + Port: intstr.FromInt(7001), + }, + }, + InitialDelaySeconds: 1000, + TimeoutSeconds: 5, + PeriodSeconds: 100, + SuccessThreshold: 2, + FailureThreshold: 3, + }, + }, + }, + }, + { + name: "case add node probe config(exist same config)", + pod: &v1.Pod{ + TypeMeta: metav1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + Namespace: "sp1", + UID: types.UID("111-222"), + }, + Spec: v1.PodSpec{ + NodeName: "node1", + }, + }, + podNodeProbe: &appsv1alpha1.NodePodProbe{ + TypeMeta: metav1.TypeMeta{ + Kind: "NodePodProbe", + APIVersion: "apps.kruise.io/v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + }, + Spec: appsv1alpha1.NodePodProbeSpec{ + PodProbes: []appsv1alpha1.PodProbe{ + { + Name: "pod2", + Namespace: "sp2", + UID: fmt.Sprintf("%s", "222-111"), + Probes: []appsv1alpha1.ContainerProbe{ + { + Name: "pod2-c2", + ContainerName: "c2", + Probe: appsv1alpha1.ContainerProbeSpec{ + Probe: v1.Probe{ + ProbeHandler: v1.ProbeHandler{ + Exec: &v1.ExecAction{ + Command: []string{"/bin/sh", "-c", "/healthy.sh"}, + }, + }, + }, + }, + }, + }, + }, + { + Name: "pod1", + Namespace: "sp1", + UID: fmt.Sprintf("%s", "111-222"), + Probes: []appsv1alpha1.ContainerProbe{ + { + Name: "pod1-c1", + ContainerName: "c1", + Probe: appsv1alpha1.ContainerProbeSpec{ + Probe: v1.Probe{ + ProbeHandler: v1.ProbeHandler{ + HTTPGet: &v1.HTTPGetAction{ + Path: "/health", + Port: intstr.IntOrString{Type: intstr.Int, IntVal: 7001}, + }, + }, + InitialDelaySeconds: 1000, + TimeoutSeconds: 5, + PeriodSeconds: 100, + SuccessThreshold: 2, + FailureThreshold: 3, + }, + }, + }, + }, + }, + }, + }, + }, + expectPodNodeProbe: &appsv1alpha1.NodePodProbe{ + TypeMeta: metav1.TypeMeta{ + Kind: "NodePodProbe", + APIVersion: "apps.kruise.io/v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + }, + Spec: appsv1alpha1.NodePodProbeSpec{ + PodProbes: []appsv1alpha1.PodProbe{ + { + Name: "pod2", + Namespace: "sp2", + UID: fmt.Sprintf("%s", "222-111"), + Probes: []appsv1alpha1.ContainerProbe{ + { + Name: "pod2-c2", + ContainerName: "c2", + Probe: appsv1alpha1.ContainerProbeSpec{ + Probe: v1.Probe{ + ProbeHandler: v1.ProbeHandler{ + Exec: &v1.ExecAction{ + Command: []string{"/bin/sh", "-c", "/healthy.sh"}, + }, + }, + }, + }, + }, + }, + }, + { + Name: "pod1", + Namespace: "sp1", + UID: fmt.Sprintf("%s", "111-222"), + Probes: []appsv1alpha1.ContainerProbe{ + { + Name: "pod1-c1", + ContainerName: "c1", + Probe: appsv1alpha1.ContainerProbeSpec{ + Probe: v1.Probe{ + ProbeHandler: v1.ProbeHandler{ + HTTPGet: &v1.HTTPGetAction{ + Path: "/health", + Port: intstr.IntOrString{Type: intstr.Int, IntVal: 7001}, + }, + }, + InitialDelaySeconds: 1000, + TimeoutSeconds: 5, + PeriodSeconds: 100, + SuccessThreshold: 2, + FailureThreshold: 3, + }, + }, + }, + { + Name: "pod1-c2", + ContainerName: "c2", + Probe: appsv1alpha1.ContainerProbeSpec{ + Probe: v1.Probe{ + ProbeHandler: v1.ProbeHandler{ + TCPSocket: &v1.TCPSocketAction{ + Port: intstr.IntOrString{Type: intstr.Int, IntVal: 7001}, + }, + }, + InitialDelaySeconds: 1000, + TimeoutSeconds: 5, + PeriodSeconds: 100, + SuccessThreshold: 2, + FailureThreshold: 3, + }, + }, + }, + }, + }, + }, + }, + }, + containersLivenessProbeConfig: []livenessprobeUtils.ContainerLivenessProbe{ + { + Name: "c1", + LivenessProbe: v1.Probe{ + ProbeHandler: v1.ProbeHandler{ + HTTPGet: &v1.HTTPGetAction{ + Path: "/health", + Port: intstr.FromInt(7001), + }, + }, + InitialDelaySeconds: 1000, + TimeoutSeconds: 5, + PeriodSeconds: 100, + SuccessThreshold: 2, + FailureThreshold: 3, + }, + }, + { + Name: "c2", + LivenessProbe: v1.Probe{ + ProbeHandler: v1.ProbeHandler{ + TCPSocket: &v1.TCPSocketAction{ + Port: intstr.FromInt(7001), + }, + }, + InitialDelaySeconds: 1000, + TimeoutSeconds: 5, + PeriodSeconds: 100, + SuccessThreshold: 2, + FailureThreshold: 3, + }, + }, + }, + }, + } + + for _, tc := range testCase { + t.Run(tc.name, func(t *testing.T) { + fakeClient := fake.NewClientBuilder().WithScheme(scheme).Build() + // create pod + err := fakeClient.Create(context.TODO(), tc.pod) + if err != nil { + t.Errorf("Failed to create pod: %s/%s, err: %v", tc.pod.Namespace, tc.pod.Name, err) + } + // create nodeProbe + err = fakeClient.Create(context.TODO(), tc.podNodeProbe) + if err != nil { + t.Errorf("Failed to create node pod probe: %s, err: %v", tc.podNodeProbe.Name, err) + } + recon := ReconcileEnhancedLivenessProbe2NodeProbe{Client: fakeClient} + err = recon.retryOnConflictAddNodeProbeConfig(tc.pod, tc.containersLivenessProbeConfig) + if err != nil { + t.Errorf("Failed to retry on conflict add node probe config, err: %v", err) + } + getNodeProbe := appsv1alpha1.NodePodProbe{} + err = fakeClient.Get(context.TODO(), types.NamespacedName{Name: recon.GetPodNodeName(tc.pod)}, &getNodeProbe) + if err != nil { + t.Errorf("Failed to get node probe, err: %v", err) + } + if !checkNodePodProbeSpecEqual(tc.expectPodNodeProbe, &getNodeProbe) { + t.Errorf("No match, expect: %v, but: %v", util.DumpJSON(tc.expectPodNodeProbe), util.DumpJSON(getNodeProbe)) + } + }) + } + +} + +func TestRetryOnConflictDelNodeProbeConfig(t *testing.T) { + testCase := []struct { + name string + pod *v1.Pod + podNodeProbe *appsv1alpha1.NodePodProbe + expectPodNodeProbe *appsv1alpha1.NodePodProbe + }{ + { + name: "case del node probe config", + pod: &v1.Pod{ + TypeMeta: metav1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + Namespace: "sp1", + UID: types.UID("111-222"), + }, + Spec: v1.PodSpec{ + NodeName: "node1", + }, + }, + podNodeProbe: &appsv1alpha1.NodePodProbe{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + }, + Spec: appsv1alpha1.NodePodProbeSpec{ + PodProbes: []appsv1alpha1.PodProbe{ + { + Name: "pod2", + Namespace: "sp2", + UID: fmt.Sprintf("%s", "222-111"), + Probes: []appsv1alpha1.ContainerProbe{ + { + Name: "pod2-c2", + ContainerName: "c2", + Probe: appsv1alpha1.ContainerProbeSpec{ + Probe: v1.Probe{ + ProbeHandler: v1.ProbeHandler{ + Exec: &v1.ExecAction{ + Command: []string{"/bin/sh", "-c", "/healthy.sh"}, + }, + }, + }, + }, + }, + }, + }, + { + Name: "pod1", + Namespace: "sp1", + UID: fmt.Sprintf("%s", "111-222"), + Probes: []appsv1alpha1.ContainerProbe{ + { + Name: "pod1-c1", + ContainerName: "c1", + Probe: appsv1alpha1.ContainerProbeSpec{ + Probe: v1.Probe{ + ProbeHandler: v1.ProbeHandler{ + HTTPGet: &v1.HTTPGetAction{ + Path: "/health", + Port: intstr.IntOrString{Type: intstr.Int, IntVal: 7001}, + }, + }, + InitialDelaySeconds: 1000, + TimeoutSeconds: 5, + PeriodSeconds: 100, + SuccessThreshold: 2, + FailureThreshold: 3, + }, + }, + }, + { + Name: "pod1-c2", + ContainerName: "c2", + Probe: appsv1alpha1.ContainerProbeSpec{ + Probe: v1.Probe{ + ProbeHandler: v1.ProbeHandler{ + TCPSocket: &v1.TCPSocketAction{ + Port: intstr.IntOrString{Type: intstr.Int, IntVal: 7001}, + }, + }, + InitialDelaySeconds: 1000, + TimeoutSeconds: 5, + PeriodSeconds: 100, + SuccessThreshold: 2, + FailureThreshold: 3, + }, + }, + }, + }, + }, + }, + }, + }, + expectPodNodeProbe: &appsv1alpha1.NodePodProbe{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + }, + Spec: appsv1alpha1.NodePodProbeSpec{ + PodProbes: []appsv1alpha1.PodProbe{ + { + Name: "pod2", + Namespace: "sp2", + UID: fmt.Sprintf("%s", "222-111"), + Probes: []appsv1alpha1.ContainerProbe{ + { + Name: "pod2-c2", + ContainerName: "c2", + Probe: appsv1alpha1.ContainerProbeSpec{ + Probe: v1.Probe{ + ProbeHandler: v1.ProbeHandler{ + Exec: &v1.ExecAction{ + Command: []string{"/bin/sh", "-c", "/healthy.sh"}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + { + name: "case full del node probe config", + pod: &v1.Pod{ + TypeMeta: metav1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + Namespace: "sp1", + UID: types.UID("111-222"), + }, + Spec: v1.PodSpec{ + NodeName: "node1", + }, + }, + podNodeProbe: &appsv1alpha1.NodePodProbe{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + }, + Spec: appsv1alpha1.NodePodProbeSpec{ + PodProbes: []appsv1alpha1.PodProbe{ + { + Name: "pod1", + Namespace: "sp1", + UID: fmt.Sprintf("%s", "111-222"), + Probes: []appsv1alpha1.ContainerProbe{ + { + Name: "pod1-c1", + ContainerName: "c1", + Probe: appsv1alpha1.ContainerProbeSpec{ + Probe: v1.Probe{ + ProbeHandler: v1.ProbeHandler{ + HTTPGet: &v1.HTTPGetAction{ + Path: "/health", + Port: intstr.IntOrString{Type: intstr.Int, IntVal: 7001}, + }, + }, + InitialDelaySeconds: 1000, + TimeoutSeconds: 5, + PeriodSeconds: 100, + SuccessThreshold: 2, + FailureThreshold: 3, + }, + }, + }, + { + Name: "pod1-c2", + ContainerName: "c2", + Probe: appsv1alpha1.ContainerProbeSpec{ + Probe: v1.Probe{ + ProbeHandler: v1.ProbeHandler{ + TCPSocket: &v1.TCPSocketAction{ + Port: intstr.IntOrString{Type: intstr.Int, IntVal: 7001}, + }, + }, + InitialDelaySeconds: 1000, + TimeoutSeconds: 5, + PeriodSeconds: 100, + SuccessThreshold: 2, + FailureThreshold: 3, + }, + }, + }, + }, + }, + }, + }, + }, + expectPodNodeProbe: &appsv1alpha1.NodePodProbe{}, + }, + } + + for _, tc := range testCase { + t.Run(tc.name, func(t *testing.T) { + fakeClient := fake.NewClientBuilder().WithScheme(scheme).Build() + // create pod + err := fakeClient.Create(context.TODO(), tc.pod) + if err != nil { + t.Errorf("Failed to create pod: %s/%s, err: %v", tc.pod.Namespace, tc.pod.Name, err) + } + // create nodeProbe + err = fakeClient.Create(context.TODO(), tc.podNodeProbe) + if err != nil { + t.Errorf("Failed to create node pod probe: %s, err: %v", tc.podNodeProbe.Name, err) + } + recon := ReconcileEnhancedLivenessProbe2NodeProbe{Client: fakeClient} + err = recon.retryOnConflictDelNodeProbeConfig(tc.pod) + if err != nil { + t.Errorf("Failed to retry on conflict del node probe config, err: %v", err) + } + getNodeProbe := appsv1alpha1.NodePodProbe{} + err = fakeClient.Get(context.TODO(), types.NamespacedName{Name: recon.GetPodNodeName(tc.pod)}, &getNodeProbe) + if err != nil { + if !errors.IsNotFound(err) { + t.Errorf("Failed to get node probe, err: %v", err) + } + } + if !checkNodePodProbeSpecEqual(tc.expectPodNodeProbe, &getNodeProbe) { + t.Errorf("No match, expect: %v, but: %v", util.DumpJSON(tc.expectPodNodeProbe), util.DumpJSON(getNodeProbe)) + } + }) + } +} + +func TestAddNodeProbeConfig(t *testing.T) { + testCase := []struct { + name string + pod *v1.Pod + podNodeProbe *appsv1alpha1.NodePodProbe + expectPodNodeProbe *appsv1alpha1.NodePodProbe + }{ + { + name: "no found node probe config, create it", + pod: &v1.Pod{ + TypeMeta: metav1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + Namespace: "sp1", + Annotations: map[string]string{ + alpha1.AnnotationNativeContainerProbeContext: `[{"name":"c1","livenessProbe":{"httpGet":{"path":"/health","port":7001},"initialDelaySeconds":1000,"timeoutSeconds":5,"periodSeconds":100,"successThreshold":2,"failureThreshold":3}},{"name":"c2","livenessProbe":{"tcpSocket":{"port":7001},"initialDelaySeconds":1000,"timeoutSeconds":5,"periodSeconds":100,"successThreshold":2,"failureThreshold":3}}]`, + }, + UID: types.UID("111-222"), + }, + Spec: v1.PodSpec{ + NodeName: "node1", + }, + }, + podNodeProbe: &appsv1alpha1.NodePodProbe{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node2", //no found node probe config for this pod + }, + Spec: appsv1alpha1.NodePodProbeSpec{ + PodProbes: []appsv1alpha1.PodProbe{ + { + Name: "pod2", + Namespace: "sp2", + UID: fmt.Sprintf("%s", "222-111"), + Probes: []appsv1alpha1.ContainerProbe{ + { + Name: "pod2-c2", + ContainerName: "c2", + Probe: appsv1alpha1.ContainerProbeSpec{ + Probe: v1.Probe{ + ProbeHandler: v1.ProbeHandler{ + Exec: &v1.ExecAction{ + Command: []string{"/bin/sh", "-c", "/healthy.sh"}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + expectPodNodeProbe: &appsv1alpha1.NodePodProbe{ + TypeMeta: metav1.TypeMeta{ + Kind: "NodePodProbe", + APIVersion: "apps.kruise.io/v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + }, + Spec: appsv1alpha1.NodePodProbeSpec{ + PodProbes: []appsv1alpha1.PodProbe{ + { + Name: "pod1", + Namespace: "sp1", + UID: fmt.Sprintf("%s", "111-222"), + Probes: []appsv1alpha1.ContainerProbe{ + { + Name: "pod1-c1", + ContainerName: "c1", + Probe: appsv1alpha1.ContainerProbeSpec{ + Probe: v1.Probe{ + ProbeHandler: v1.ProbeHandler{ + HTTPGet: &v1.HTTPGetAction{ + Path: "/health", + Port: intstr.IntOrString{Type: intstr.Int, IntVal: 7001}, + }, + }, + InitialDelaySeconds: 1000, + TimeoutSeconds: 5, + PeriodSeconds: 100, + SuccessThreshold: 2, + FailureThreshold: 3, + }, + }, + }, + { + Name: "pod1-c2", + ContainerName: "c2", + Probe: appsv1alpha1.ContainerProbeSpec{ + Probe: v1.Probe{ + ProbeHandler: v1.ProbeHandler{ + TCPSocket: &v1.TCPSocketAction{ + Port: intstr.IntOrString{Type: intstr.Int, IntVal: 7001}, + }, + }, + InitialDelaySeconds: 1000, + TimeoutSeconds: 5, + PeriodSeconds: 100, + SuccessThreshold: 2, + FailureThreshold: 3, + }, + }, + }, + }, + }, + }, + }, + }, + }, + { + name: "found node probe config, update it", + pod: &v1.Pod{ + TypeMeta: metav1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + Namespace: "sp1", + Annotations: map[string]string{ + alpha1.AnnotationNativeContainerProbeContext: `[{"name":"c1","livenessProbe":{"httpGet":{"path":"/health","port":7001},"initialDelaySeconds":1000,"timeoutSeconds":5,"periodSeconds":100,"successThreshold":2,"failureThreshold":3}},{"name":"c2","livenessProbe":{"tcpSocket":{"port":7001},"initialDelaySeconds":1000,"timeoutSeconds":5,"periodSeconds":100,"successThreshold":2,"failureThreshold":3}}]`, + }, + UID: types.UID("111-222"), + }, + Spec: v1.PodSpec{ + NodeName: "node1", + }, + }, + podNodeProbe: &appsv1alpha1.NodePodProbe{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + }, + Spec: appsv1alpha1.NodePodProbeSpec{ + PodProbes: []appsv1alpha1.PodProbe{ + { + Name: "pod1", + Namespace: "sp1", + UID: fmt.Sprintf("%s", "111-222"), + Probes: []appsv1alpha1.ContainerProbe{ + { + Name: "pod1-c1", + ContainerName: "c1", + Probe: appsv1alpha1.ContainerProbeSpec{ + Probe: v1.Probe{ + ProbeHandler: v1.ProbeHandler{ + TCPSocket: &v1.TCPSocketAction{ + Port: intstr.IntOrString{Type: intstr.Int, IntVal: 7001}, + }, + }, + InitialDelaySeconds: 1000, + TimeoutSeconds: 5, + PeriodSeconds: 100, + SuccessThreshold: 2, + FailureThreshold: 3, + }, + }, + }, + { + Name: "pod1-c2", + ContainerName: "c2", + Probe: appsv1alpha1.ContainerProbeSpec{ + Probe: v1.Probe{ + ProbeHandler: v1.ProbeHandler{ + TCPSocket: &v1.TCPSocketAction{ + Port: intstr.IntOrString{Type: intstr.Int, IntVal: 7001}, + }, + }, + InitialDelaySeconds: 1000, + TimeoutSeconds: 5, + PeriodSeconds: 100, + SuccessThreshold: 2, + FailureThreshold: 3, + }, + }, + }, + }, + }, + }, + }, + }, + expectPodNodeProbe: &appsv1alpha1.NodePodProbe{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + }, + Spec: appsv1alpha1.NodePodProbeSpec{ + PodProbes: []appsv1alpha1.PodProbe{ + { + Name: "pod1", + Namespace: "sp1", + UID: fmt.Sprintf("%s", "111-222"), + Probes: []appsv1alpha1.ContainerProbe{ + { + Name: "pod1-c1", + ContainerName: "c1", + Probe: appsv1alpha1.ContainerProbeSpec{ + Probe: v1.Probe{ + ProbeHandler: v1.ProbeHandler{ + HTTPGet: &v1.HTTPGetAction{ + Path: "/health", + Port: intstr.IntOrString{Type: intstr.Int, IntVal: 7001}, + }, + }, + InitialDelaySeconds: 1000, + TimeoutSeconds: 5, + PeriodSeconds: 100, + SuccessThreshold: 2, + FailureThreshold: 3, + }, + }, + }, + { + Name: "pod1-c2", + ContainerName: "c2", + Probe: appsv1alpha1.ContainerProbeSpec{ + Probe: v1.Probe{ + ProbeHandler: v1.ProbeHandler{ + TCPSocket: &v1.TCPSocketAction{ + Port: intstr.IntOrString{Type: intstr.Int, IntVal: 7001}, + }, + }, + InitialDelaySeconds: 1000, + TimeoutSeconds: 5, + PeriodSeconds: 100, + SuccessThreshold: 2, + FailureThreshold: 3, + }, + }, + }, + }, + }, + }, + }, + }, + }, + } + + for _, tc := range testCase { + t.Run(tc.name, func(t *testing.T) { + fakeClient := fake.NewClientBuilder().WithScheme(scheme).Build() + err := fakeClient.Create(context.TODO(), tc.pod) + if err != nil { + t.Errorf("Failed to create pod: %s/%s, err: %v", tc.pod.Namespace, tc.pod.Name, err) + } + err = fakeClient.Create(context.TODO(), tc.podNodeProbe) + if err != nil { + t.Errorf("Failed to create node pod probe: %s, err: %v", tc.podNodeProbe.Name, err) + } + recon := ReconcileEnhancedLivenessProbe2NodeProbe{Client: fakeClient} + err = recon.addNodeProbeConfig(tc.pod) + if err != nil { + t.Errorf("Failed to add node probe config, err: %v", err) + } + getNodeProbe := appsv1alpha1.NodePodProbe{} + err = fakeClient.Get(context.TODO(), types.NamespacedName{Name: recon.GetPodNodeName(tc.pod)}, &getNodeProbe) + if err != nil { + if !errors.IsNotFound(err) { + t.Errorf("Failed to get node probe, err: %v", err) + } + } + if !checkNodePodProbeSpecEqual(tc.expectPodNodeProbe, &getNodeProbe) { + t.Errorf("No match, expect: %v, but: %v", util.DumpJSON(tc.expectPodNodeProbe), util.DumpJSON(getNodeProbe)) + } + }) + } +} + +func TestDelNodeProbeConfig(t *testing.T) { + testCase := []struct { + name string + pod *v1.Pod + podNodeProbe *appsv1alpha1.NodePodProbe + expectPodNodeProbe *appsv1alpha1.NodePodProbe + }{ + { + name: "case del node probe config", + pod: &v1.Pod{ + TypeMeta: metav1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + Namespace: "sp1", + UID: types.UID("111-222"), + }, + Spec: v1.PodSpec{ + NodeName: "node1", + }, + }, + podNodeProbe: &appsv1alpha1.NodePodProbe{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + }, + Spec: appsv1alpha1.NodePodProbeSpec{ + PodProbes: []appsv1alpha1.PodProbe{ + { + Name: "pod2", + Namespace: "sp2", + UID: fmt.Sprintf("%s", "222-111"), + Probes: []appsv1alpha1.ContainerProbe{ + { + Name: "pod2-c2", + ContainerName: "c2", + Probe: appsv1alpha1.ContainerProbeSpec{ + Probe: v1.Probe{ + ProbeHandler: v1.ProbeHandler{ + Exec: &v1.ExecAction{ + Command: []string{"/bin/sh", "-c", "/healthy.sh"}, + }, + }, + }, + }, + }, + }, + }, + { + Name: "pod1", + Namespace: "sp1", + UID: fmt.Sprintf("%s", "111-222"), + Probes: []appsv1alpha1.ContainerProbe{ + { + Name: "pod1-c1", + ContainerName: "c1", + Probe: appsv1alpha1.ContainerProbeSpec{ + Probe: v1.Probe{ + ProbeHandler: v1.ProbeHandler{ + HTTPGet: &v1.HTTPGetAction{ + Path: "/health", + Port: intstr.IntOrString{Type: intstr.Int, IntVal: 7001}, + }, + }, + InitialDelaySeconds: 1000, + TimeoutSeconds: 5, + PeriodSeconds: 100, + SuccessThreshold: 2, + FailureThreshold: 3, + }, + }, + }, + { + Name: "pod1-c2", + ContainerName: "c2", + Probe: appsv1alpha1.ContainerProbeSpec{ + Probe: v1.Probe{ + ProbeHandler: v1.ProbeHandler{ + TCPSocket: &v1.TCPSocketAction{ + Port: intstr.IntOrString{Type: intstr.Int, IntVal: 7001}, + }, + }, + InitialDelaySeconds: 1000, + TimeoutSeconds: 5, + PeriodSeconds: 100, + SuccessThreshold: 2, + FailureThreshold: 3, + }, + }, + }, + }, + }, + }, + }, + }, + expectPodNodeProbe: &appsv1alpha1.NodePodProbe{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + }, + Spec: appsv1alpha1.NodePodProbeSpec{ + PodProbes: []appsv1alpha1.PodProbe{ + { + Name: "pod2", + Namespace: "sp2", + UID: fmt.Sprintf("%s", "222-111"), + Probes: []appsv1alpha1.ContainerProbe{ + { + Name: "pod2-c2", + ContainerName: "c2", + Probe: appsv1alpha1.ContainerProbeSpec{ + Probe: v1.Probe{ + ProbeHandler: v1.ProbeHandler{ + Exec: &v1.ExecAction{ + Command: []string{"/bin/sh", "-c", "/healthy.sh"}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + { + name: "case full del node probe config", + pod: &v1.Pod{ + TypeMeta: metav1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + Namespace: "sp1", + UID: types.UID("111-222"), + }, + Spec: v1.PodSpec{ + NodeName: "node1", + }, + }, + podNodeProbe: &appsv1alpha1.NodePodProbe{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + }, + Spec: appsv1alpha1.NodePodProbeSpec{ + PodProbes: []appsv1alpha1.PodProbe{ + { + Name: "pod1", + Namespace: "sp1", + UID: fmt.Sprintf("%s", "111-222"), + Probes: []appsv1alpha1.ContainerProbe{ + { + Name: "pod1-c1", + ContainerName: "c1", + Probe: appsv1alpha1.ContainerProbeSpec{ + Probe: v1.Probe{ + ProbeHandler: v1.ProbeHandler{ + HTTPGet: &v1.HTTPGetAction{ + Path: "/health", + Port: intstr.IntOrString{Type: intstr.Int, IntVal: 7001}, + }, + }, + InitialDelaySeconds: 1000, + TimeoutSeconds: 5, + PeriodSeconds: 100, + SuccessThreshold: 2, + FailureThreshold: 3, + }, + }, + }, + { + Name: "pod1-c2", + ContainerName: "c2", + Probe: appsv1alpha1.ContainerProbeSpec{ + Probe: v1.Probe{ + ProbeHandler: v1.ProbeHandler{ + TCPSocket: &v1.TCPSocketAction{ + Port: intstr.IntOrString{Type: intstr.Int, IntVal: 7001}, + }, + }, + InitialDelaySeconds: 1000, + TimeoutSeconds: 5, + PeriodSeconds: 100, + SuccessThreshold: 2, + FailureThreshold: 3, + }, + }, + }, + }, + }, + }, + }, + }, + expectPodNodeProbe: &appsv1alpha1.NodePodProbe{}, + }, + } + + for _, tc := range testCase { + t.Run(tc.name, func(t *testing.T) { + fakeClient := fake.NewClientBuilder().WithScheme(scheme).Build() + // create pod + err := fakeClient.Create(context.TODO(), tc.pod) + if err != nil { + t.Errorf("Failed to create pod: %s/%s, err: %v", tc.pod.Namespace, tc.pod.Name, err) + } + // create nodeProbe + err = fakeClient.Create(context.TODO(), tc.podNodeProbe) + if err != nil { + t.Errorf("Failed to create node pod probe: %s, err: %v", tc.podNodeProbe.Name, err) + } + recon := ReconcileEnhancedLivenessProbe2NodeProbe{Client: fakeClient} + err = recon.delNodeProbeConfig(tc.pod) + if err != nil { + t.Errorf("Failed to retry on conflict del node probe config, err: %v", err) + } + getNodeProbe := appsv1alpha1.NodePodProbe{} + err = fakeClient.Get(context.TODO(), types.NamespacedName{Name: recon.GetPodNodeName(tc.pod)}, &getNodeProbe) + if err != nil { + if !errors.IsNotFound(err) { + t.Errorf("Failed to get node probe, err: %v", err) + } + } + if !checkNodePodProbeSpecEqual(tc.expectPodNodeProbe, &getNodeProbe) { + t.Errorf("No match, expect: %v, but: %v", util.DumpJSON(tc.expectPodNodeProbe), util.DumpJSON(getNodeProbe)) + } + }) + } +} diff --git a/pkg/controller/nodepodprobe/node_pod_probe_controller.go b/pkg/controller/nodepodprobe/node_pod_probe_controller.go index 4ff6792df3..42b6d139ae 100644 --- a/pkg/controller/nodepodprobe/node_pod_probe_controller.go +++ b/pkg/controller/nodepodprobe/node_pod_probe_controller.go @@ -22,14 +22,6 @@ import ( "reflect" "strings" - appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" - "github.com/openkruise/kruise/pkg/features" - "github.com/openkruise/kruise/pkg/util" - utilclient "github.com/openkruise/kruise/pkg/util/client" - "github.com/openkruise/kruise/pkg/util/controllerfinder" - utildiscovery "github.com/openkruise/kruise/pkg/util/discovery" - utilfeature "github.com/openkruise/kruise/pkg/util/feature" - "github.com/openkruise/kruise/pkg/util/ratelimiter" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" @@ -44,6 +36,15 @@ import ( "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" + + appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" + "github.com/openkruise/kruise/pkg/features" + "github.com/openkruise/kruise/pkg/util" + utilclient "github.com/openkruise/kruise/pkg/util/client" + "github.com/openkruise/kruise/pkg/util/controllerfinder" + utildiscovery "github.com/openkruise/kruise/pkg/util/discovery" + utilfeature "github.com/openkruise/kruise/pkg/util/feature" + "github.com/openkruise/kruise/pkg/util/ratelimiter" ) func init() { diff --git a/pkg/util/livenessprobe/livenessprobe_utils.go b/pkg/util/livenessprobe/livenessprobe_utils.go new file mode 100644 index 0000000000..30c92c574b --- /dev/null +++ b/pkg/util/livenessprobe/livenessprobe_utils.go @@ -0,0 +1,10 @@ +package livenessprobe + +import ( + v1 "k8s.io/api/core/v1" +) + +type ContainerLivenessProbe struct { + Name string `json:"name"` + LivenessProbe v1.Probe `json:"livenessProbe"` +} diff --git a/pkg/webhook/pod/mutating/enhancedlivenessprobe_handler.go b/pkg/webhook/pod/mutating/enhancedlivenessprobe_handler.go index 602592a0c6..2e09a17e86 100644 --- a/pkg/webhook/pod/mutating/enhancedlivenessprobe_handler.go +++ b/pkg/webhook/pod/mutating/enhancedlivenessprobe_handler.go @@ -12,13 +12,9 @@ import ( alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" "github.com/openkruise/kruise/pkg/util" + livenessprobeUtils "github.com/openkruise/kruise/pkg/util/livenessprobe" ) -type containerLivenessProbe struct { - Name string `json:"name"` - LivenessProbe v1.Probe `json:"livenessProbe"` -} - func (h *PodCreateHandler) enhancedLivenessProbeWhenPodCreate(ctx context.Context, req admission.Request, pod *v1.Pod) (skip bool, err error) { if len(req.AdmissionRequest.SubResource) > 0 || @@ -51,13 +47,13 @@ func (h *PodCreateHandler) enhancedLivenessProbeWhenPodCreate(ctx context.Contex // 1. the json string of the pod containers native livenessProbe configurations. // 2. the error reason of the function. func removeAndBackUpPodContainerLivenessProbe(pod *v1.Pod) (string, error) { - containersLivenessProbe := []containerLivenessProbe{} + containersLivenessProbe := []livenessprobeUtils.ContainerLivenessProbe{} for index := range pod.Spec.Containers { getContainer := &pod.Spec.Containers[index] if getContainer.LivenessProbe == nil { continue } - containersLivenessProbe = append(containersLivenessProbe, containerLivenessProbe{ + containersLivenessProbe = append(containersLivenessProbe, livenessprobeUtils.ContainerLivenessProbe{ Name: getContainer.Name, LivenessProbe: *getContainer.LivenessProbe, }) diff --git a/test/e2e/apps/imagelistpulljobs.go b/test/e2e/apps/imagelistpulljobs.go index 885bf18d27..fddd88f93b 100644 --- a/test/e2e/apps/imagelistpulljobs.go +++ b/test/e2e/apps/imagelistpulljobs.go @@ -111,7 +111,7 @@ var _ = SIGDescribe("PullImages", func() { CompletionPolicy: appsv1alpha1.CompletionPolicy{ Type: appsv1alpha1.Always, ActiveDeadlineSeconds: utilpointer.Int64Ptr(50), - TTLSecondsAfterFinished: utilpointer.Int32Ptr(20), + TTLSecondsAfterFinished: utilpointer.Int32Ptr(40), }, }, } diff --git a/test/e2e/apps/livenessprobemapnodeprobe.go b/test/e2e/apps/livenessprobemapnodeprobe.go new file mode 100644 index 0000000000..c06c692ac6 --- /dev/null +++ b/test/e2e/apps/livenessprobemapnodeprobe.go @@ -0,0 +1,89 @@ +package apps + +import ( + "encoding/json" + "fmt" + "time" + + "github.com/onsi/ginkgo" + "github.com/onsi/gomega" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/rand" + clientset "k8s.io/client-go/kubernetes" + + alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" + appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" + kruiseclientset "github.com/openkruise/kruise/pkg/client/clientset/versioned" + livenessprobeUtils "github.com/openkruise/kruise/pkg/util/livenessprobe" + "github.com/openkruise/kruise/test/e2e/framework" +) + +// e2e test for enhanced liveness probe map node probe +var _ = SIGDescribe("EnhancedLivenessProbeMapNodeProbe", func() { + f := framework.NewDefaultFramework("enhanced_livenessprobe_map_modeprobe") + var ns string + var c clientset.Interface + var kc kruiseclientset.Interface + var tester *framework.ELivenessProbeMapNodeProbeTester + var pods []*v1.Pod + var randStr string + + ginkgo.BeforeEach(func() { + c = f.ClientSet + kc = f.KruiseClientSet + ns = f.Namespace.Name + tester = framework.NewELivenessProbeMapNodeProbeTester(c, kc, ns) + randStr = rand.String(10) + }) + + ginkgo.AfterEach(func() { + err := tester.CleanAllTestResources() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }) + + framework.KruiseDescribe("Create pod and check the related nodePodProbe object", func() { + containerLivenessProbeConfig := v1.Probe{ + ProbeHandler: v1.ProbeHandler{ + Exec: &v1.ExecAction{ + Command: []string{"/bin/sh", "-c", "/healthy.sh"}, + }, + }, + InitialDelaySeconds: 1000, + TimeoutSeconds: 5, + PeriodSeconds: 100, + SuccessThreshold: 2, + FailureThreshold: 3, + } + cName := "app" + containersProbe := []livenessprobeUtils.ContainerLivenessProbe{ + { + Name: cName, + LivenessProbe: containerLivenessProbeConfig, + }, + } + containerProbeConfigRaw, err := json.Marshal(containersProbe) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + ginkgo.By("Create CloneSet and wait 1 pod ready") + pods = tester.CreateTestCloneSetAndGetPods(randStr, 1, map[string]string{ + alpha1.AnnotationUsingEnhancedLiveness: "true", + }, nil, []v1.Container{ + { + Name: cName, + Image: WebserverImage, + LivenessProbe: &containerLivenessProbeConfig, + }, + }) + ginkgo.By(fmt.Sprintf("Assert pod annotation %v", alpha1.AnnotationNativeContainerProbeContext)) + gomega.Eventually(func() string { + return pods[0].Annotations[alpha1.AnnotationNativeContainerProbeContext] + }, 5*time.Second, 1*time.Second).Should(gomega.Equal(string(containerProbeConfigRaw))) + + ginkgo.By("Wait NodePodProbe create completion") + gomega.Eventually(func() *appsv1alpha1.NodePodProbe { + gotNodePodProbe, err := tester.GetNodePodProbe(pods[0].Spec.NodeName) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + return gotNodePodProbe + }, 70*time.Second, time.Second).ShouldNot(gomega.BeNil()) + }) +}) diff --git a/test/e2e/apps/pullimages.go b/test/e2e/apps/pullimages.go index 3596e27b72..3d12f5e350 100644 --- a/test/e2e/apps/pullimages.go +++ b/test/e2e/apps/pullimages.go @@ -394,12 +394,12 @@ var _ = SIGDescribe("PullImage", func() { return job.Status.Desired }, 3*time.Second, time.Second).Should(gomega.Equal(int32(1))) - ginkgo.By("Wait completed in 180s") + ginkgo.By("Wait completed in 2700s") gomega.Eventually(func() bool { job, err = testerForImagePullJob.GetJob(job) gomega.Expect(err).NotTo(gomega.HaveOccurred()) return job.Status.CompletionTime != nil - }, 180*time.Second, 3*time.Second).Should(gomega.Equal(true)) + }, 270*time.Second, 3*time.Second).Should(gomega.Equal(true)) gomega.Expect(job.Status.Succeeded).To(gomega.Equal(int32(1))) ginkgo.By("Delete job") diff --git a/test/e2e/framework/livenessprobemapnodeprobe_utils.go b/test/e2e/framework/livenessprobemapnodeprobe_utils.go new file mode 100644 index 0000000000..76517d586c --- /dev/null +++ b/test/e2e/framework/livenessprobemapnodeprobe_utils.go @@ -0,0 +1,98 @@ +package framework + +import ( + "context" + "time" + + "github.com/onsi/gomega" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clientset "k8s.io/client-go/kubernetes" + + appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" + kruiseclientset "github.com/openkruise/kruise/pkg/client/clientset/versioned" +) + +type ELivenessProbeMapNodeProbeTester struct { + c clientset.Interface + kc kruiseclientset.Interface + ns string +} + +func NewELivenessProbeMapNodeProbeTester(c clientset.Interface, kc kruiseclientset.Interface, ns string) *ELivenessProbeMapNodeProbeTester { + return &ELivenessProbeMapNodeProbeTester{ + c: c, + kc: kc, + ns: ns, + } +} + +func (t *ELivenessProbeMapNodeProbeTester) CreateTestCloneSetAndGetPods(randStr string, replicas int32, annotations, labels map[string]string, containers []v1.Container) (pods []*v1.Pod) { + set := &appsv1alpha1.CloneSet{ + ObjectMeta: metav1.ObjectMeta{Namespace: t.ns, Name: "clone-foo-" + randStr}, + Spec: appsv1alpha1.CloneSetSpec{ + Replicas: &replicas, + Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"rand": randStr}}, + Template: v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{"rand": randStr}, + }, + Spec: v1.PodSpec{ + Affinity: &v1.Affinity{ + PodAntiAffinity: &v1.PodAntiAffinity{ + PreferredDuringSchedulingIgnoredDuringExecution: []v1.WeightedPodAffinityTerm{ + { + Weight: 100, + PodAffinityTerm: v1.PodAffinityTerm{TopologyKey: v1.LabelHostname, LabelSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"rand": randStr}}}, + }, + }, + }, + }, + Containers: containers, + }, + }, + }, + } + if len(annotations) != 0 { + set.Spec.Template.Annotations = annotations + } + if len(labels) != 0 { + for lk, lv := range labels { + set.Spec.Template.Labels[lk] = lv + } + } + + var err error + if _, err = t.kc.AppsV1alpha1().CloneSets(t.ns).Create(context.TODO(), set, metav1.CreateOptions{}); err != nil { + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + } + + // Wait for 60s + gomega.Eventually(func() int32 { + set, err = t.kc.AppsV1alpha1().CloneSets(t.ns).Get(context.TODO(), set.Name, metav1.GetOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + return set.Status.ReadyReplicas + }, 120*time.Second, 3*time.Second).Should(gomega.Equal(replicas)) + + podList, err := t.c.CoreV1().Pods(t.ns).List(context.TODO(), metav1.ListOptions{LabelSelector: "rand=" + randStr}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + for i := range podList.Items { + p := &podList.Items[i] + pods = append(pods, p) + } + return +} + +func (t *ELivenessProbeMapNodeProbeTester) GetNodePodProbe(name string) (*appsv1alpha1.NodePodProbe, error) { + return t.kc.AppsV1alpha1().NodePodProbes().Get(context.TODO(), name, metav1.GetOptions{}) +} + +func (t *ELivenessProbeMapNodeProbeTester) CleanAllTestResources() error { + if err := t.kc.AppsV1alpha1().NodePodProbes().DeleteCollection(context.TODO(), metav1.DeleteOptions{}, metav1.ListOptions{}); err != nil { + return err + } + if err := t.kc.AppsV1alpha1().CloneSets(t.ns).DeleteCollection(context.TODO(), metav1.DeleteOptions{}, metav1.ListOptions{}); err != nil { + return err + } + return nil +}