From 5db41dde93458e0517e11fa8a68a3438ef4a489e Mon Sep 17 00:00:00 2001 From: Thiago Gonzaga Date: Fri, 1 Oct 2021 18:38:21 -0300 Subject: [PATCH] PTX-3561 Torpedo reports 2/3 pods are ready, when all pods are ready Signed-off-by: Thiago Gonzaga --- k8s/apps/daemonsets.go | 26 ++++++++------------------ k8s/apps/deployments.go | 26 ++++++++------------------ k8s/apps/replicasets.go | 26 ++++++++------------------ k8s/apps/statefulsets.go | 20 +++++++++----------- 4 files changed, 33 insertions(+), 65 deletions(-) diff --git a/k8s/apps/daemonsets.go b/k8s/apps/daemonsets.go index 8649da2e..7f07e2f6 100644 --- a/k8s/apps/daemonsets.go +++ b/k8s/apps/daemonsets.go @@ -125,32 +125,22 @@ func (c *Client) ValidateDaemonSet(name, namespace string, timeout time.Duration } } - if ds.Status.DesiredNumberScheduled != ds.Status.NumberReady { - return "", true, &schederrors.ErrAppNotReady{ - ID: name, - Cause: fmt.Sprintf("Expected ready: %v Actual ready:%v Current pods overview:\n%s", - ds.Status.DesiredNumberScheduled, ds.Status.NumberReady, podsOverviewString), - } - } - - var notReadyPods []string var readyCount int32 for _, pod := range pods { - if !common.IsPodReady(pod) { - notReadyPods = append(notReadyPods, pod.Name) - } else { + if common.IsPodReady(pod) { readyCount++ } } - if readyCount == ds.Status.DesiredNumberScheduled { - return "", false, nil + if ds.Status.DesiredNumberScheduled != readyCount { + return "", true, &schederrors.ErrAppNotReady{ + ID: name, + Cause: fmt.Sprintf("Expected ready: %v Actual ready:%v Current pods overview:\n%s", + ds.Status.DesiredNumberScheduled, readyCount, podsOverviewString), + } } - return "", true, &schederrors.ErrAppNotReady{ - ID: ds.Name, - Cause: fmt.Sprintf("Pod(s): %#v not yet ready", notReadyPods), - } + return "", false, nil } if _, err := task.DoRetryWithTimeout(t, timeout, 15*time.Second); err != nil { diff --git a/k8s/apps/deployments.go b/k8s/apps/deployments.go index f15ef207..00ec0bf7 100644 --- a/k8s/apps/deployments.go +++ b/k8s/apps/deployments.go @@ -156,14 +156,6 @@ func (c *Client) ValidateDeployment(deployment *appsv1.Deployment, timeout, retr } } - if requiredReplicas > dep.Status.ReadyReplicas { - return "", true, &schederrors.ErrAppNotReady{ - ID: dep.Name, - Cause: fmt.Sprintf("Expected replicas: %v Ready replicas: %v Current pods overview:\n%s", - requiredReplicas, dep.Status.ReadyReplicas, podsOverviewString), - } - } - if requiredReplicas != dep.Status.UpdatedReplicas && shared { return "", true, &schederrors.ErrAppNotReady{ ID: dep.Name, @@ -173,24 +165,22 @@ func (c *Client) ValidateDeployment(deployment *appsv1.Deployment, timeout, retr } // look for "requiredReplicas" number of pods in ready state - var notReadyPods []string var readyCount int32 for _, pod := range pods { - if !common.IsPodReady(pod) { - notReadyPods = append(notReadyPods, pod.Name) - } else { + if common.IsPodReady(pod) { readyCount++ } } - if readyCount >= requiredReplicas { - return "", false, nil + if requiredReplicas > readyCount { + return "", true, &schederrors.ErrAppNotReady{ + ID: dep.Name, + Cause: fmt.Sprintf("Expected replicas: %v Ready replicas: %v Current pods overview:\n%s", + requiredReplicas, readyCount, podsOverviewString), + } } - return "", true, &schederrors.ErrAppNotReady{ - ID: dep.Name, - Cause: fmt.Sprintf("Pod(s): %#v not yet ready", notReadyPods), - } + return "", false, nil } if _, err := task.DoRetryWithTimeout(t, timeout, retryInterval); err != nil { diff --git a/k8s/apps/replicasets.go b/k8s/apps/replicasets.go index 46fc18c5..f0901d59 100644 --- a/k8s/apps/replicasets.go +++ b/k8s/apps/replicasets.go @@ -130,32 +130,22 @@ func (c *Client) ValidateReplicaSet(name, namespace string, timeout time.Duratio } } - if rs.Status.Replicas != rs.Status.ReadyReplicas { - return "", true, &schederrors.ErrAppNotReady{ - ID: name, - Cause: fmt.Sprintf("Expected ready: %v Actual ready:%v Current pods overview:\n%s", - rs.Status.Replicas, rs.Status.ReadyReplicas, podsOverviewString), - } - } - - var notReadyPods []string var readyCount int32 for _, pod := range pods { - if !common.IsPodReady(pod) { - notReadyPods = append(notReadyPods, pod.Name) - } else { + if common.IsPodReady(pod) { readyCount++ } } - if readyCount == rs.Status.Replicas { - return "", false, nil + if rs.Status.Replicas != readyCount { + return "", true, &schederrors.ErrAppNotReady{ + ID: name, + Cause: fmt.Sprintf("Expected ready: %v Actual ready:%v Current pods overview:\n%s", + rs.Status.Replicas, readyCount, podsOverviewString), + } } + return "", false, nil - return "", true, &schederrors.ErrAppNotReady{ - ID: rs.Name, - Cause: fmt.Sprintf("Pod(s): %#v not yet ready", notReadyPods), - } } if _, err := task.DoRetryWithTimeout(t, timeout, 15*time.Second); err != nil { diff --git a/k8s/apps/statefulsets.go b/k8s/apps/statefulsets.go index 68e3f313..6abf6459 100644 --- a/k8s/apps/statefulsets.go +++ b/k8s/apps/statefulsets.go @@ -138,20 +138,18 @@ func (c *Client) ValidateStatefulSet(statefulset *appsv1.StatefulSet, timeout ti } } - if *sset.Spec.Replicas != sset.Status.ReadyReplicas { - return "", true, &schederrors.ErrAppNotReady{ - ID: sset.Name, - Cause: fmt.Sprintf("Expected replicas: %v Ready replicas: %v Current pods overview:\n%s", - *sset.Spec.Replicas, sset.Status.ReadyReplicas, podsOverviewString), + var readyCount int32 + for _, pod := range pods { + if common.IsPodReady(pod) { + readyCount++ } } - for _, pod := range pods { - if !common.IsPodReady(pod) { - return "", true, &schederrors.ErrAppNotReady{ - ID: sset.Name, - Cause: fmt.Sprintf("Pod: %v is not yet ready", pod.Name), - } + if *sset.Spec.Replicas != readyCount { + return "", true, &schederrors.ErrAppNotReady{ + ID: sset.Name, + Cause: fmt.Sprintf("Expected replicas: %v Ready replicas: %v Current pods overview:\n%s", + *sset.Spec.Replicas, readyCount, podsOverviewString), } }