diff --git a/pkg/health/health.go b/pkg/health/health.go index 3c39cb6..124c427 100644 --- a/pkg/health/health.go +++ b/pkg/health/health.go @@ -5,6 +5,15 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" ) +type Health string + +const ( + HealthHealthy Health = "healthy" + HealthUnhealthy Health = "unhealthy" + HealthUnknown Health = "unknown" + HealthWarning Health = "warning" +) + // Represents resource health status type HealthStatusCode string @@ -155,13 +164,6 @@ func GetHealthCheckFunc(gvk schema.GroupVersionKind) func(obj *unstructured.Unst case "HelmRepository", "GitRepository": return getFluxRepositoryHealth } - - // case "apiregistration.k8s.io": - // switch gvk.Kind { - // case APIServiceKind: - // return getAPIServiceHealth - // } - case "networking.k8s.io": switch gvk.Kind { case IngressKind: diff --git a/pkg/health/health_argo.go b/pkg/health/health_argo.go index 97a8b85..0e496f6 100644 --- a/pkg/health/health_argo.go +++ b/pkg/health/health_argo.go @@ -33,14 +33,16 @@ func GetArgoWorkflowHealth(obj *unstructured.Unstructured) (*HealthStatus, error return nil, err } switch wf.Status.Phase { - case "", nodePending, nodeRunning: - return &HealthStatus{Status: HealthStatusProgressing, Message: wf.Status.Message}, nil + case "", nodePending: + return &HealthStatus{Health: HealthHealthy, Status: HealthStatusProgressing, Message: wf.Status.Message}, nil + case nodeRunning: + return &HealthStatus{Ready: true, Health: HealthHealthy, Status: HealthStatusProgressing, Message: wf.Status.Message}, nil case nodeSucceeded: - return &HealthStatus{Status: HealthStatusHealthy, Message: wf.Status.Message}, nil + return &HealthStatus{Ready: true, Health: HealthHealthy, Status: HealthStatusHealthy, Message: wf.Status.Message}, nil case nodeFailed, nodeError: - return &HealthStatus{Status: HealthStatusDegraded, Message: wf.Status.Message}, nil + return &HealthStatus{Health: HealthUnhealthy, Status: HealthStatusDegraded, Message: wf.Status.Message}, nil } - return &HealthStatus{Status: HealthStatusUnknown, Message: wf.Status.Message}, nil + return &HealthStatus{Health: HealthUnknown, Status: HealthStatusUnknown, Message: wf.Status.Message}, nil } // An agnostic workflow object only considers Status.Phase and Status.Message. It is agnostic to the API version or any @@ -59,17 +61,18 @@ func getArgoApplicationHealth(obj *unstructured.Unstructured) (*HealthStatus, er switch app.Status.Health.Status { case HealthStatusProgressing: - return &HealthStatus{Status: HealthStatusProgressing, Message: app.Status.Health.Message}, nil + return &HealthStatus{Health: HealthHealthy, Status: HealthStatusProgressing, Message: app.Status.Health.Message}, nil case HealthStatusHealthy: - return &HealthStatus{Status: HealthStatusHealthy, Message: app.Status.Health.Message}, nil + return &HealthStatus{Ready: true, Health: HealthHealthy, Status: HealthStatusHealthy, Message: app.Status.Health.Message}, nil case HealthStatusSuspended: - return &HealthStatus{Status: HealthStatusSuspended, Message: app.Status.Health.Message}, nil + return &HealthStatus{Health: HealthHealthy, Status: HealthStatusSuspended, Message: app.Status.Health.Message}, nil case HealthStatusDegraded: - return &HealthStatus{Status: HealthStatusDegraded, Message: app.Status.Health.Message}, nil + return &HealthStatus{Health: HealthUnhealthy, Status: HealthStatusDegraded, Message: app.Status.Health.Message}, nil case HealthStatusMissing: - return &HealthStatus{Status: HealthStatusMissing, Message: app.Status.Health.Message}, nil + return &HealthStatus{Health: HealthUnhealthy, Status: HealthStatusMissing, Message: app.Status.Health.Message}, nil case HealthStatusUnknown: - return &HealthStatus{Status: HealthStatusUnknown, Message: app.Status.Health.Message}, nil + return &HealthStatus{Health: HealthUnknown, Status: HealthStatusUnknown, Message: app.Status.Health.Message}, nil } - return &HealthStatus{Status: HealthStatusUnknown, Message: app.Status.Health.Message}, nil + + return &HealthStatus{Health: HealthUnknown, Status: HealthStatusUnknown, Message: app.Status.Health.Message}, nil } diff --git a/pkg/health/health_daemonset.go b/pkg/health/health_daemonset.go index 9bf252e..ecc56b7 100644 --- a/pkg/health/health_daemonset.go +++ b/pkg/health/health_daemonset.go @@ -28,24 +28,31 @@ func getAppsv1DaemonSetHealth(daemon *appsv1.DaemonSet) (*HealthStatus, error) { if daemon.Generation <= daemon.Status.ObservedGeneration { if daemon.Spec.UpdateStrategy.Type == appsv1.OnDeleteDaemonSetStrategyType { return &HealthStatus{ + Health: HealthHealthy, + Ready: daemon.Status.NumberAvailable != 0, Status: HealthStatusHealthy, Message: fmt.Sprintf("daemon set %d out of %d new pods have been updated", daemon.Status.UpdatedNumberScheduled, daemon.Status.DesiredNumberScheduled), }, nil } if daemon.Status.UpdatedNumberScheduled < daemon.Status.DesiredNumberScheduled { return &HealthStatus{ + Health: HealthHealthy, + Ready: daemon.Status.NumberAvailable != 0, Status: HealthStatusProgressing, Message: fmt.Sprintf("Waiting for daemon set %q rollout to finish: %d out of %d new pods have been updated...", daemon.Name, daemon.Status.UpdatedNumberScheduled, daemon.Status.DesiredNumberScheduled), }, nil } if daemon.Status.NumberAvailable < daemon.Status.DesiredNumberScheduled { return &HealthStatus{ + Health: HealthHealthy, + Ready: daemon.Status.NumberAvailable != 0, Status: HealthStatusProgressing, Message: fmt.Sprintf("Waiting for daemon set %q rollout to finish: %d of %d updated pods are available...", daemon.Name, daemon.Status.NumberAvailable, daemon.Status.DesiredNumberScheduled), }, nil } } else { return &HealthStatus{ + Health: HealthUnhealthy, Status: HealthStatusProgressing, Message: "Waiting for rollout to finish: observed daemon set generation less than desired generation", }, nil diff --git a/pkg/health/health_deployment.go b/pkg/health/health_deployment.go index 8bbccb0..d03f501 100644 --- a/pkg/health/health_deployment.go +++ b/pkg/health/health_deployment.go @@ -30,38 +30,50 @@ func getAppsv1DeploymentHealth(deployment *appsv1.Deployment) (*HealthStatus, er Message: "Deployment is paused", }, nil } + // Borrowed at kubernetes/kubectl/rollout_status.go https://github.com/kubernetes/kubernetes/blob/5232ad4a00ec93942d0b2c6359ee6cd1201b46bc/pkg/kubectl/rollout_status.go#L80 if deployment.Generation <= deployment.Status.ObservedGeneration { cond := getAppsv1DeploymentCondition(deployment.Status, appsv1.DeploymentProgressing) if cond != nil && cond.Reason == "ProgressDeadlineExceeded" { return &HealthStatus{ + Health: HealthWarning, Status: HealthStatusDegraded, Message: fmt.Sprintf("Deployment %q exceeded its progress deadline", deployment.Name), }, nil } else if deployment.Spec.Replicas != nil && deployment.Status.UpdatedReplicas < *deployment.Spec.Replicas { return &HealthStatus{ - Status: HealthStatusProgressing, + Ready: *deployment.Spec.Replicas != 0, + Health: HealthHealthy, Message: fmt.Sprintf("Waiting for rollout to finish: %d out of %d new replicas have been updated...", deployment.Status.UpdatedReplicas, *deployment.Spec.Replicas), + Status: HealthStatusProgressing, }, nil } else if deployment.Status.Replicas > deployment.Status.UpdatedReplicas { return &HealthStatus{ + Ready: *deployment.Spec.Replicas != 0, Status: HealthStatusProgressing, + Health: HealthHealthy, Message: fmt.Sprintf("Waiting for rollout to finish: %d old replicas are pending termination...", deployment.Status.Replicas-deployment.Status.UpdatedReplicas), }, nil } else if deployment.Status.AvailableReplicas < deployment.Status.UpdatedReplicas { return &HealthStatus{ + Ready: *deployment.Spec.Replicas != 0, Status: HealthStatusProgressing, + Health: HealthHealthy, Message: fmt.Sprintf("Waiting for rollout to finish: %d of %d updated replicas are available...", deployment.Status.AvailableReplicas, deployment.Status.UpdatedReplicas), }, nil } } else { return &HealthStatus{ + Ready: *deployment.Spec.Replicas != 0, Status: HealthStatusProgressing, + Health: HealthHealthy, Message: "Waiting for rollout to finish: observed deployment generation less than desired generation", }, nil } return &HealthStatus{ + Ready: *deployment.Spec.Replicas != 0, + Health: HealthHealthy, Status: HealthStatusHealthy, }, nil } diff --git a/pkg/health/health_flux.go b/pkg/health/health_flux.go index a0c29bb..867ad7b 100644 --- a/pkg/health/health_flux.go +++ b/pkg/health/health_flux.go @@ -47,29 +47,30 @@ func getFluxKustomizationHealth(obj *unstructured.Unstructured) (*HealthStatus, msg := fmt.Sprintf("%s: %s", c.Reason, c.Message) if c.Type == fluxHealthy { if c.Status == v1.ConditionTrue { - return &HealthStatus{Status: HealthStatusHealthy, Message: msg}, nil + return &HealthStatus{Health: HealthHealthy, Status: HealthStatusHealthy, Message: msg}, nil } else { - return &HealthStatus{Status: HealthStatusDegraded, Message: msg}, nil + return &HealthStatus{Health: HealthHealthy, Status: HealthStatusDegraded, Message: msg}, nil } } if c.Type == fluxReady { if c.Status == v1.ConditionTrue { - return &HealthStatus{Status: HealthStatusHealthy, Message: msg}, nil + return &HealthStatus{Ready: true, Health: HealthHealthy, Status: HealthStatusHealthy, Message: msg}, nil } else { - return &HealthStatus{Status: HealthStatusDegraded, Message: msg}, nil + return &HealthStatus{Health: HealthUnhealthy, Status: HealthStatusDegraded, Message: msg}, nil } } // All conditions apart from Healthy/Ready should be false if c.Status == v1.ConditionTrue { return &HealthStatus{ + Health: HealthUnhealthy, Status: HealthStatusDegraded, Message: msg, }, nil } } - return &HealthStatus{Status: HealthStatusUnknown, Message: ""}, nil + return &HealthStatus{Health: HealthUnknown, Status: HealthStatusUnknown, Message: ""}, nil } type helmStatusType string @@ -109,29 +110,30 @@ func getFluxHelmReleaseHealth(obj *unstructured.Unstructured) (*HealthStatus, er msg := fmt.Sprintf("%s: %s", c.Reason, c.Message) if c.Type == helmReleased { if c.Status == v1.ConditionTrue { - return &HealthStatus{Status: HealthStatusHealthy, Message: msg}, nil + return &HealthStatus{Health: HealthHealthy, Status: HealthStatusHealthy, Message: msg}, nil } else { - return &HealthStatus{Status: HealthStatusDegraded, Message: msg}, nil + return &HealthStatus{Health: HealthHealthy, Status: HealthStatusDegraded, Message: msg}, nil } } if c.Type == helmReady { if c.Status == v1.ConditionTrue { - return &HealthStatus{Status: HealthStatusHealthy, Message: msg}, nil + return &HealthStatus{Ready: true, Health: HealthHealthy, Status: HealthStatusHealthy, Message: msg}, nil } else { - return &HealthStatus{Status: HealthStatusDegraded, Message: msg}, nil + return &HealthStatus{Health: HealthUnhealthy, Status: HealthStatusDegraded, Message: msg}, nil } } // All conditions apart from Healthy/Ready should be false if c.Status == v1.ConditionTrue { return &HealthStatus{ + Health: HealthUnhealthy, Status: HealthStatusDegraded, Message: msg, }, nil } } - return &HealthStatus{Status: HealthStatusUnknown, Message: ""}, nil + return &HealthStatus{Health: HealthUnknown, Status: HealthStatusUnknown, Message: ""}, nil } type fluxRepoStatusType string @@ -174,19 +176,21 @@ func getFluxRepositoryHealth(obj *unstructured.Unstructured) (*HealthStatus, err msg := fmt.Sprintf("%s: %s", c.Reason, c.Message) if c.Type == fluxRepoReady { if c.Status == v1.ConditionTrue { - return &HealthStatus{Status: HealthStatusHealthy, Message: msg}, nil + return &HealthStatus{Ready: true, Health: HealthHealthy, Status: HealthStatusHealthy, Message: msg}, nil } else { - return &HealthStatus{Status: HealthStatusDegraded, Message: msg}, nil + return &HealthStatus{Health: HealthHealthy, Status: HealthStatusDegraded, Message: msg}, nil } } // All conditions apart from Healthy/Ready should be false if c.Status == v1.ConditionTrue { return &HealthStatus{ + Health: HealthUnhealthy, Status: HealthStatusDegraded, Message: msg, }, nil } } - return &HealthStatus{Status: HealthStatusUnknown, Message: ""}, nil + + return &HealthStatus{Health: HealthUnknown, Status: HealthStatusUnknown, Message: ""}, nil } diff --git a/pkg/health/health_hpa.go b/pkg/health/health_hpa.go index 749df85..6e58319 100644 --- a/pkg/health/health_hpa.go +++ b/pkg/health/health_hpa.go @@ -14,6 +14,7 @@ import ( var ( progressingStatus = &HealthStatus{ + Health: HealthHealthy, Status: HealthStatusProgressing, Message: "Waiting to Autoscale", } @@ -133,6 +134,7 @@ func checkConditions(conditions []hpaCondition, progressingStatus *HealthStatus) for _, condition := range conditions { if isDegraded(&condition) { return &HealthStatus{ + Health: HealthUnhealthy, Status: HealthStatusDegraded, Message: condition.Message, }, nil @@ -140,6 +142,7 @@ func checkConditions(conditions []hpaCondition, progressingStatus *HealthStatus) if isHealthy(&condition) { return &HealthStatus{ + Health: HealthHealthy, Status: HealthStatusHealthy, Message: condition.Message, }, nil diff --git a/pkg/health/health_ingress.go b/pkg/health/health_ingress.go index ee697f9..89ebefb 100644 --- a/pkg/health/health_ingress.go +++ b/pkg/health/health_ingress.go @@ -11,10 +11,13 @@ func getIngressHealth(obj *unstructured.Unstructured) (*HealthStatus, error) { if err != nil { return &HealthStatus{Status: HealthStatusError, Message: fmt.Sprintf("failed to ingress status: %v", err)}, nil } else if !found { - return &HealthStatus{Status: HealthStatusPending, Message: "ingress loadbalancer status not found"}, nil + return &HealthStatus{Health: HealthHealthy, Status: HealthStatusPending, Message: "ingress loadbalancer status not found"}, nil } - health := HealthStatus{} + health := HealthStatus{ + // Ready: false, // not possible to decide this from the information available + Health: HealthHealthy, + } if len(ingresses) > 0 { health.Status = HealthStatusHealthy } else { diff --git a/pkg/health/health_job.go b/pkg/health/health_job.go index 9565069..a692133 100644 --- a/pkg/health/health_job.go +++ b/pkg/health/health_job.go @@ -2,6 +2,7 @@ package health import ( "fmt" + corev1 "k8s.io/api/core/v1" batchv1 "k8s.io/api/batch/v1" @@ -25,11 +26,15 @@ func getJobHealth(obj *unstructured.Unstructured) (*HealthStatus, error) { } func getBatchv1JobHealth(job *batchv1.Job) (*HealthStatus, error) { - failed := false - var failMsg string - complete := false - var message string - isSuspended := false + var ( + complete bool + failed bool + isSuspended bool + + message string + failMsg string + ) + for _, condition := range job.Status.Conditions { switch condition.Type { case batchv1.JobFailed: @@ -40,31 +45,36 @@ func getBatchv1JobHealth(job *batchv1.Job) (*HealthStatus, error) { complete = true message = condition.Message case batchv1.JobSuspended: - complete = true message = condition.Message if condition.Status == corev1.ConditionTrue { isSuspended = true } } } - if !complete { + + if !complete && !isSuspended { return &HealthStatus{ + Health: HealthHealthy, Status: HealthStatusProgressing, Message: message, }, nil } else if failed { return &HealthStatus{ + Health: HealthUnhealthy, Status: HealthStatusDegraded, Message: failMsg, }, nil } else if isSuspended { return &HealthStatus{ + Health: HealthHealthy, Status: HealthStatusSuspended, Message: failMsg, }, nil } else { return &HealthStatus{ + Ready: true, Status: HealthStatusHealthy, + Health: HealthHealthy, Message: message, }, nil } diff --git a/pkg/health/health_namespace.go b/pkg/health/health_namespace.go index e7ab9b4..65ca761 100644 --- a/pkg/health/health_namespace.go +++ b/pkg/health/health_namespace.go @@ -16,11 +16,14 @@ func getNamespaceHealth(obj *unstructured.Unstructured) (*HealthStatus, error) { if node.Status.Phase == v1.NamespaceActive { return &HealthStatus{ + Ready: true, + Health: HealthHealthy, Status: HealthStatusHealthy, }, nil } return &HealthStatus{ + Health: HealthUnhealthy, Status: HealthStatusDeleting, }, nil } diff --git a/pkg/health/health_node.go b/pkg/health/health_node.go index 43b8087..95aa64f 100644 --- a/pkg/health/health_node.go +++ b/pkg/health/health_node.go @@ -12,12 +12,14 @@ import ( func getNodeHealth(obj *unstructured.Unstructured) (*HealthStatus, error) { var node v1.Node if err := runtime.DefaultUnstructuredConverter.FromUnstructured(obj.Object, &node); err != nil { - return nil, fmt.Errorf("failed to convert unstructured Node to typed: %v", err) } + for _, cond := range node.Status.Conditions { if cond.Type == v1.NodeReady && cond.Status == v1.ConditionTrue { return &HealthStatus{ + Ready: true, + Health: HealthHealthy, Status: HealthStatusHealthy, }, nil } @@ -30,5 +32,6 @@ func getNodeHealth(obj *unstructured.Unstructured) (*HealthStatus, error) { }, nil } } + return nil, errors.New("no conditions matched for node status") } diff --git a/pkg/health/health_pod.go b/pkg/health/health_pod.go index 6c80c1d..7b5815f 100644 --- a/pkg/health/health_pod.go +++ b/pkg/health/health_pod.go @@ -33,12 +33,14 @@ func getCorev1PodHealth(pod *corev1.Pod) (*HealthStatus, error) { // completed. if pod.Spec.RestartPolicy == corev1.RestartPolicyAlways { var status HealthStatusCode + var health Health var messages []string for _, containerStatus := range pod.Status.ContainerStatuses { waiting := containerStatus.State.Waiting // Article listing common container errors: https://medium.com/kokster/debugging-crashloopbackoffs-with-init-containers-26f79e9fb5bf if waiting != nil && (strings.HasPrefix(waiting.Reason, "Err") || strings.HasSuffix(waiting.Reason, "Error") || strings.HasSuffix(waiting.Reason, "BackOff")) { + health = HealthUnhealthy status = HealthStatusDegraded messages = append(messages, waiting.Message) } @@ -46,6 +48,7 @@ func getCorev1PodHealth(pod *corev1.Pod) (*HealthStatus, error) { if status != "" { return &HealthStatus{ + Health: health, Status: status, Message: strings.Join(messages, ", "), }, nil @@ -70,32 +73,36 @@ func getCorev1PodHealth(pod *corev1.Pod) (*HealthStatus, error) { switch pod.Status.Phase { case corev1.PodPending: return &HealthStatus{ + Health: HealthHealthy, Status: HealthStatusProgressing, Message: pod.Status.Message, }, nil case corev1.PodSucceeded: return &HealthStatus{ + Health: HealthHealthy, Status: HealthStatusHealthy, Message: pod.Status.Message, }, nil case corev1.PodFailed: if pod.Status.Message != "" { // Pod has a nice error message. Use that. - return &HealthStatus{Status: HealthStatusDegraded, Message: pod.Status.Message}, nil + return &HealthStatus{Health: HealthUnhealthy, Status: HealthStatusDegraded, Message: pod.Status.Message}, nil } for _, ctr := range append(pod.Status.InitContainerStatuses, pod.Status.ContainerStatuses...) { if msg := getFailMessage(&ctr); msg != "" { - return &HealthStatus{Status: HealthStatusDegraded, Message: msg}, nil + return &HealthStatus{Health: HealthUnhealthy, Status: HealthStatusDegraded, Message: msg}, nil } } - return &HealthStatus{Status: HealthStatusDegraded, Message: ""}, nil + return &HealthStatus{Health: HealthUnhealthy, Status: HealthStatusDegraded, Message: ""}, nil case corev1.PodRunning: switch pod.Spec.RestartPolicy { case corev1.RestartPolicyAlways: // if pod is ready, it is automatically healthy if IsPodReady(pod) { return &HealthStatus{ + Health: HealthHealthy, + Ready: true, Status: HealthStatusHealthy, Message: pod.Status.Message, }, nil @@ -104,6 +111,7 @@ func getCorev1PodHealth(pod *corev1.Pod) (*HealthStatus, error) { for _, ctrStatus := range pod.Status.ContainerStatuses { if ctrStatus.LastTerminationState.Terminated != nil { return &HealthStatus{ + Health: HealthUnhealthy, Status: HealthStatusDegraded, Message: pod.Status.Message, }, nil @@ -111,6 +119,7 @@ func getCorev1PodHealth(pod *corev1.Pod) (*HealthStatus, error) { } // otherwise we are progressing towards a ready state return &HealthStatus{ + Health: HealthHealthy, Status: HealthStatusProgressing, Message: pod.Status.Message, }, nil @@ -119,12 +128,15 @@ func getCorev1PodHealth(pod *corev1.Pod) (*HealthStatus, error) { // These pods are typically resource hooks. Thus, we consider these as Progressing // instead of healthy. return &HealthStatus{ + Health: HealthHealthy, Status: HealthStatusProgressing, Message: pod.Status.Message, }, nil } } + return &HealthStatus{ + Health: HealthUnknown, Status: HealthStatusUnknown, Message: pod.Status.Message, }, nil diff --git a/pkg/health/health_pvc.go b/pkg/health/health_pvc.go index 5e934e1..ca5caac 100644 --- a/pkg/health/health_pvc.go +++ b/pkg/health/health_pvc.go @@ -24,16 +24,20 @@ func getPVCHealth(obj *unstructured.Unstructured) (*HealthStatus, error) { } func getCorev1PVCHealth(pvc *corev1.PersistentVolumeClaim) (*HealthStatus, error) { - var status HealthStatusCode + health := HealthStatus{Health: HealthHealthy} switch pvc.Status.Phase { case corev1.ClaimLost: - status = HealthStatusDegraded + health.Health = HealthUnhealthy + health.Status = HealthStatusDegraded case corev1.ClaimPending: - status = HealthStatusProgressing + health.Status = HealthStatusProgressing case corev1.ClaimBound: - status = HealthStatusHealthy + health.Ready = true + health.Status = HealthStatusHealthy default: - status = HealthStatusUnknown + health.Health = HealthUnknown + health.Status = HealthStatusUnknown } - return &HealthStatus{Status: status}, nil + + return &health, nil } diff --git a/pkg/health/health_replicaset.go b/pkg/health/health_replicaset.go index 5435f20..4220a0a 100644 --- a/pkg/health/health_replicaset.go +++ b/pkg/health/health_replicaset.go @@ -26,20 +26,25 @@ func getReplicaSetHealth(obj *unstructured.Unstructured) (*HealthStatus, error) func getAppsv1ReplicaSetHealth(replicaSet *appsv1.ReplicaSet) (*HealthStatus, error) { if replicaSet.Generation <= replicaSet.Status.ObservedGeneration { - cond := getAppsv1ReplicaSetCondition(replicaSet.Status, appsv1.ReplicaSetReplicaFailure) - if cond != nil && cond.Status == corev1.ConditionTrue { + failCondition := getAppsv1ReplicaSetCondition(replicaSet.Status, appsv1.ReplicaSetReplicaFailure) + if failCondition != nil && failCondition.Status == corev1.ConditionTrue { return &HealthStatus{ + Health: HealthUnhealthy, Status: HealthStatusDegraded, - Message: cond.Message, + Message: failCondition.Message, }, nil } else if replicaSet.Spec.Replicas != nil && replicaSet.Status.AvailableReplicas < *replicaSet.Spec.Replicas { return &HealthStatus{ + Ready: replicaSet.Status.AvailableReplicas != 0, + Health: HealthHealthy, Status: HealthStatusProgressing, Message: fmt.Sprintf("Waiting for rollout to finish: %d out of %d new replicas are available...", replicaSet.Status.AvailableReplicas, *replicaSet.Spec.Replicas), }, nil } } else { return &HealthStatus{ + Ready: replicaSet.Status.AvailableReplicas != 0, + Health: HealthHealthy, Status: HealthStatusProgressing, Message: "Waiting for rollout to finish: observed replica set generation less than desired generation", }, nil diff --git a/pkg/health/health_service.go b/pkg/health/health_service.go index 5e7b124..f09c703 100644 --- a/pkg/health/health_service.go +++ b/pkg/health/health_service.go @@ -24,7 +24,7 @@ func getServiceHealth(obj *unstructured.Unstructured) (*HealthStatus, error) { } func getCorev1ServiceHealth(service *corev1.Service) (*HealthStatus, error) { - health := HealthStatus{Status: HealthStatusHealthy} + health := HealthStatus{Health: HealthHealthy, Status: HealthStatusHealthy} if service.Spec.Type == corev1.ServiceTypeLoadBalancer { if len(service.Status.LoadBalancer.Ingress) > 0 { health.Status = HealthStatusHealthy diff --git a/pkg/health/health_statefulset.go b/pkg/health/health_statefulset.go index 30e4f7d..77cfc0d 100644 --- a/pkg/health/health_statefulset.go +++ b/pkg/health/health_statefulset.go @@ -27,12 +27,14 @@ func getAppsv1StatefulSetHealth(sts *appsv1.StatefulSet) (*HealthStatus, error) // Borrowed at kubernetes/kubectl/rollout_status.go https://github.com/kubernetes/kubernetes/blob/5232ad4a00ec93942d0b2c6359ee6cd1201b46bc/pkg/kubectl/rollout_status.go#L131 if sts.Status.ObservedGeneration == 0 || sts.Generation > sts.Status.ObservedGeneration { return &HealthStatus{ + Health: HealthHealthy, Status: HealthStatusProgressing, Message: "Waiting for statefulset spec update to be observed...", }, nil } if sts.Spec.Replicas != nil && sts.Status.ReadyReplicas < *sts.Spec.Replicas { return &HealthStatus{ + Health: HealthHealthy, Status: HealthStatusProgressing, Message: fmt.Sprintf("Waiting for %d pods to be ready...", *sts.Spec.Replicas-sts.Status.ReadyReplicas), }, nil @@ -41,6 +43,7 @@ func getAppsv1StatefulSetHealth(sts *appsv1.StatefulSet) (*HealthStatus, error) if sts.Spec.Replicas != nil && sts.Spec.UpdateStrategy.RollingUpdate.Partition != nil { if sts.Status.UpdatedReplicas < (*sts.Spec.Replicas - *sts.Spec.UpdateStrategy.RollingUpdate.Partition) { return &HealthStatus{ + Health: HealthHealthy, Status: HealthStatusProgressing, Message: fmt.Sprintf("Waiting for partitioned roll out to finish: %d out of %d new pods have been updated...", sts.Status.UpdatedReplicas, (*sts.Spec.Replicas - *sts.Spec.UpdateStrategy.RollingUpdate.Partition)), @@ -48,23 +51,31 @@ func getAppsv1StatefulSetHealth(sts *appsv1.StatefulSet) (*HealthStatus, error) } } return &HealthStatus{ + Ready: true, + Health: HealthHealthy, Status: HealthStatusHealthy, Message: fmt.Sprintf("partitioned roll out complete: %d new pods have been updated...", sts.Status.UpdatedReplicas), }, nil } if sts.Spec.UpdateStrategy.Type == appsv1.OnDeleteStatefulSetStrategyType { return &HealthStatus{ + Ready: true, + Health: HealthHealthy, Status: HealthStatusHealthy, Message: fmt.Sprintf("statefulset has %d ready pods", sts.Status.ReadyReplicas), }, nil } if sts.Status.UpdateRevision != sts.Status.CurrentRevision { return &HealthStatus{ + Health: HealthHealthy, + Ready: true, Status: HealthStatusProgressing, Message: fmt.Sprintf("waiting for statefulset rolling update to complete %d pods at revision %s...", sts.Status.UpdatedReplicas, sts.Status.UpdateRevision), }, nil } return &HealthStatus{ + Health: HealthHealthy, + Ready: true, Status: HealthStatusHealthy, Message: fmt.Sprintf("statefulset rolling update complete %d pods at revision %s...", sts.Status.CurrentReplicas, sts.Status.CurrentRevision), }, nil diff --git a/pkg/health/utils.go b/pkg/health/utils.go index 70b0323..a73cded 100644 --- a/pkg/health/utils.go +++ b/pkg/health/utils.go @@ -27,6 +27,8 @@ const ( ) type HealthStatus struct { + Ready bool `json:"ready"` + Health Health `json:"health"` // Status holds the status code of the application or resource Status HealthStatusCode `json:"status,omitempty" protobuf:"bytes,1,opt,name=status"` // Message is a human-readable informational message describing the health status