feat: add health & ready to healthStatus

pkg/health/health.go

@@ -5,6 +5,15 @@ import (
 	"k8s.io/apimachinery/pkg/runtime/schema"
 )
 
+type Health string
+
+const (
+	HealthHealthy   Health = "healthy"
+	HealthUnhealthy Health = "unhealthy"
+	HealthUnknown   Health = "unknown"
+	HealthWarning   Health = "warning"
+)
+
 // Represents resource health status
 type HealthStatusCode string
 
@@ -155,13 +164,6 @@ func GetHealthCheckFunc(gvk schema.GroupVersionKind) func(obj *unstructured.Unst
 		case "HelmRepository", "GitRepository":
 			return getFluxRepositoryHealth
 		}
-
-	// case "apiregistration.k8s.io":
-	// 	switch gvk.Kind {
-	// 	case APIServiceKind:
-	// 		return getAPIServiceHealth
-	// 	}
-
 	case "networking.k8s.io":
 		switch gvk.Kind {
 		case IngressKind:

pkg/health/health_argo.go

@@ -33,14 +33,16 @@ func GetArgoWorkflowHealth(obj *unstructured.Unstructured) (*HealthStatus, error
 		return nil, err
 	}
 	switch wf.Status.Phase {
-	case "", nodePending, nodeRunning:
-		return &HealthStatus{Status: HealthStatusProgressing, Message: wf.Status.Message}, nil
+	case "", nodePending:
+		return &HealthStatus{Health: HealthHealthy, Status: HealthStatusProgressing, Message: wf.Status.Message}, nil
+	case nodeRunning:
+		return &HealthStatus{Ready: true, Health: HealthHealthy, Status: HealthStatusProgressing, Message: wf.Status.Message}, nil
 	case nodeSucceeded:
-		return &HealthStatus{Status: HealthStatusHealthy, Message: wf.Status.Message}, nil
+		return &HealthStatus{Ready: true, Health: HealthHealthy, Status: HealthStatusHealthy, Message: wf.Status.Message}, nil
 	case nodeFailed, nodeError:
-		return &HealthStatus{Status: HealthStatusDegraded, Message: wf.Status.Message}, nil
+		return &HealthStatus{Health: HealthUnhealthy, Status: HealthStatusDegraded, Message: wf.Status.Message}, nil
 	}
-	return &HealthStatus{Status: HealthStatusUnknown, Message: wf.Status.Message}, nil
+	return &HealthStatus{Health: HealthUnknown, Status: HealthStatusUnknown, Message: wf.Status.Message}, nil
 }
 
 // An agnostic workflow object only considers Status.Phase and Status.Message. It is agnostic to the API version or any
@@ -59,17 +61,18 @@ func getArgoApplicationHealth(obj *unstructured.Unstructured) (*HealthStatus, er
 
 	switch app.Status.Health.Status {
 	case HealthStatusProgressing:
-		return &HealthStatus{Status: HealthStatusProgressing, Message: app.Status.Health.Message}, nil
+		return &HealthStatus{Health: HealthHealthy, Status: HealthStatusProgressing, Message: app.Status.Health.Message}, nil
 	case HealthStatusHealthy:
-		return &HealthStatus{Status: HealthStatusHealthy, Message: app.Status.Health.Message}, nil
+		return &HealthStatus{Ready: true, Health: HealthHealthy, Status: HealthStatusHealthy, Message: app.Status.Health.Message}, nil
 	case HealthStatusSuspended:
-		return &HealthStatus{Status: HealthStatusSuspended, Message: app.Status.Health.Message}, nil
+		return &HealthStatus{Health: HealthHealthy, Status: HealthStatusSuspended, Message: app.Status.Health.Message}, nil
 	case HealthStatusDegraded:
-		return &HealthStatus{Status: HealthStatusDegraded, Message: app.Status.Health.Message}, nil
+		return &HealthStatus{Health: HealthUnhealthy, Status: HealthStatusDegraded, Message: app.Status.Health.Message}, nil
 	case HealthStatusMissing:
-		return &HealthStatus{Status: HealthStatusMissing, Message: app.Status.Health.Message}, nil
+		return &HealthStatus{Health: HealthUnhealthy, Status: HealthStatusMissing, Message: app.Status.Health.Message}, nil
 	case HealthStatusUnknown:
-		return &HealthStatus{Status: HealthStatusUnknown, Message: app.Status.Health.Message}, nil
+		return &HealthStatus{Health: HealthUnknown, Status: HealthStatusUnknown, Message: app.Status.Health.Message}, nil
 	}
-	return &HealthStatus{Status: HealthStatusUnknown, Message: app.Status.Health.Message}, nil
+
+	return &HealthStatus{Health: HealthUnknown, Status: HealthStatusUnknown, Message: app.Status.Health.Message}, nil
 }

pkg/health/health_daemonset.go

@@ -28,24 +28,31 @@ func getAppsv1DaemonSetHealth(daemon *appsv1.DaemonSet) (*HealthStatus, error) {
 	if daemon.Generation <= daemon.Status.ObservedGeneration {
 		if daemon.Spec.UpdateStrategy.Type == appsv1.OnDeleteDaemonSetStrategyType {
 			return &HealthStatus{
+				Health:  HealthHealthy,
+				Ready:   daemon.Status.NumberAvailable != 0,
 				Status:  HealthStatusHealthy,
 				Message: fmt.Sprintf("daemon set %d out of %d new pods have been updated", daemon.Status.UpdatedNumberScheduled, daemon.Status.DesiredNumberScheduled),
 			}, nil
 		}
 		if daemon.Status.UpdatedNumberScheduled < daemon.Status.DesiredNumberScheduled {
 			return &HealthStatus{
+				Health:  HealthHealthy,
+				Ready:   daemon.Status.NumberAvailable != 0,
 				Status:  HealthStatusProgressing,
 				Message: fmt.Sprintf("Waiting for daemon set %q rollout to finish: %d out of %d new pods have been updated...", daemon.Name, daemon.Status.UpdatedNumberScheduled, daemon.Status.DesiredNumberScheduled),
 			}, nil
 		}
 		if daemon.Status.NumberAvailable < daemon.Status.DesiredNumberScheduled {
 			return &HealthStatus{
+				Health:  HealthHealthy,
+				Ready:   daemon.Status.NumberAvailable != 0,
 				Status:  HealthStatusProgressing,
 				Message: fmt.Sprintf("Waiting for daemon set %q rollout to finish: %d of %d updated pods are available...", daemon.Name, daemon.Status.NumberAvailable, daemon.Status.DesiredNumberScheduled),
 			}, nil
 		}
 	} else {
 		return &HealthStatus{
+			Health:  HealthUnhealthy,
 			Status:  HealthStatusProgressing,
 			Message: "Waiting for rollout to finish: observed daemon set generation less than desired generation",
 		}, nil

pkg/health/health_deployment.go

@@ -30,38 +30,50 @@ func getAppsv1DeploymentHealth(deployment *appsv1.Deployment) (*HealthStatus, er
 			Message: "Deployment is paused",
 		}, nil
 	}
+
 	// Borrowed at kubernetes/kubectl/rollout_status.go https://github.com/kubernetes/kubernetes/blob/5232ad4a00ec93942d0b2c6359ee6cd1201b46bc/pkg/kubectl/rollout_status.go#L80
 	if deployment.Generation <= deployment.Status.ObservedGeneration {
 		cond := getAppsv1DeploymentCondition(deployment.Status, appsv1.DeploymentProgressing)
 		if cond != nil && cond.Reason == "ProgressDeadlineExceeded" {
 			return &HealthStatus{
+				Health:  HealthWarning,
 				Status:  HealthStatusDegraded,
 				Message: fmt.Sprintf("Deployment %q exceeded its progress deadline", deployment.Name),
 			}, nil
 		} else if deployment.Spec.Replicas != nil && deployment.Status.UpdatedReplicas < *deployment.Spec.Replicas {
 			return &HealthStatus{
-				Status:  HealthStatusProgressing,
+				Ready:   *deployment.Spec.Replicas != 0,
+				Health:  HealthHealthy,
 				Message: fmt.Sprintf("Waiting for rollout to finish: %d out of %d new replicas have been updated...", deployment.Status.UpdatedReplicas, *deployment.Spec.Replicas),
+				Status:  HealthStatusProgressing,
 			}, nil
 		} else if deployment.Status.Replicas > deployment.Status.UpdatedReplicas {
 			return &HealthStatus{
+				Ready:   *deployment.Spec.Replicas != 0,
 				Status:  HealthStatusProgressing,
+				Health:  HealthHealthy,
 				Message: fmt.Sprintf("Waiting for rollout to finish: %d old replicas are pending termination...", deployment.Status.Replicas-deployment.Status.UpdatedReplicas),
 			}, nil
 		} else if deployment.Status.AvailableReplicas < deployment.Status.UpdatedReplicas {
 			return &HealthStatus{
+				Ready:   *deployment.Spec.Replicas != 0,
 				Status:  HealthStatusProgressing,
+				Health:  HealthHealthy,
 				Message: fmt.Sprintf("Waiting for rollout to finish: %d of %d updated replicas are available...", deployment.Status.AvailableReplicas, deployment.Status.UpdatedReplicas),
 			}, nil
 		}
 	} else {
 		return &HealthStatus{
+			Ready:   *deployment.Spec.Replicas != 0,
 			Status:  HealthStatusProgressing,
+			Health:  HealthHealthy,
 			Message: "Waiting for rollout to finish: observed deployment generation less than desired generation",
 		}, nil
 	}
 
 	return &HealthStatus{
+		Ready:  *deployment.Spec.Replicas != 0,
+		Health: HealthHealthy,
 		Status: HealthStatusHealthy,
 	}, nil
 }

pkg/health/health_flux.go

@@ -47,29 +47,30 @@ func getFluxKustomizationHealth(obj *unstructured.Unstructured) (*HealthStatus,
 		msg := fmt.Sprintf("%s: %s", c.Reason, c.Message)
 		if c.Type == fluxHealthy {
 			if c.Status == v1.ConditionTrue {
-				return &HealthStatus{Status: HealthStatusHealthy, Message: msg}, nil
+				return &HealthStatus{Health: HealthHealthy, Status: HealthStatusHealthy, Message: msg}, nil
 			} else {
-				return &HealthStatus{Status: HealthStatusDegraded, Message: msg}, nil
+				return &HealthStatus{Health: HealthHealthy, Status: HealthStatusDegraded, Message: msg}, nil
 			}
 		}
 
 		if c.Type == fluxReady {
 			if c.Status == v1.ConditionTrue {
-				return &HealthStatus{Status: HealthStatusHealthy, Message: msg}, nil
+				return &HealthStatus{Ready: true, Health: HealthHealthy, Status: HealthStatusHealthy, Message: msg}, nil
 			} else {
-				return &HealthStatus{Status: HealthStatusDegraded, Message: msg}, nil
+				return &HealthStatus{Health: HealthUnhealthy, Status: HealthStatusDegraded, Message: msg}, nil
 			}
 		}
 		// All conditions apart from Healthy/Ready should be false
 		if c.Status == v1.ConditionTrue {
 			return &HealthStatus{
+				Health:  HealthUnhealthy,
 				Status:  HealthStatusDegraded,
 				Message: msg,
 			}, nil
 		}
 	}
 
-	return &HealthStatus{Status: HealthStatusUnknown, Message: ""}, nil
+	return &HealthStatus{Health: HealthUnknown, Status: HealthStatusUnknown, Message: ""}, nil
 }
 
 type helmStatusType string
@@ -109,29 +110,30 @@ func getFluxHelmReleaseHealth(obj *unstructured.Unstructured) (*HealthStatus, er
 		msg := fmt.Sprintf("%s: %s", c.Reason, c.Message)
 		if c.Type == helmReleased {
 			if c.Status == v1.ConditionTrue {
-				return &HealthStatus{Status: HealthStatusHealthy, Message: msg}, nil
+				return &HealthStatus{Health: HealthHealthy, Status: HealthStatusHealthy, Message: msg}, nil
 			} else {
-				return &HealthStatus{Status: HealthStatusDegraded, Message: msg}, nil
+				return &HealthStatus{Health: HealthHealthy, Status: HealthStatusDegraded, Message: msg}, nil
 			}
 		}
 
 		if c.Type == helmReady {
 			if c.Status == v1.ConditionTrue {
-				return &HealthStatus{Status: HealthStatusHealthy, Message: msg}, nil
+				return &HealthStatus{Ready: true, Health: HealthHealthy, Status: HealthStatusHealthy, Message: msg}, nil
 			} else {
-				return &HealthStatus{Status: HealthStatusDegraded, Message: msg}, nil
+				return &HealthStatus{Health: HealthUnhealthy, Status: HealthStatusDegraded, Message: msg}, nil
 			}
 		}
 		// All conditions apart from Healthy/Ready should be false
 		if c.Status == v1.ConditionTrue {
 			return &HealthStatus{
+				Health:  HealthUnhealthy,
 				Status:  HealthStatusDegraded,
 				Message: msg,
 			}, nil
 		}
 	}
 
-	return &HealthStatus{Status: HealthStatusUnknown, Message: ""}, nil
+	return &HealthStatus{Health: HealthUnknown, Status: HealthStatusUnknown, Message: ""}, nil
 }
 
 type fluxRepoStatusType string
@@ -174,19 +176,21 @@ func getFluxRepositoryHealth(obj *unstructured.Unstructured) (*HealthStatus, err
 		msg := fmt.Sprintf("%s: %s", c.Reason, c.Message)
 		if c.Type == fluxRepoReady {
 			if c.Status == v1.ConditionTrue {
-				return &HealthStatus{Status: HealthStatusHealthy, Message: msg}, nil
+				return &HealthStatus{Ready: true, Health: HealthHealthy, Status: HealthStatusHealthy, Message: msg}, nil
 			} else {
-				return &HealthStatus{Status: HealthStatusDegraded, Message: msg}, nil
+				return &HealthStatus{Health: HealthHealthy, Status: HealthStatusDegraded, Message: msg}, nil
 			}
 		}
 
 		// All conditions apart from Healthy/Ready should be false
 		if c.Status == v1.ConditionTrue {
 			return &HealthStatus{
+				Health:  HealthUnhealthy,
 				Status:  HealthStatusDegraded,
 				Message: msg,
 			}, nil
 		}
 	}
-	return &HealthStatus{Status: HealthStatusUnknown, Message: ""}, nil
+
+	return &HealthStatus{Health: HealthUnknown, Status: HealthStatusUnknown, Message: ""}, nil
 }

pkg/health/health_hpa.go

@@ -14,6 +14,7 @@ import (
 
 var (
 	progressingStatus = &HealthStatus{
+		Health:  HealthHealthy,
 		Status:  HealthStatusProgressing,
 		Message: "Waiting to Autoscale",
 	}
@@ -133,13 +134,15 @@ func checkConditions(conditions []hpaCondition, progressingStatus *HealthStatus)
 	for _, condition := range conditions {
 		if isDegraded(&condition) {
 			return &HealthStatus{
+				Health:  HealthUnhealthy,
 				Status:  HealthStatusDegraded,
 				Message: condition.Message,
 			}, nil
 		}
 
 		if isHealthy(&condition) {
 			return &HealthStatus{
+				Health:  HealthHealthy,
 				Status:  HealthStatusHealthy,
 				Message: condition.Message,
 			}, nil

pkg/health/health_ingress.go

@@ -11,10 +11,13 @@ func getIngressHealth(obj *unstructured.Unstructured) (*HealthStatus, error) {
 	if err != nil {
 		return &HealthStatus{Status: HealthStatusError, Message: fmt.Sprintf("failed to ingress status: %v", err)}, nil
 	} else if !found {
-		return &HealthStatus{Status: HealthStatusPending, Message: "ingress loadbalancer status not found"}, nil
+		return &HealthStatus{Health: HealthHealthy, Status: HealthStatusPending, Message: "ingress loadbalancer status not found"}, nil
 	}
 
-	health := HealthStatus{}
+	health := HealthStatus{
+		// Ready:  false, // not possible to decide this from the information available
+		Health: HealthHealthy,
+	}
 	if len(ingresses) > 0 {
 		health.Status = HealthStatusHealthy
 	} else {

pkg/health/health_job.go

@@ -2,6 +2,7 @@ package health
 
 import (
 	"fmt"
+
 	corev1 "k8s.io/api/core/v1"
 
 	batchv1 "k8s.io/api/batch/v1"
@@ -25,11 +26,15 @@ func getJobHealth(obj *unstructured.Unstructured) (*HealthStatus, error) {
 }
 
 func getBatchv1JobHealth(job *batchv1.Job) (*HealthStatus, error) {
-	failed := false
-	var failMsg string
-	complete := false
-	var message string
-	isSuspended := false
+	var (
+		complete    bool
+		failed      bool
+		isSuspended bool
+
+		message string
+		failMsg string
+	)
+
 	for _, condition := range job.Status.Conditions {
 		switch condition.Type {
 		case batchv1.JobFailed:
@@ -40,31 +45,36 @@ func getBatchv1JobHealth(job *batchv1.Job) (*HealthStatus, error) {
 			complete = true
 			message = condition.Message
 		case batchv1.JobSuspended:
-			complete = true
 			message = condition.Message
 			if condition.Status == corev1.ConditionTrue {
 				isSuspended = true
 			}
 		}
 	}
-	if !complete {
+
+	if !complete && !isSuspended {
 		return &HealthStatus{
+			Health:  HealthHealthy,
 			Status:  HealthStatusProgressing,
 			Message: message,
 		}, nil
 	} else if failed {
 		return &HealthStatus{
+			Health:  HealthUnhealthy,
 			Status:  HealthStatusDegraded,
 			Message: failMsg,
 		}, nil
 	} else if isSuspended {
 		return &HealthStatus{
+			Health:  HealthHealthy,
 			Status:  HealthStatusSuspended,
 			Message: failMsg,
 		}, nil
 	} else {
 		return &HealthStatus{
+			Ready:   true,
 			Status:  HealthStatusHealthy,
+			Health:  HealthHealthy,
 			Message: message,
 		}, nil
 	}

pkg/health/health_namespace.go

@@ -16,11 +16,14 @@ func getNamespaceHealth(obj *unstructured.Unstructured) (*HealthStatus, error) {
 
 	if node.Status.Phase == v1.NamespaceActive {
 		return &HealthStatus{
+			Ready:  true,
+			Health: HealthHealthy,
 			Status: HealthStatusHealthy,
 		}, nil
 	}
 
 	return &HealthStatus{
+		Health: HealthUnhealthy,
 		Status: HealthStatusDeleting,
 	}, nil
 }