sts, replicaset, daemonset, argo, flux, ns, pvc, svc

flanksource · Apr 9, 2024 · 39928df · 39928df
1 parent 0b99cf1
commit 39928df
Show file tree

Hide file tree

Showing 10 changed files with 81 additions and 47 deletions.
diff --git a/pkg/health/health.go b/pkg/health/health.go
@@ -8,9 +8,10 @@ import (
 type Health string
 
 const (
-	HealthHealthy   Health = "Healthy"
-	HealthUnhealthy Health = "Unhealthy"
-	HealthWarning   Health = "Warning"
+	HealthHealthy   Health = "healthy"
+	HealthUnhealthy Health = "unhealthy"
+	HealthUnknown   Health = "unknown"
+	HealthWarning   Health = "warning"
 )
 
 // Represents resource health status
@@ -163,13 +164,6 @@ func GetHealthCheckFunc(gvk schema.GroupVersionKind) func(obj *unstructured.Unst
 		case "HelmRepository", "GitRepository":
 			return getFluxRepositoryHealth
 		}
-
-	// case "apiregistration.k8s.io":
-	// 	switch gvk.Kind {
-	// 	case APIServiceKind:
-	// 		return getAPIServiceHealth
-	// 	}
-
 	case "networking.k8s.io":
 		switch gvk.Kind {
 		case IngressKind:

diff --git a/pkg/health/health_argo.go b/pkg/health/health_argo.go
@@ -33,14 +33,16 @@ func GetArgoWorkflowHealth(obj *unstructured.Unstructured) (*HealthStatus, error
 		return nil, err
 	}
 	switch wf.Status.Phase {
-	case "", nodePending, nodeRunning:
-		return &HealthStatus{Status: HealthStatusProgressing, Message: wf.Status.Message}, nil
+	case "", nodePending:
+		return &HealthStatus{Health: HealthHealthy, Status: HealthStatusProgressing, Message: wf.Status.Message}, nil
+	case nodeRunning:
+		return &HealthStatus{Ready: true, Health: HealthHealthy, Status: HealthStatusProgressing, Message: wf.Status.Message}, nil
 	case nodeSucceeded:
-		return &HealthStatus{Status: HealthStatusHealthy, Message: wf.Status.Message}, nil
+		return &HealthStatus{Ready: true, Health: HealthHealthy, Status: HealthStatusHealthy, Message: wf.Status.Message}, nil
 	case nodeFailed, nodeError:
-		return &HealthStatus{Status: HealthStatusDegraded, Message: wf.Status.Message}, nil
+		return &HealthStatus{Health: HealthUnhealthy, Status: HealthStatusDegraded, Message: wf.Status.Message}, nil
 	}
-	return &HealthStatus{Status: HealthStatusUnknown, Message: wf.Status.Message}, nil
+	return &HealthStatus{Health: HealthUnknown, Status: HealthStatusUnknown, Message: wf.Status.Message}, nil
 }
 
 // An agnostic workflow object only considers Status.Phase and Status.Message. It is agnostic to the API version or any
@@ -59,17 +61,18 @@ func getArgoApplicationHealth(obj *unstructured.Unstructured) (*HealthStatus, er
 
 	switch app.Status.Health.Status {
 	case HealthStatusProgressing:
-		return &HealthStatus{Status: HealthStatusProgressing, Message: app.Status.Health.Message}, nil
+		return &HealthStatus{Health: HealthHealthy, Status: HealthStatusProgressing, Message: app.Status.Health.Message}, nil
 	case HealthStatusHealthy:
-		return &HealthStatus{Status: HealthStatusHealthy, Message: app.Status.Health.Message}, nil
+		return &HealthStatus{Ready: true, Health: HealthHealthy, Status: HealthStatusHealthy, Message: app.Status.Health.Message}, nil
 	case HealthStatusSuspended:
-		return &HealthStatus{Status: HealthStatusSuspended, Message: app.Status.Health.Message}, nil
+		return &HealthStatus{Health: HealthHealthy, Status: HealthStatusSuspended, Message: app.Status.Health.Message}, nil
 	case HealthStatusDegraded:
-		return &HealthStatus{Status: HealthStatusDegraded, Message: app.Status.Health.Message}, nil
+		return &HealthStatus{Health: HealthUnhealthy, Status: HealthStatusDegraded, Message: app.Status.Health.Message}, nil
 	case HealthStatusMissing:
-		return &HealthStatus{Status: HealthStatusMissing, Message: app.Status.Health.Message}, nil
+		return &HealthStatus{Health: HealthUnhealthy, Status: HealthStatusMissing, Message: app.Status.Health.Message}, nil
 	case HealthStatusUnknown:
-		return &HealthStatus{Status: HealthStatusUnknown, Message: app.Status.Health.Message}, nil
+		return &HealthStatus{Health: HealthUnknown, Status: HealthStatusUnknown, Message: app.Status.Health.Message}, nil
 	}
-	return &HealthStatus{Status: HealthStatusUnknown, Message: app.Status.Health.Message}, nil
+
+	return &HealthStatus{Health: HealthUnknown, Status: HealthStatusUnknown, Message: app.Status.Health.Message}, nil
 }
diff --git a/pkg/health/health_daemonset.go b/pkg/health/health_daemonset.go
@@ -28,24 +28,31 @@ func getAppsv1DaemonSetHealth(daemon *appsv1.DaemonSet) (*HealthStatus, error) {
 	if daemon.Generation <= daemon.Status.ObservedGeneration {
 		if daemon.Spec.UpdateStrategy.Type == appsv1.OnDeleteDaemonSetStrategyType {
 			return &HealthStatus{
+				Health:  HealthHealthy,
+				Ready:   daemon.Status.NumberAvailable != 0,
 				Status:  HealthStatusHealthy,
 				Message: fmt.Sprintf("daemon set %d out of %d new pods have been updated", daemon.Status.UpdatedNumberScheduled, daemon.Status.DesiredNumberScheduled),
 			}, nil
 		}
 		if daemon.Status.UpdatedNumberScheduled < daemon.Status.DesiredNumberScheduled {
 			return &HealthStatus{
+				Health:  HealthHealthy,
+				Ready:   daemon.Status.NumberAvailable != 0,
 				Status:  HealthStatusProgressing,
 				Message: fmt.Sprintf("Waiting for daemon set %q rollout to finish: %d out of %d new pods have been updated...", daemon.Name, daemon.Status.UpdatedNumberScheduled, daemon.Status.DesiredNumberScheduled),
 			}, nil
 		}
 		if daemon.Status.NumberAvailable < daemon.Status.DesiredNumberScheduled {
 			return &HealthStatus{
+				Health:  HealthHealthy,
+				Ready:   daemon.Status.NumberAvailable != 0,
 				Status:  HealthStatusProgressing,
 				Message: fmt.Sprintf("Waiting for daemon set %q rollout to finish: %d of %d updated pods are available...", daemon.Name, daemon.Status.NumberAvailable, daemon.Status.DesiredNumberScheduled),
 			}, nil
 		}
 	} else {
 		return &HealthStatus{
+			Health:  HealthUnhealthy,
 			Status:  HealthStatusProgressing,
 			Message: "Waiting for rollout to finish: observed daemon set generation less than desired generation",
 		}, nil

diff --git a/pkg/health/health_flux.go b/pkg/health/health_flux.go
@@ -47,29 +47,30 @@ func getFluxKustomizationHealth(obj *unstructured.Unstructured) (*HealthStatus,
 		msg := fmt.Sprintf("%s: %s", c.Reason, c.Message)
 		if c.Type == fluxHealthy {
 			if c.Status == v1.ConditionTrue {
-				return &HealthStatus{Status: HealthStatusHealthy, Message: msg}, nil
+				return &HealthStatus{Health: HealthHealthy, Status: HealthStatusHealthy, Message: msg}, nil
 			} else {
-				return &HealthStatus{Status: HealthStatusDegraded, Message: msg}, nil
+				return &HealthStatus{Health: HealthHealthy, Status: HealthStatusDegraded, Message: msg}, nil
 			}
 		}
 
 		if c.Type == fluxReady {
 			if c.Status == v1.ConditionTrue {
-				return &HealthStatus{Status: HealthStatusHealthy, Message: msg}, nil
+				return &HealthStatus{Ready: true, Health: HealthHealthy, Status: HealthStatusHealthy, Message: msg}, nil
 			} else {
-				return &HealthStatus{Status: HealthStatusDegraded, Message: msg}, nil
+				return &HealthStatus{Health: HealthUnhealthy, Status: HealthStatusDegraded, Message: msg}, nil
 			}
 		}
 		// All conditions apart from Healthy/Ready should be false
 		if c.Status == v1.ConditionTrue {
 			return &HealthStatus{
+				Health:  HealthUnhealthy,
 				Status:  HealthStatusDegraded,
 				Message: msg,
 			}, nil
 		}
 	}
 
-	return &HealthStatus{Status: HealthStatusUnknown, Message: ""}, nil
+	return &HealthStatus{Health: HealthUnknown, Status: HealthStatusUnknown, Message: ""}, nil
 }
 
 type helmStatusType string
@@ -109,29 +110,30 @@ func getFluxHelmReleaseHealth(obj *unstructured.Unstructured) (*HealthStatus, er
 		msg := fmt.Sprintf("%s: %s", c.Reason, c.Message)
 		if c.Type == helmReleased {
 			if c.Status == v1.ConditionTrue {
-				return &HealthStatus{Status: HealthStatusHealthy, Message: msg}, nil
+				return &HealthStatus{Health: HealthHealthy, Status: HealthStatusHealthy, Message: msg}, nil
 			} else {
-				return &HealthStatus{Status: HealthStatusDegraded, Message: msg}, nil
+				return &HealthStatus{Health: HealthHealthy, Status: HealthStatusDegraded, Message: msg}, nil
 			}
 		}
 
 		if c.Type == helmReady {
 			if c.Status == v1.ConditionTrue {
-				return &HealthStatus{Status: HealthStatusHealthy, Message: msg}, nil
+				return &HealthStatus{Ready: true, Health: HealthHealthy, Status: HealthStatusHealthy, Message: msg}, nil
 			} else {
-				return &HealthStatus{Status: HealthStatusDegraded, Message: msg}, nil
+				return &HealthStatus{Health: HealthUnhealthy, Status: HealthStatusDegraded, Message: msg}, nil
 			}
 		}
 		// All conditions apart from Healthy/Ready should be false
 		if c.Status == v1.ConditionTrue {
 			return &HealthStatus{
+				Health:  HealthUnhealthy,
 				Status:  HealthStatusDegraded,
 				Message: msg,
 			}, nil
 		}
 	}
 
-	return &HealthStatus{Status: HealthStatusUnknown, Message: ""}, nil
+	return &HealthStatus{Health: HealthUnknown, Status: HealthStatusUnknown, Message: ""}, nil
 }
 
 type fluxRepoStatusType string
@@ -174,19 +176,21 @@ func getFluxRepositoryHealth(obj *unstructured.Unstructured) (*HealthStatus, err
 		msg := fmt.Sprintf("%s: %s", c.Reason, c.Message)
 		if c.Type == fluxRepoReady {
 			if c.Status == v1.ConditionTrue {
-				return &HealthStatus{Status: HealthStatusHealthy, Message: msg}, nil
+				return &HealthStatus{Ready: true, Health: HealthHealthy, Status: HealthStatusHealthy, Message: msg}, nil
 			} else {
-				return &HealthStatus{Status: HealthStatusDegraded, Message: msg}, nil
+				return &HealthStatus{Health: HealthHealthy, Status: HealthStatusDegraded, Message: msg}, nil
 			}
 		}
 
 		// All conditions apart from Healthy/Ready should be false
 		if c.Status == v1.ConditionTrue {
 			return &HealthStatus{
+				Health:  HealthUnhealthy,
 				Status:  HealthStatusDegraded,
 				Message: msg,
 			}, nil
 		}
 	}
-	return &HealthStatus{Status: HealthStatusUnknown, Message: ""}, nil
+
+	return &HealthStatus{Health: HealthUnknown, Status: HealthStatusUnknown, Message: ""}, nil
 }
diff --git a/pkg/health/health_ingress.go b/pkg/health/health_ingress.go
@@ -11,10 +11,13 @@ func getIngressHealth(obj *unstructured.Unstructured) (*HealthStatus, error) {
 	if err != nil {
 		return &HealthStatus{Status: HealthStatusError, Message: fmt.Sprintf("failed to ingress status: %v", err)}, nil
 	} else if !found {
-		return &HealthStatus{Status: HealthStatusPending, Message: "ingress loadbalancer status not found"}, nil
+		return &HealthStatus{Health: HealthHealthy, Status: HealthStatusPending, Message: "ingress loadbalancer status not found"}, nil
 	}
 
-	health := HealthStatus{}
+	health := HealthStatus{
+		// Ready:  false, // not possible to decide this from the information available
+		Health: HealthHealthy,
+	}
 	if len(ingresses) > 0 {
 		health.Status = HealthStatusHealthy
 	} else {

diff --git a/pkg/health/health_namespace.go b/pkg/health/health_namespace.go
@@ -16,11 +16,14 @@ func getNamespaceHealth(obj *unstructured.Unstructured) (*HealthStatus, error) {
 
 	if node.Status.Phase == v1.NamespaceActive {
 		return &HealthStatus{
+			Ready:  true,
+			Health: HealthHealthy,
 			Status: HealthStatusHealthy,
 		}, nil
 	}
 
 	return &HealthStatus{
+		Health: HealthUnhealthy,
 		Status: HealthStatusDeleting,
 	}, nil
 }
diff --git a/pkg/health/health_pvc.go b/pkg/health/health_pvc.go
@@ -24,16 +24,20 @@ func getPVCHealth(obj *unstructured.Unstructured) (*HealthStatus, error) {
 }
 
 func getCorev1PVCHealth(pvc *corev1.PersistentVolumeClaim) (*HealthStatus, error) {
-	var status HealthStatusCode
+	health := HealthStatus{Health: HealthHealthy}
 	switch pvc.Status.Phase {
 	case corev1.ClaimLost:
-		status = HealthStatusDegraded
+		health.Health = HealthUnhealthy
+		health.Status = HealthStatusDegraded
 	case corev1.ClaimPending:
-		status = HealthStatusProgressing
+		health.Status = HealthStatusProgressing
 	case corev1.ClaimBound:
-		status = HealthStatusHealthy
+		health.Ready = true
+		health.Status = HealthStatusHealthy
 	default:
-		status = HealthStatusUnknown
+		health.Health = HealthUnknown
+		health.Status = HealthStatusUnknown
 	}
-	return &HealthStatus{Status: status}, nil
+
+	return &health, nil
 }
diff --git a/pkg/health/health_replicaset.go b/pkg/health/health_replicaset.go
@@ -26,20 +26,25 @@ func getReplicaSetHealth(obj *unstructured.Unstructured) (*HealthStatus, error)
 
 func getAppsv1ReplicaSetHealth(replicaSet *appsv1.ReplicaSet) (*HealthStatus, error) {
 	if replicaSet.Generation <= replicaSet.Status.ObservedGeneration {
-		cond := getAppsv1ReplicaSetCondition(replicaSet.Status, appsv1.ReplicaSetReplicaFailure)
-		if cond != nil && cond.Status == corev1.ConditionTrue {
+		failCondition := getAppsv1ReplicaSetCondition(replicaSet.Status, appsv1.ReplicaSetReplicaFailure)
+		if failCondition != nil && failCondition.Status == corev1.ConditionTrue {
 			return &HealthStatus{
+				Health:  HealthUnhealthy,
 				Status:  HealthStatusDegraded,
-				Message: cond.Message,
+				Message: failCondition.Message,
 			}, nil
 		} else if replicaSet.Spec.Replicas != nil && replicaSet.Status.AvailableReplicas < *replicaSet.Spec.Replicas {
 			return &HealthStatus{
+				Ready:   replicaSet.Status.AvailableReplicas != 0,
+				Health:  HealthHealthy,
 				Status:  HealthStatusProgressing,
 				Message: fmt.Sprintf("Waiting for rollout to finish: %d out of %d new replicas are available...", replicaSet.Status.AvailableReplicas, *replicaSet.Spec.Replicas),
 			}, nil
 		}
 	} else {
 		return &HealthStatus{
+			Ready:   replicaSet.Status.AvailableReplicas != 0,
+			Health:  HealthHealthy,
 			Status:  HealthStatusProgressing,
 			Message: "Waiting for rollout to finish: observed replica set generation less than desired generation",
 		}, nil

diff --git a/pkg/health/health_service.go b/pkg/health/health_service.go
@@ -24,7 +24,7 @@ func getServiceHealth(obj *unstructured.Unstructured) (*HealthStatus, error) {
 }
 
 func getCorev1ServiceHealth(service *corev1.Service) (*HealthStatus, error) {
-	health := HealthStatus{Status: HealthStatusHealthy}
+	health := HealthStatus{Health: HealthHealthy, Status: HealthStatusHealthy}
 	if service.Spec.Type == corev1.ServiceTypeLoadBalancer {
 		if len(service.Status.LoadBalancer.Ingress) > 0 {
 			health.Status = HealthStatusHealthy

diff --git a/pkg/health/health_statefulset.go b/pkg/health/health_statefulset.go
@@ -27,12 +27,14 @@ func getAppsv1StatefulSetHealth(sts *appsv1.StatefulSet) (*HealthStatus, error)
 	// Borrowed at kubernetes/kubectl/rollout_status.go https://github.com/kubernetes/kubernetes/blob/5232ad4a00ec93942d0b2c6359ee6cd1201b46bc/pkg/kubectl/rollout_status.go#L131
 	if sts.Status.ObservedGeneration == 0 || sts.Generation > sts.Status.ObservedGeneration {
 		return &HealthStatus{
+			Health:  HealthHealthy,
 			Status:  HealthStatusProgressing,
 			Message: "Waiting for statefulset spec update to be observed...",
 		}, nil
 	}
 	if sts.Spec.Replicas != nil && sts.Status.ReadyReplicas < *sts.Spec.Replicas {
 		return &HealthStatus{
+			Health:  HealthHealthy,
 			Status:  HealthStatusProgressing,
 			Message: fmt.Sprintf("Waiting for %d pods to be ready...", *sts.Spec.Replicas-sts.Status.ReadyReplicas),
 		}, nil
@@ -41,30 +43,39 @@ func getAppsv1StatefulSetHealth(sts *appsv1.StatefulSet) (*HealthStatus, error)
 		if sts.Spec.Replicas != nil && sts.Spec.UpdateStrategy.RollingUpdate.Partition != nil {
 			if sts.Status.UpdatedReplicas < (*sts.Spec.Replicas - *sts.Spec.UpdateStrategy.RollingUpdate.Partition) {
 				return &HealthStatus{
+					Health: HealthHealthy,
 					Status: HealthStatusProgressing,
 					Message: fmt.Sprintf("Waiting for partitioned roll out to finish: %d out of %d new pods have been updated...",
 						sts.Status.UpdatedReplicas, (*sts.Spec.Replicas - *sts.Spec.UpdateStrategy.RollingUpdate.Partition)),
 				}, nil
 			}
 		}
 		return &HealthStatus{
+			Ready:   true,
+			Health:  HealthHealthy,
 			Status:  HealthStatusHealthy,
 			Message: fmt.Sprintf("partitioned roll out complete: %d new pods have been updated...", sts.Status.UpdatedReplicas),
 		}, nil
 	}
 	if sts.Spec.UpdateStrategy.Type == appsv1.OnDeleteStatefulSetStrategyType {
 		return &HealthStatus{
+			Ready:   true,
+			Health:  HealthHealthy,
 			Status:  HealthStatusHealthy,
 			Message: fmt.Sprintf("statefulset has %d ready pods", sts.Status.ReadyReplicas),
 		}, nil
 	}
 	if sts.Status.UpdateRevision != sts.Status.CurrentRevision {
 		return &HealthStatus{
+			Health:  HealthHealthy,
+			Ready:   true,
 			Status:  HealthStatusProgressing,
 			Message: fmt.Sprintf("waiting for statefulset rolling update to complete %d pods at revision %s...", sts.Status.UpdatedReplicas, sts.Status.UpdateRevision),
 		}, nil
 	}
 	return &HealthStatus{
+		Health:  HealthHealthy,
+		Ready:   true,
 		Status:  HealthStatusHealthy,
 		Message: fmt.Sprintf("statefulset rolling update complete %d pods at revision %s...", sts.Status.CurrentReplicas, sts.Status.CurrentRevision),
 	}, nil