diff --git a/pkg/health/health.go b/pkg/health/health.go index ad12778..124c427 100644 --- a/pkg/health/health.go +++ b/pkg/health/health.go @@ -8,9 +8,10 @@ import ( type Health string const ( - HealthHealthy Health = "Healthy" - HealthUnhealthy Health = "Unhealthy" - HealthWarning Health = "Warning" + HealthHealthy Health = "healthy" + HealthUnhealthy Health = "unhealthy" + HealthUnknown Health = "unknown" + HealthWarning Health = "warning" ) // Represents resource health status @@ -163,13 +164,6 @@ func GetHealthCheckFunc(gvk schema.GroupVersionKind) func(obj *unstructured.Unst case "HelmRepository", "GitRepository": return getFluxRepositoryHealth } - - // case "apiregistration.k8s.io": - // switch gvk.Kind { - // case APIServiceKind: - // return getAPIServiceHealth - // } - case "networking.k8s.io": switch gvk.Kind { case IngressKind: diff --git a/pkg/health/health_argo.go b/pkg/health/health_argo.go index 97a8b85..0e496f6 100644 --- a/pkg/health/health_argo.go +++ b/pkg/health/health_argo.go @@ -33,14 +33,16 @@ func GetArgoWorkflowHealth(obj *unstructured.Unstructured) (*HealthStatus, error return nil, err } switch wf.Status.Phase { - case "", nodePending, nodeRunning: - return &HealthStatus{Status: HealthStatusProgressing, Message: wf.Status.Message}, nil + case "", nodePending: + return &HealthStatus{Health: HealthHealthy, Status: HealthStatusProgressing, Message: wf.Status.Message}, nil + case nodeRunning: + return &HealthStatus{Ready: true, Health: HealthHealthy, Status: HealthStatusProgressing, Message: wf.Status.Message}, nil case nodeSucceeded: - return &HealthStatus{Status: HealthStatusHealthy, Message: wf.Status.Message}, nil + return &HealthStatus{Ready: true, Health: HealthHealthy, Status: HealthStatusHealthy, Message: wf.Status.Message}, nil case nodeFailed, nodeError: - return &HealthStatus{Status: HealthStatusDegraded, Message: wf.Status.Message}, nil + return &HealthStatus{Health: HealthUnhealthy, Status: HealthStatusDegraded, Message: wf.Status.Message}, nil } - return &HealthStatus{Status: HealthStatusUnknown, Message: wf.Status.Message}, nil + return &HealthStatus{Health: HealthUnknown, Status: HealthStatusUnknown, Message: wf.Status.Message}, nil } // An agnostic workflow object only considers Status.Phase and Status.Message. It is agnostic to the API version or any @@ -59,17 +61,18 @@ func getArgoApplicationHealth(obj *unstructured.Unstructured) (*HealthStatus, er switch app.Status.Health.Status { case HealthStatusProgressing: - return &HealthStatus{Status: HealthStatusProgressing, Message: app.Status.Health.Message}, nil + return &HealthStatus{Health: HealthHealthy, Status: HealthStatusProgressing, Message: app.Status.Health.Message}, nil case HealthStatusHealthy: - return &HealthStatus{Status: HealthStatusHealthy, Message: app.Status.Health.Message}, nil + return &HealthStatus{Ready: true, Health: HealthHealthy, Status: HealthStatusHealthy, Message: app.Status.Health.Message}, nil case HealthStatusSuspended: - return &HealthStatus{Status: HealthStatusSuspended, Message: app.Status.Health.Message}, nil + return &HealthStatus{Health: HealthHealthy, Status: HealthStatusSuspended, Message: app.Status.Health.Message}, nil case HealthStatusDegraded: - return &HealthStatus{Status: HealthStatusDegraded, Message: app.Status.Health.Message}, nil + return &HealthStatus{Health: HealthUnhealthy, Status: HealthStatusDegraded, Message: app.Status.Health.Message}, nil case HealthStatusMissing: - return &HealthStatus{Status: HealthStatusMissing, Message: app.Status.Health.Message}, nil + return &HealthStatus{Health: HealthUnhealthy, Status: HealthStatusMissing, Message: app.Status.Health.Message}, nil case HealthStatusUnknown: - return &HealthStatus{Status: HealthStatusUnknown, Message: app.Status.Health.Message}, nil + return &HealthStatus{Health: HealthUnknown, Status: HealthStatusUnknown, Message: app.Status.Health.Message}, nil } - return &HealthStatus{Status: HealthStatusUnknown, Message: app.Status.Health.Message}, nil + + return &HealthStatus{Health: HealthUnknown, Status: HealthStatusUnknown, Message: app.Status.Health.Message}, nil } diff --git a/pkg/health/health_daemonset.go b/pkg/health/health_daemonset.go index 9bf252e..ecc56b7 100644 --- a/pkg/health/health_daemonset.go +++ b/pkg/health/health_daemonset.go @@ -28,24 +28,31 @@ func getAppsv1DaemonSetHealth(daemon *appsv1.DaemonSet) (*HealthStatus, error) { if daemon.Generation <= daemon.Status.ObservedGeneration { if daemon.Spec.UpdateStrategy.Type == appsv1.OnDeleteDaemonSetStrategyType { return &HealthStatus{ + Health: HealthHealthy, + Ready: daemon.Status.NumberAvailable != 0, Status: HealthStatusHealthy, Message: fmt.Sprintf("daemon set %d out of %d new pods have been updated", daemon.Status.UpdatedNumberScheduled, daemon.Status.DesiredNumberScheduled), }, nil } if daemon.Status.UpdatedNumberScheduled < daemon.Status.DesiredNumberScheduled { return &HealthStatus{ + Health: HealthHealthy, + Ready: daemon.Status.NumberAvailable != 0, Status: HealthStatusProgressing, Message: fmt.Sprintf("Waiting for daemon set %q rollout to finish: %d out of %d new pods have been updated...", daemon.Name, daemon.Status.UpdatedNumberScheduled, daemon.Status.DesiredNumberScheduled), }, nil } if daemon.Status.NumberAvailable < daemon.Status.DesiredNumberScheduled { return &HealthStatus{ + Health: HealthHealthy, + Ready: daemon.Status.NumberAvailable != 0, Status: HealthStatusProgressing, Message: fmt.Sprintf("Waiting for daemon set %q rollout to finish: %d of %d updated pods are available...", daemon.Name, daemon.Status.NumberAvailable, daemon.Status.DesiredNumberScheduled), }, nil } } else { return &HealthStatus{ + Health: HealthUnhealthy, Status: HealthStatusProgressing, Message: "Waiting for rollout to finish: observed daemon set generation less than desired generation", }, nil diff --git a/pkg/health/health_flux.go b/pkg/health/health_flux.go index a0c29bb..867ad7b 100644 --- a/pkg/health/health_flux.go +++ b/pkg/health/health_flux.go @@ -47,29 +47,30 @@ func getFluxKustomizationHealth(obj *unstructured.Unstructured) (*HealthStatus, msg := fmt.Sprintf("%s: %s", c.Reason, c.Message) if c.Type == fluxHealthy { if c.Status == v1.ConditionTrue { - return &HealthStatus{Status: HealthStatusHealthy, Message: msg}, nil + return &HealthStatus{Health: HealthHealthy, Status: HealthStatusHealthy, Message: msg}, nil } else { - return &HealthStatus{Status: HealthStatusDegraded, Message: msg}, nil + return &HealthStatus{Health: HealthHealthy, Status: HealthStatusDegraded, Message: msg}, nil } } if c.Type == fluxReady { if c.Status == v1.ConditionTrue { - return &HealthStatus{Status: HealthStatusHealthy, Message: msg}, nil + return &HealthStatus{Ready: true, Health: HealthHealthy, Status: HealthStatusHealthy, Message: msg}, nil } else { - return &HealthStatus{Status: HealthStatusDegraded, Message: msg}, nil + return &HealthStatus{Health: HealthUnhealthy, Status: HealthStatusDegraded, Message: msg}, nil } } // All conditions apart from Healthy/Ready should be false if c.Status == v1.ConditionTrue { return &HealthStatus{ + Health: HealthUnhealthy, Status: HealthStatusDegraded, Message: msg, }, nil } } - return &HealthStatus{Status: HealthStatusUnknown, Message: ""}, nil + return &HealthStatus{Health: HealthUnknown, Status: HealthStatusUnknown, Message: ""}, nil } type helmStatusType string @@ -109,29 +110,30 @@ func getFluxHelmReleaseHealth(obj *unstructured.Unstructured) (*HealthStatus, er msg := fmt.Sprintf("%s: %s", c.Reason, c.Message) if c.Type == helmReleased { if c.Status == v1.ConditionTrue { - return &HealthStatus{Status: HealthStatusHealthy, Message: msg}, nil + return &HealthStatus{Health: HealthHealthy, Status: HealthStatusHealthy, Message: msg}, nil } else { - return &HealthStatus{Status: HealthStatusDegraded, Message: msg}, nil + return &HealthStatus{Health: HealthHealthy, Status: HealthStatusDegraded, Message: msg}, nil } } if c.Type == helmReady { if c.Status == v1.ConditionTrue { - return &HealthStatus{Status: HealthStatusHealthy, Message: msg}, nil + return &HealthStatus{Ready: true, Health: HealthHealthy, Status: HealthStatusHealthy, Message: msg}, nil } else { - return &HealthStatus{Status: HealthStatusDegraded, Message: msg}, nil + return &HealthStatus{Health: HealthUnhealthy, Status: HealthStatusDegraded, Message: msg}, nil } } // All conditions apart from Healthy/Ready should be false if c.Status == v1.ConditionTrue { return &HealthStatus{ + Health: HealthUnhealthy, Status: HealthStatusDegraded, Message: msg, }, nil } } - return &HealthStatus{Status: HealthStatusUnknown, Message: ""}, nil + return &HealthStatus{Health: HealthUnknown, Status: HealthStatusUnknown, Message: ""}, nil } type fluxRepoStatusType string @@ -174,19 +176,21 @@ func getFluxRepositoryHealth(obj *unstructured.Unstructured) (*HealthStatus, err msg := fmt.Sprintf("%s: %s", c.Reason, c.Message) if c.Type == fluxRepoReady { if c.Status == v1.ConditionTrue { - return &HealthStatus{Status: HealthStatusHealthy, Message: msg}, nil + return &HealthStatus{Ready: true, Health: HealthHealthy, Status: HealthStatusHealthy, Message: msg}, nil } else { - return &HealthStatus{Status: HealthStatusDegraded, Message: msg}, nil + return &HealthStatus{Health: HealthHealthy, Status: HealthStatusDegraded, Message: msg}, nil } } // All conditions apart from Healthy/Ready should be false if c.Status == v1.ConditionTrue { return &HealthStatus{ + Health: HealthUnhealthy, Status: HealthStatusDegraded, Message: msg, }, nil } } - return &HealthStatus{Status: HealthStatusUnknown, Message: ""}, nil + + return &HealthStatus{Health: HealthUnknown, Status: HealthStatusUnknown, Message: ""}, nil } diff --git a/pkg/health/health_ingress.go b/pkg/health/health_ingress.go index ee697f9..89ebefb 100644 --- a/pkg/health/health_ingress.go +++ b/pkg/health/health_ingress.go @@ -11,10 +11,13 @@ func getIngressHealth(obj *unstructured.Unstructured) (*HealthStatus, error) { if err != nil { return &HealthStatus{Status: HealthStatusError, Message: fmt.Sprintf("failed to ingress status: %v", err)}, nil } else if !found { - return &HealthStatus{Status: HealthStatusPending, Message: "ingress loadbalancer status not found"}, nil + return &HealthStatus{Health: HealthHealthy, Status: HealthStatusPending, Message: "ingress loadbalancer status not found"}, nil } - health := HealthStatus{} + health := HealthStatus{ + // Ready: false, // not possible to decide this from the information available + Health: HealthHealthy, + } if len(ingresses) > 0 { health.Status = HealthStatusHealthy } else { diff --git a/pkg/health/health_namespace.go b/pkg/health/health_namespace.go index e7ab9b4..65ca761 100644 --- a/pkg/health/health_namespace.go +++ b/pkg/health/health_namespace.go @@ -16,11 +16,14 @@ func getNamespaceHealth(obj *unstructured.Unstructured) (*HealthStatus, error) { if node.Status.Phase == v1.NamespaceActive { return &HealthStatus{ + Ready: true, + Health: HealthHealthy, Status: HealthStatusHealthy, }, nil } return &HealthStatus{ + Health: HealthUnhealthy, Status: HealthStatusDeleting, }, nil } diff --git a/pkg/health/health_pvc.go b/pkg/health/health_pvc.go index 5e934e1..ca5caac 100644 --- a/pkg/health/health_pvc.go +++ b/pkg/health/health_pvc.go @@ -24,16 +24,20 @@ func getPVCHealth(obj *unstructured.Unstructured) (*HealthStatus, error) { } func getCorev1PVCHealth(pvc *corev1.PersistentVolumeClaim) (*HealthStatus, error) { - var status HealthStatusCode + health := HealthStatus{Health: HealthHealthy} switch pvc.Status.Phase { case corev1.ClaimLost: - status = HealthStatusDegraded + health.Health = HealthUnhealthy + health.Status = HealthStatusDegraded case corev1.ClaimPending: - status = HealthStatusProgressing + health.Status = HealthStatusProgressing case corev1.ClaimBound: - status = HealthStatusHealthy + health.Ready = true + health.Status = HealthStatusHealthy default: - status = HealthStatusUnknown + health.Health = HealthUnknown + health.Status = HealthStatusUnknown } - return &HealthStatus{Status: status}, nil + + return &health, nil } diff --git a/pkg/health/health_replicaset.go b/pkg/health/health_replicaset.go index 5435f20..4220a0a 100644 --- a/pkg/health/health_replicaset.go +++ b/pkg/health/health_replicaset.go @@ -26,20 +26,25 @@ func getReplicaSetHealth(obj *unstructured.Unstructured) (*HealthStatus, error) func getAppsv1ReplicaSetHealth(replicaSet *appsv1.ReplicaSet) (*HealthStatus, error) { if replicaSet.Generation <= replicaSet.Status.ObservedGeneration { - cond := getAppsv1ReplicaSetCondition(replicaSet.Status, appsv1.ReplicaSetReplicaFailure) - if cond != nil && cond.Status == corev1.ConditionTrue { + failCondition := getAppsv1ReplicaSetCondition(replicaSet.Status, appsv1.ReplicaSetReplicaFailure) + if failCondition != nil && failCondition.Status == corev1.ConditionTrue { return &HealthStatus{ + Health: HealthUnhealthy, Status: HealthStatusDegraded, - Message: cond.Message, + Message: failCondition.Message, }, nil } else if replicaSet.Spec.Replicas != nil && replicaSet.Status.AvailableReplicas < *replicaSet.Spec.Replicas { return &HealthStatus{ + Ready: replicaSet.Status.AvailableReplicas != 0, + Health: HealthHealthy, Status: HealthStatusProgressing, Message: fmt.Sprintf("Waiting for rollout to finish: %d out of %d new replicas are available...", replicaSet.Status.AvailableReplicas, *replicaSet.Spec.Replicas), }, nil } } else { return &HealthStatus{ + Ready: replicaSet.Status.AvailableReplicas != 0, + Health: HealthHealthy, Status: HealthStatusProgressing, Message: "Waiting for rollout to finish: observed replica set generation less than desired generation", }, nil diff --git a/pkg/health/health_service.go b/pkg/health/health_service.go index 5e7b124..f09c703 100644 --- a/pkg/health/health_service.go +++ b/pkg/health/health_service.go @@ -24,7 +24,7 @@ func getServiceHealth(obj *unstructured.Unstructured) (*HealthStatus, error) { } func getCorev1ServiceHealth(service *corev1.Service) (*HealthStatus, error) { - health := HealthStatus{Status: HealthStatusHealthy} + health := HealthStatus{Health: HealthHealthy, Status: HealthStatusHealthy} if service.Spec.Type == corev1.ServiceTypeLoadBalancer { if len(service.Status.LoadBalancer.Ingress) > 0 { health.Status = HealthStatusHealthy diff --git a/pkg/health/health_statefulset.go b/pkg/health/health_statefulset.go index 30e4f7d..77cfc0d 100644 --- a/pkg/health/health_statefulset.go +++ b/pkg/health/health_statefulset.go @@ -27,12 +27,14 @@ func getAppsv1StatefulSetHealth(sts *appsv1.StatefulSet) (*HealthStatus, error) // Borrowed at kubernetes/kubectl/rollout_status.go https://github.com/kubernetes/kubernetes/blob/5232ad4a00ec93942d0b2c6359ee6cd1201b46bc/pkg/kubectl/rollout_status.go#L131 if sts.Status.ObservedGeneration == 0 || sts.Generation > sts.Status.ObservedGeneration { return &HealthStatus{ + Health: HealthHealthy, Status: HealthStatusProgressing, Message: "Waiting for statefulset spec update to be observed...", }, nil } if sts.Spec.Replicas != nil && sts.Status.ReadyReplicas < *sts.Spec.Replicas { return &HealthStatus{ + Health: HealthHealthy, Status: HealthStatusProgressing, Message: fmt.Sprintf("Waiting for %d pods to be ready...", *sts.Spec.Replicas-sts.Status.ReadyReplicas), }, nil @@ -41,6 +43,7 @@ func getAppsv1StatefulSetHealth(sts *appsv1.StatefulSet) (*HealthStatus, error) if sts.Spec.Replicas != nil && sts.Spec.UpdateStrategy.RollingUpdate.Partition != nil { if sts.Status.UpdatedReplicas < (*sts.Spec.Replicas - *sts.Spec.UpdateStrategy.RollingUpdate.Partition) { return &HealthStatus{ + Health: HealthHealthy, Status: HealthStatusProgressing, Message: fmt.Sprintf("Waiting for partitioned roll out to finish: %d out of %d new pods have been updated...", sts.Status.UpdatedReplicas, (*sts.Spec.Replicas - *sts.Spec.UpdateStrategy.RollingUpdate.Partition)), @@ -48,23 +51,31 @@ func getAppsv1StatefulSetHealth(sts *appsv1.StatefulSet) (*HealthStatus, error) } } return &HealthStatus{ + Ready: true, + Health: HealthHealthy, Status: HealthStatusHealthy, Message: fmt.Sprintf("partitioned roll out complete: %d new pods have been updated...", sts.Status.UpdatedReplicas), }, nil } if sts.Spec.UpdateStrategy.Type == appsv1.OnDeleteStatefulSetStrategyType { return &HealthStatus{ + Ready: true, + Health: HealthHealthy, Status: HealthStatusHealthy, Message: fmt.Sprintf("statefulset has %d ready pods", sts.Status.ReadyReplicas), }, nil } if sts.Status.UpdateRevision != sts.Status.CurrentRevision { return &HealthStatus{ + Health: HealthHealthy, + Ready: true, Status: HealthStatusProgressing, Message: fmt.Sprintf("waiting for statefulset rolling update to complete %d pods at revision %s...", sts.Status.UpdatedReplicas, sts.Status.UpdateRevision), }, nil } return &HealthStatus{ + Health: HealthHealthy, + Ready: true, Status: HealthStatusHealthy, Message: fmt.Sprintf("statefulset rolling update complete %d pods at revision %s...", sts.Status.CurrentReplicas, sts.Status.CurrentRevision), }, nil