Skip to content

Commit

Permalink
golines
Browse files Browse the repository at this point in the history
  • Loading branch information
adityathebe authored and moshloop committed Oct 23, 2024
1 parent d0d9cf8 commit c30c0fd
Show file tree
Hide file tree
Showing 16 changed files with 442 additions and 108 deletions.
5 changes: 5 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -5,3 +5,8 @@ test:
.PHONY: lint
lint:
golangci-lint run

if command -v golines > /dev/null 2>&1; then \
golines -m 120 -w pkg/; \
golines -m 120 -w events/; \
fi
8 changes: 6 additions & 2 deletions pkg/health/health.go
Original file line number Diff line number Diff line change
Expand Up @@ -109,8 +109,12 @@ func GetHealthByConfigType(configType string, obj map[string]any) HealthStatus {
}

// GetResourceHealth returns the health of a k8s resource
func GetResourceHealth(obj *unstructured.Unstructured, healthOverride HealthOverride) (health *HealthStatus, err error) {
if obj.GetDeletionTimestamp() != nil && !obj.GetDeletionTimestamp().IsZero() && time.Since(obj.GetDeletionTimestamp().Time) > time.Hour {
func GetResourceHealth(
obj *unstructured.Unstructured,
healthOverride HealthOverride,
) (health *HealthStatus, err error) {
if obj.GetDeletionTimestamp() != nil && !obj.GetDeletionTimestamp().IsZero() &&
time.Since(obj.GetDeletionTimestamp().Time) > time.Hour {
terminatingFor := time.Since(obj.GetDeletionTimestamp().Time)
return &HealthStatus{
Status: "TerminatingStalled",
Expand Down
14 changes: 12 additions & 2 deletions pkg/health/health_argo.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,9 +36,19 @@ func GetArgoWorkflowHealth(obj *unstructured.Unstructured) (*HealthStatus, error
case "", nodePending:
return &HealthStatus{Health: HealthHealthy, Status: HealthStatusProgressing, Message: wf.Status.Message}, nil
case nodeRunning:
return &HealthStatus{Ready: true, Health: HealthHealthy, Status: HealthStatusProgressing, Message: wf.Status.Message}, nil
return &HealthStatus{
Ready: true,
Health: HealthHealthy,
Status: HealthStatusProgressing,
Message: wf.Status.Message,
}, nil
case nodeSucceeded:
return &HealthStatus{Ready: true, Health: HealthHealthy, Status: HealthStatusHealthy, Message: wf.Status.Message}, nil
return &HealthStatus{
Ready: true,
Health: HealthHealthy,
Status: HealthStatusHealthy,
Message: wf.Status.Message,
}, nil
case nodeFailed, nodeError:
return &HealthStatus{Health: HealthUnhealthy, Status: HealthStatusDegraded, Message: wf.Status.Message}, nil
}
Expand Down
4 changes: 3 additions & 1 deletion pkg/health/health_aws.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,9 @@ const (
func GetAWSResourceHealth(resourceType, status string) (health HealthStatus) {
if resourceStatuses, found := awsResourceHealthmap[resourceType]; found {
if v, found := resourceStatuses[strings.ToLower(status)]; found {
v.Status = HealthStatusCode(lo.Capitalize(strings.ReplaceAll(strings.ReplaceAll(status, "-", " "), "_", " ")))
v.Status = HealthStatusCode(
lo.Capitalize(strings.ReplaceAll(strings.ReplaceAll(status, "-", " "), "_", " ")),
)
return v
}
}
Expand Down
22 changes: 14 additions & 8 deletions pkg/health/health_cronjob.go
Original file line number Diff line number Diff line change
Expand Up @@ -49,17 +49,23 @@ func getBatchv1CronJobHealth(job *batchv1.CronJob) (*HealthStatus, error) {

if job.Status.LastSuccessfulTime.Before(job.Status.LastScheduleTime) {
return &HealthStatus{
Ready: true, // The cronjob did in fact run
Health: HealthUnhealthy,
Status: HealthStatusError,
Message: "Last run failed, last successful run was" + job.Status.LastSuccessfulTime.Format("2006-01-02 15:04:05 -0700"),
Ready: true, // The cronjob did in fact run
Health: HealthUnhealthy,
Status: HealthStatusError,
Message: "Last run failed, last successful run was" + job.Status.LastSuccessfulTime.Format(
"2006-01-02 15:04:05 -0700",
),
}, nil
}

return &HealthStatus{
Ready: true,
Health: HealthHealthy,
Status: HealthStatusCompleted,
Message: fmt.Sprintf("Last run at %s in %s", job.Status.LastScheduleTime.Format("2006-01-02 15:04:05 -0700"), job.Status.LastSuccessfulTime.Sub(job.Status.LastScheduleTime.Time)),
Ready: true,
Health: HealthHealthy,
Status: HealthStatusCompleted,
Message: fmt.Sprintf(
"Last run at %s in %s",
job.Status.LastScheduleTime.Format("2006-01-02 15:04:05 -0700"),
job.Status.LastSuccessfulTime.Sub(job.Status.LastScheduleTime.Time),
),
}, nil
}
35 changes: 24 additions & 11 deletions pkg/health/health_daemonset.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,8 @@ func getAppsv1DaemonSetHealth(daemon *appsv1.DaemonSet) (*HealthStatus, error) {
health = HealthUnhealthy
}

if daemon.Generation == daemon.Status.ObservedGeneration && daemon.Status.UpdatedNumberScheduled == daemon.Status.DesiredNumberScheduled {
if daemon.Generation == daemon.Status.ObservedGeneration &&
daemon.Status.UpdatedNumberScheduled == daemon.Status.DesiredNumberScheduled {
return &HealthStatus{
Health: HealthHealthy,
Ready: true,
Expand All @@ -45,24 +46,36 @@ func getAppsv1DaemonSetHealth(daemon *appsv1.DaemonSet) (*HealthStatus, error) {

if daemon.Spec.UpdateStrategy.Type == appsv1.OnDeleteDaemonSetStrategyType {
return &HealthStatus{
Health: health,
Ready: daemon.Status.NumberAvailable == daemon.Status.DesiredNumberScheduled,
Status: HealthStatusRunning,
Message: fmt.Sprintf("%d of %d pods updated", daemon.Status.UpdatedNumberScheduled, daemon.Status.DesiredNumberScheduled),
Health: health,
Ready: daemon.Status.NumberAvailable == daemon.Status.DesiredNumberScheduled,
Status: HealthStatusRunning,
Message: fmt.Sprintf(
"%d of %d pods updated",
daemon.Status.UpdatedNumberScheduled,
daemon.Status.DesiredNumberScheduled,
),
}, nil
}
if daemon.Status.UpdatedNumberScheduled < daemon.Status.DesiredNumberScheduled {
return &HealthStatus{
Health: health,
Status: HealthStatusRollingOut,
Message: fmt.Sprintf("%d of %d pods updated", daemon.Status.UpdatedNumberScheduled, daemon.Status.DesiredNumberScheduled),
Health: health,
Status: HealthStatusRollingOut,
Message: fmt.Sprintf(
"%d of %d pods updated",
daemon.Status.UpdatedNumberScheduled,
daemon.Status.DesiredNumberScheduled,
),
}, nil
}
if daemon.Status.NumberAvailable < daemon.Status.DesiredNumberScheduled {
return &HealthStatus{
Health: health,
Status: HealthStatusRollingOut,
Message: fmt.Sprintf("%d of %d pods ready", daemon.Status.NumberAvailable, daemon.Status.DesiredNumberScheduled),
Health: health,
Status: HealthStatusRollingOut,
Message: fmt.Sprintf(
"%d of %d pods ready",
daemon.Status.NumberAvailable,
daemon.Status.DesiredNumberScheduled,
),
}, nil
}

Expand Down
13 changes: 9 additions & 4 deletions pkg/health/health_deployment.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,9 @@ func getAppsv1DeploymentHealth(deployment *appsv1.Deployment, obj *unstructured.
var containersWaitingForReadiness []string
for _, container := range deployment.Spec.Template.Spec.Containers {
if container.ReadinessProbe != nil && container.ReadinessProbe.InitialDelaySeconds > 0 {
deadline := deployment.CreationTimestamp.Add(time.Second * time.Duration(container.ReadinessProbe.InitialDelaySeconds))
deadline := deployment.CreationTimestamp.Add(
time.Second * time.Duration(container.ReadinessProbe.InitialDelaySeconds),
)
if time.Now().Before(deadline) {
containersWaitingForReadiness = append(containersWaitingForReadiness, container.Name)
}
Expand All @@ -38,9 +40,12 @@ func getAppsv1DeploymentHealth(deployment *appsv1.Deployment, obj *unstructured.

if len(containersWaitingForReadiness) > 0 {
return &HealthStatus{
Health: HealthUnknown,
Status: HealthStatusStarting,
Message: fmt.Sprintf("Container(s) %s is waiting for readiness probe", strings.Join(containersWaitingForReadiness, ",")),
Health: HealthUnknown,
Status: HealthStatusStarting,
Message: fmt.Sprintf(
"Container(s) %s is waiting for readiness probe",
strings.Join(containersWaitingForReadiness, ","),
),
}, nil
}

Expand Down
35 changes: 27 additions & 8 deletions pkg/health/health_pod.go
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,8 @@ func getCorev1PodHealth(pod *corev1.Pod) (*HealthStatus, error) {
getCommonContainerError := func(containerStatus *corev1.ContainerStatus) *HealthStatus {
waiting := containerStatus.State.Waiting
// Article listing common container errors: https://medium.com/kokster/debugging-crashloopbackoffs-with-init-containers-26f79e9fb5bf
if waiting != nil && (strings.HasPrefix(waiting.Reason, "Err") || strings.HasSuffix(waiting.Reason, "Error") || strings.HasSuffix(waiting.Reason, "BackOff")) {
if waiting != nil &&
(strings.HasPrefix(waiting.Reason, "Err") || strings.HasSuffix(waiting.Reason, "Error") || strings.HasSuffix(waiting.Reason, "BackOff")) {
return &HealthStatus{
Status: HealthStatusCode(waiting.Reason),
Health: HealthUnhealthy,
Expand Down Expand Up @@ -115,7 +116,8 @@ func getCorev1PodHealth(pod *corev1.Pod) (*HealthStatus, error) {
switch pod.Status.Phase {
case corev1.PodPending:
for _, ctrStatus := range pod.Status.InitContainerStatuses {
if ctrStatus.LastTerminationState.Terminated != nil && ctrStatus.LastTerminationState.Terminated.Reason == "Error" {
if ctrStatus.LastTerminationState.Terminated != nil &&
ctrStatus.LastTerminationState.Terminated.Reason == "Error" {
// A pending pod whose container was previously terminated with error should be marked as unhealthy (instead of unknown)
return &HealthStatus{
Health: HealthUnhealthy,
Expand Down Expand Up @@ -156,7 +158,12 @@ func getCorev1PodHealth(pod *corev1.Pod) (*HealthStatus, error) {
case corev1.PodFailed:
if pod.Status.Message != "" {
// Pod has a nice error message. Use that.
return &HealthStatus{Health: HealthUnhealthy, Status: HealthStatusError, Ready: true, Message: pod.Status.Message}, nil
return &HealthStatus{
Health: HealthUnhealthy,
Status: HealthStatusError,
Ready: true,
Message: pod.Status.Message,
}, nil
}
for _, ctr := range append(pod.Status.InitContainerStatuses, pod.Status.ContainerStatuses...) {
if msg := getFailMessage(&ctr); msg != "" {
Expand Down Expand Up @@ -186,7 +193,11 @@ func getCorev1PodHealth(pod *corev1.Pod) (*HealthStatus, error) {

if possiblyInRestartLoop {
lastTerminatedTime := s.LastTerminationState.Terminated.FinishedAt.Time
h.Message = fmt.Sprintf("%s has restarted %d time(s)", s.Name, pod.Status.ContainerStatuses[0].RestartCount)
h.Message = fmt.Sprintf(
"%s has restarted %d time(s)",
s.Name,
pod.Status.ContainerStatuses[0].RestartCount,
)

if s.LastTerminationState.Terminated.Reason != "Completed" {
h.Status = HealthStatusCode(s.LastTerminationState.Terminated.Reason)
Expand Down Expand Up @@ -243,16 +254,24 @@ func getCorev1PodHealth(pod *corev1.Pod) (*HealthStatus, error) {
continue
}

if c.Status.State.Running != nil && time.Since(c.Status.State.Running.StartedAt.Time) <= time.Duration(c.Spec.ReadinessProbe.InitialDelaySeconds)*time.Second {
if c.Status.State.Running != nil &&
time.Since(
c.Status.State.Running.StartedAt.Time,
) <= time.Duration(
c.Spec.ReadinessProbe.InitialDelaySeconds,
)*time.Second {
containersWaitingForReadinessProbe = append(containersWaitingForReadinessProbe, c.Spec.Name)
}
}

// otherwise we are progressing towards a ready state
return &HealthStatus{
Health: HealthUnknown,
Status: HealthStatusStarting,
Message: fmt.Sprintf("Container %s is waiting for readiness probe", strings.Join(containersWaitingForReadinessProbe, ",")),
Health: HealthUnknown,
Status: HealthStatusStarting,
Message: fmt.Sprintf(
"Container %s is waiting for readiness probe",
strings.Join(containersWaitingForReadinessProbe, ","),
),
}, nil

case corev1.RestartPolicyOnFailure, corev1.RestartPolicyNever:
Expand Down
21 changes: 15 additions & 6 deletions pkg/health/health_replicaset.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,9 @@ func getAppsv1ReplicaSetHealth(replicaSet *appsv1.ReplicaSet) (*HealthStatus, er
var containersWaitingForReadiness []string
for _, container := range replicaSet.Spec.Template.Spec.Containers {
if container.ReadinessProbe != nil && container.ReadinessProbe.InitialDelaySeconds > 0 {
deadline := replicaSet.CreationTimestamp.Add(time.Second * time.Duration(container.ReadinessProbe.InitialDelaySeconds))
deadline := replicaSet.CreationTimestamp.Add(
time.Second * time.Duration(container.ReadinessProbe.InitialDelaySeconds),
)
if time.Now().Before(deadline) {
containersWaitingForReadiness = append(containersWaitingForReadiness, container.Name)
}
Expand All @@ -45,9 +47,12 @@ func getAppsv1ReplicaSetHealth(replicaSet *appsv1.ReplicaSet) (*HealthStatus, er

if len(containersWaitingForReadiness) > 0 {
return &HealthStatus{
Health: HealthUnknown,
Status: HealthStatusStarting,
Message: fmt.Sprintf("Container(s) %s is waiting for readiness probe", strings.Join(containersWaitingForReadiness, ",")),
Health: HealthUnknown,
Status: HealthStatusStarting,
Message: fmt.Sprintf(
"Container(s) %s is waiting for readiness probe",
strings.Join(containersWaitingForReadiness, ","),
),
}, nil
}

Expand All @@ -73,7 +78,8 @@ func getAppsv1ReplicaSetHealth(replicaSet *appsv1.ReplicaSet) (*HealthStatus, er
health = HealthUnknown
}

if replicaSet.Generation == replicaSet.Status.ObservedGeneration && replicaSet.Status.ReadyReplicas == *replicaSet.Spec.Replicas {
if replicaSet.Generation == replicaSet.Status.ObservedGeneration &&
replicaSet.Status.ReadyReplicas == *replicaSet.Spec.Replicas {
return &HealthStatus{
Health: health,
Status: HealthStatusRunning,
Expand Down Expand Up @@ -112,7 +118,10 @@ func getAppsv1ReplicaSetHealth(replicaSet *appsv1.ReplicaSet) (*HealthStatus, er
}, nil
}

func getAppsv1ReplicaSetCondition(status appsv1.ReplicaSetStatus, condType appsv1.ReplicaSetConditionType) *appsv1.ReplicaSetCondition {
func getAppsv1ReplicaSetCondition(
status appsv1.ReplicaSetStatus,
condType appsv1.ReplicaSetConditionType,
) *appsv1.ReplicaSetCondition {
for i := range status.Conditions {
c := status.Conditions[i]
if c.Type == condType {
Expand Down
13 changes: 9 additions & 4 deletions pkg/health/health_statefulset.go
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,9 @@ func getAppsv1StatefulSetHealth(sts *appsv1.StatefulSet) (*HealthStatus, error)
var containersWaitingForReadiness []string
for _, container := range sts.Spec.Template.Spec.Containers {
if container.ReadinessProbe != nil && container.ReadinessProbe.InitialDelaySeconds > 0 {
deadline := sts.CreationTimestamp.Add(time.Second * time.Duration(container.ReadinessProbe.InitialDelaySeconds))
deadline := sts.CreationTimestamp.Add(
time.Second * time.Duration(container.ReadinessProbe.InitialDelaySeconds),
)
if time.Now().Before(deadline) {
containersWaitingForReadiness = append(containersWaitingForReadiness, container.Name)
}
Expand All @@ -50,9 +52,12 @@ func getAppsv1StatefulSetHealth(sts *appsv1.StatefulSet) (*HealthStatus, error)

if len(containersWaitingForReadiness) > 0 {
return &HealthStatus{
Health: HealthUnknown,
Status: HealthStatusStarting,
Message: fmt.Sprintf("Container(s) %s is waiting for readiness probe", strings.Join(containersWaitingForReadiness, ",")),
Health: HealthUnknown,
Status: HealthStatusStarting,
Message: fmt.Sprintf(
"Container(s) %s is waiting for readiness probe",
strings.Join(containersWaitingForReadiness, ","),
),
}, nil
}

Expand Down
Loading

0 comments on commit c30c0fd

Please sign in to comment.