From 5014c3f6e0db4716fe12d2836631606b0b514c2b Mon Sep 17 00:00:00 2001 From: Comte Date: Fri, 28 Jun 2024 11:37:14 +0200 Subject: [PATCH] Return the list of controllers to better determine service's health (#428) --- .../api/services/impl/HelmAppsService.java | 13 +++ .../impl/HelmReleaseHealthResolver.java | 93 +++++++++++++++++++ .../model/service/HealthCheckResult.java | 70 ++++++++++++++ .../insee/onyxia/model/service/Service.java | 11 +++ 4 files changed, 187 insertions(+) create mode 100644 onyxia-api/src/main/java/fr/insee/onyxia/api/services/impl/HelmReleaseHealthResolver.java create mode 100644 onyxia-model/src/main/java/fr/insee/onyxia/model/service/HealthCheckResult.java diff --git a/onyxia-api/src/main/java/fr/insee/onyxia/api/services/impl/HelmAppsService.java b/onyxia-api/src/main/java/fr/insee/onyxia/api/services/impl/HelmAppsService.java index 4b599ba4..d3a014b5 100644 --- a/onyxia-api/src/main/java/fr/insee/onyxia/api/services/impl/HelmAppsService.java +++ b/onyxia-api/src/main/java/fr/insee/onyxia/api/services/impl/HelmAppsService.java @@ -1,5 +1,6 @@ package fr.insee.onyxia.api.services.impl; +import static fr.insee.onyxia.api.services.impl.HelmReleaseHealthResolver.checkHelmReleaseHealth; import static fr.insee.onyxia.api.services.impl.ServiceUrlResolver.getServiceUrls; import com.fasterxml.jackson.databind.JsonNode; @@ -619,6 +620,18 @@ private Service getServiceFromRelease( service.setUrls(List.of()); } + try { + List controllers = + checkHelmReleaseHealth(release.getNamespace(), manifest, client); + service.setControllers(controllers); + } catch (Exception e) { + LOGGER.warn( + "Failed to retrieve controllers for release {} namespace {}", + release.getName(), + release.getNamespace(), + e); + service.setControllers(List.of()); + } service.setInstances(1); service.setTasks( diff --git a/onyxia-api/src/main/java/fr/insee/onyxia/api/services/impl/HelmReleaseHealthResolver.java b/onyxia-api/src/main/java/fr/insee/onyxia/api/services/impl/HelmReleaseHealthResolver.java new file mode 100644 index 00000000..50a2ab8b --- /dev/null +++ b/onyxia-api/src/main/java/fr/insee/onyxia/api/services/impl/HelmReleaseHealthResolver.java @@ -0,0 +1,93 @@ +package fr.insee.onyxia.api.services.impl; + +import fr.insee.onyxia.model.service.HealthCheckResult; +import io.fabric8.kubernetes.api.model.HasMetadata; +import io.fabric8.kubernetes.api.model.apps.DaemonSet; +import io.fabric8.kubernetes.api.model.apps.Deployment; +import io.fabric8.kubernetes.api.model.apps.StatefulSet; +import io.fabric8.kubernetes.client.KubernetesClient; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public final class HelmReleaseHealthResolver { + + private static final Logger LOGGER = LoggerFactory.getLogger(HelmReleaseHealthResolver.class); + + static List checkHelmReleaseHealth( + String namespace, String manifest, KubernetesClient kubernetesClient) { + // Identify the Helm release secret + List resources; + try (InputStream inputStream = + new ByteArrayInputStream(manifest.getBytes(StandardCharsets.UTF_8))) { + resources = kubernetesClient.load(inputStream).items(); + } catch (IOException e) { + throw new RuntimeException("Exception during loading manifest", e); + } + + return checkHealth(namespace, resources, kubernetesClient); + } + + private static List checkHealth( + String namespace, List resources, KubernetesClient kubernetesClient) { + List results = new ArrayList<>(); + for (HasMetadata resource : resources) { + String name = resource.getMetadata().getName(); + String kind = resource.getKind(); + HealthCheckResult result = new HealthCheckResult(); + result.setName(name); + result.setKind(kind); + HealthCheckResult.HealthDetails details = new HealthCheckResult.HealthDetails(); + try { + switch (kind) { + case "Deployment": + Deployment deployment = + kubernetesClient + .apps() + .deployments() + .inNamespace(namespace) + .withName(name) + .get(); + details.setDesired(deployment.getSpec().getReplicas()); + details.setReady(deployment.getStatus().getReadyReplicas()); + case "StatefulSet": + StatefulSet statefulset = + kubernetesClient + .apps() + .statefulSets() + .inNamespace(namespace) + .withName(name) + .get(); + details.setDesired(statefulset.getSpec().getReplicas()); + details.setReady(statefulset.getStatus().getReadyReplicas()); + case "DaemonSet": + DaemonSet daemonSet = + kubernetesClient + .apps() + .daemonSets() + .inNamespace(namespace) + .withName(name) + .get(); + details.setDesired(daemonSet.getStatus().getDesiredNumberScheduled()); + details.setReady(daemonSet.getStatus().getNumberReady()); + default: + continue; + } + } catch (Exception e) { + LOGGER.warn( + "Could not retrieve health status from resource kind {} name {} ", + resource.getKind(), + resource.getMetadata().getName()); + } + result.setDetails(details); + result.setHealthy(details.getReady() >= details.getDesired()); + results.add(result); + } + return results; + } +} diff --git a/onyxia-model/src/main/java/fr/insee/onyxia/model/service/HealthCheckResult.java b/onyxia-model/src/main/java/fr/insee/onyxia/model/service/HealthCheckResult.java new file mode 100644 index 00000000..b41e0697 --- /dev/null +++ b/onyxia-model/src/main/java/fr/insee/onyxia/model/service/HealthCheckResult.java @@ -0,0 +1,70 @@ +package fr.insee.onyxia.model.service; + +public class HealthCheckResult { + private boolean healthy; + private String name; + private String kind; + private HealthDetails details; + + public HealthCheckResult() {} + + public HealthCheckResult(boolean healthy, String name, String kind, HealthDetails details) { + this.healthy = healthy; + this.name = name; + this.kind = kind; + this.details = details; + } + + public boolean isHealthy() { + return healthy; + } + + public void setHealthy(boolean healthy) { + this.healthy = healthy; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getKind() { + return kind; + } + + public void setKind(String kind) { + this.kind = kind; + } + + public HealthDetails getDetails() { + return details; + } + + public void setDetails(HealthDetails details) { + this.details = details; + } + + public static class HealthDetails { + private int desired; + private int ready; + + public int getReady() { + return ready; + } + + public int getDesired() { + return desired; + } + + public void setReady(int ready) { + this.ready = ready; + } + + public void setDesired(int desired) { + this.desired = desired; + } + } +} diff --git a/onyxia-model/src/main/java/fr/insee/onyxia/model/service/Service.java b/onyxia-model/src/main/java/fr/insee/onyxia/model/service/Service.java index 60f422a5..76294224 100644 --- a/onyxia-model/src/main/java/fr/insee/onyxia/model/service/Service.java +++ b/onyxia-model/src/main/java/fr/insee/onyxia/model/service/Service.java @@ -92,6 +92,9 @@ public class Service { @Schema(description = "") private Map labels; + @Schema(description = "") + private List controllers; + public String getId() { return id; } @@ -276,6 +279,14 @@ public void setSuspended(boolean suspended) { this.suspended = suspended; } + public List getControllers() { + return controllers; + } + + public void setControllers(List controllers) { + this.controllers = controllers; + } + public String getCatalogId() { return catalogId; }