revert startup, debug msg for server health

.github/workflows/docker-img.yml

@@ -219,10 +219,10 @@ jobs:
         VERSION=${{ needs.setup-envvar.outputs.version }}
 
         # TODO use label exclusively in the future
-        POD=$(kubectl get pod -l role=server | grep rc-siibra-api | awk '{print $1}')
+        POD=$(kubectl get pod -l role=server -l app-flavor=rc -o json | jq -r '.items[0].metadata.name')
         echo POD: $POD
 
-        cache_str=$(kubectl exec $POD env | grep SIIBRA_CACHEDIR)
+        cache_str=$(kubectl exec $POD -- env | grep SIIBRA_CACHEDIR)
         FROM_DIR=${cache_str//SIIBRA_CACHEDIR=/}
         TO_DIR=${FROM_DIR//-rc/}
         POD_NAME=copy-cache

.helm/adhoc/copy-cache.yaml

@@ -12,11 +12,11 @@ spec:
     - /bin/ash
     args:
     - -c
-    - "echo FROM_DIR: $FROM_DIR TO_DIR: $TO_DIR && cp -r $FROM_DIR $TO_DIR"
+    - "echo FROM_DIR: $FROM_DIR TO_DIR: $TO_DIR && mkdir $TO_DIR && cp -r $FROM_DIR/* $TO_DIR/"
     resources:
       limits:
-        cpu: 200m
-        memory: 200Mi
+        cpu: 1000m
+        memory: 2000Mi
       requests:
         cpu: 200m
         memory: 200Mi

api/server/api.py

@@ -30,21 +30,36 @@
 
 siibra_version_header = "x-siibra-api-version"
 
-@asynccontextmanager
-async def lifespan():
-    on_startup()
-    metrics_on_startup()
-    yield
-    terminate()
-    metrics_on_terminate()
+
+# TODO lifespan not working properly. Fix and use lifespan in future
+# @asynccontextmanager
+# async def lifespan(app: FastAPI):
+#     on_startup()
+#     metrics_on_startup()
+#     yield
+#     terminate()
+#     metrics_on_terminate()
 
 siibra_api = FastAPI(
     title="siibra api",
     description="This is the REST api for siibra tools",
     version=__version__,
-    lifespan=lifespan,
+    # lifespan=lifespan,
 )
 
+
+@siibra_api.on_event("shutdown")
+def shutdown():
+    terminate()
+    metrics_on_terminate()
+
+
+@siibra_api.on_event("startup")
+def startup():
+    on_startup()
+    metrics_on_startup()
+
+
 for prefix_router in [*core_prefixed_routers, *volume_prefixed_routers, *compound_prefixed_routers]:
     siibra_api.include_router(prefix_router.router, prefix=prefix_router.prefix)
 

server_health.py

@@ -5,17 +5,22 @@
 url = "http://localhost:5000/metrics"
 
 def main():
-    resp = requests.get(url)
-    for line in resp.text.split("\n"):
-        if line.startswith("#"):
-            continue
-        if "last_pinged" in line:
-            label, value = line.split(" ")
-            time_checked = float(value)
-            diff_sec = time.time() - time_checked
-            if diff_sec < 60 * 5:
-                return 0
-    return 1
+    try:
+        resp = requests.get(url)
+        resp.raise_for_status()
+        for line in resp.text.split("\n"):
+            if line.startswith("#"):
+                continue
+            if "last_pinged" in line:
+                label, value = line.split(" ")
+                time_checked = float(value)
+                diff_sec = time.time() - time_checked
+                if diff_sec < 60 * 5:
+                    return 0
+        return 1
+    except Exception as e:
+        print(f"Err: {str(e)}")
+        return 2
 
 if __name__ == "__main__":
     sys.exit(main())