feat: update siibra version (feature names)

.github/workflows/deploy-helm.yml

@@ -27,10 +27,19 @@ jobs:
         if [[ $helm_status = "0" ]]
         then
           echo "tag ${{ inputs.DEPLOYMENT_NAME }} found. Update"
-          helm --set sapiVersion=${version} --kubeconfig=$kubecfg_path upgrade ${{ inputs.DEPLOYMENT_NAME }} .helm/siibra-api/
+          helm --set sapiVersion=${version} \
+            --kubeconfig=$kubecfg_path \
+            --history-max=3 \
+            upgrade \
+            ${{ inputs.DEPLOYMENT_NAME }} \
+            .helm/siibra-api/
         else
           echo "tag ${{ inputs.DEPLOYMENT_NAME }} not found. Install"
-          helm --set sapiVersion=${version} --kubeconfig=$kubecfg_path install ${{ inputs.DEPLOYMENT_NAME }} .helm/siibra-api/
+          helm --set sapiVersion=${version} \
+            --kubeconfig=$kubecfg_path \
+            install \
+            ${{ inputs.DEPLOYMENT_NAME }} \
+            .helm/siibra-api/
         fi
 
         rm $kubecfg_path

.github/workflows/docker-img.yml

@@ -196,7 +196,68 @@ jobs:
     secrets:
       okd_token: ${{ matrix.deploy-site == 'jsc' && secrets.OKD_JSC_SECRET || secrets.OKD_PROD_SECRET }}
 
+  warmup-rc-at-helm:
+    needs: setup-envvar
+    if:  ${{ github.event_name == 'release' && contains(github.ref, 'rc') }}
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - timeout-minutes: 15 # should not take more than 15 minutes to warmup cache
+      run: |
+        kubecfg_path=${{ runner.temp }}/.kube_config
+        echo "${{ secrets.KUBECONFIG }}" > $kubecfg_path
+
+        kubectl delete pod/warmup-pod
+
+        # TODO Flaky
+        # see .helm/siibra-api/templates/_helpers.tpl how siibra-api.cache-dir is defined
+
+        SIIBRA_CACHEDIR=/siibra-api-volume/${{ needs.setup-envvar.outputs.version }}-rc/
+        WARM_CACHE_YML=$(envsubst < .helm/adhoc/warm-cache.yaml)
+        echo WARM_CACHE_YML: $WARM_CACHE_YML
+
+        echo $WARM_CACHE_YML | kubectl apply -f -
+
+        while true
+        do
+          sleep 10
+          POD_PHASE=$(kubectl get pod busybox -o json | jq -r '.status.phase')
+
+          echo Possible phases: Pending, Running, Succeeded, Failed, Unknown
+          echo Found phase: $POD_PHASE
+
+          if [[ "$POD_PHASE" == "Failed" ]] || [[ "$POD_PHASE" == "Unknown" ]]
+          then
+            exit 1
+          fi
+
+          if [[ "$POD_PHASE" == "Succeeded" ]]
+          then
+            exit 0
+          fi
+        done
+
+  clear-rc-redis-cache:
+    runs-on: 'ubuntu'
+    timeout-minutes: 1 # should not take more than 1 minute to clear the cache
+    needs:
+    - warmup-rc-at-helm
+    - setup-envvar
+    steps:
+    - run: |
+        REDIS_POD=$(kubectl get pod -l app=cache-redis | grep Running | awk '{print $1}')
+        echo kubectl exec $REDIS_POD -- /bin/ash -c "redis-cli redis-cli --scan --pattern "*${{ needs.setup-envvar.outputs.version }}*" | while IFS= read -r line; do redis-cli del "$line"; done"
+        kubectl exec $REDIS_POD -- /bin/ash -c "redis-cli redis-cli --scan --pattern "*${{ needs.setup-envvar.outputs.version }}*" | while IFS= read -r line; do redis-cli del "$line"; done"
 
+  deploy-rc-via-helm:
+    needs: warmup-rc-at-helm
+    if:  ${{ github.event_name == 'release' && contains(github.ref, 'rc') }}
+    uses: ./.github/workflows/deploy-helm.yml
+    with:
+      DEPLOYMENT_NAME: rc
+    secrets:
+      KUBECONFIG: ${{ secrets.KUBECONFIG }}
+
   deploy-prod-via-helm:
     needs: setup-envvar
     if:  ${{ github.event_name == 'release' && !contains(github.ref, 'rc') }}

.helm/adhoc/warm-cache.yaml

@@ -0,0 +1,35 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  labels:
+    app: warmup-pod
+  name: warmup-pod
+spec:
+  containers:
+  - name: warmup-container
+    image: docker-registry.ebrains.eu/siibra/siibra-api:rc-worker
+    command:
+    - python
+    args:
+    - -c
+    - import siibra; siibra.cache.clear(); siibra.warm_cache()
+    resources:
+      limits:
+        cpu: 900m
+        memory: 1Gi
+      requests:
+        cpu: 500m
+        memory: 1Gi
+    volumeMounts:
+    - mountPath: /siibra-api-volume
+      name: data-volume
+    env:
+    - name: SIIBRA_CACHEDIR
+      value: "$SIIBRA_CACHEDIR"
+    - name: SKIP_CACHEINIT_MAINTENANCE
+      value: "1"
+  restartPolicy: Never
+  volumes:
+  - name: data-volume
+    persistentVolumeClaim:
+      claimName: data-volume-claim

.helm/siibra-api/templates/_helpers.tpl

@@ -5,6 +5,9 @@ Expand the name of the chart.
 {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
 {{- end }}
 
+{{/*
+Define root image
+*/}}
 {{- define "siibra-api.root-img" -}}
 {{- if eq .Values.sapiFlavor "rc" }}
 {{- "rc" }}
@@ -13,6 +16,25 @@ Expand the name of the chart.
 {{- end }}
 {{- end }}
 
+
+{{/*
+Define cache-dir. append -rc if is rc
+This is because, on deploy staging it will rm -rf cache-dir.
+This should prevent misconfiguration from deleting prod cache
+*/}}
+{{- define "siibra-api.cache-dir" -}}
+{{- if eq .Values.sapiFlavor "rc" }}
+{{/*
+N.B. *any* update here *needs* to be reflected in
+.github/workflows/docker-img.yml#jobs>warmup-rc-at-helm
+*/}}
+{{- printf "%s-%s" .Values.sapiVersion "rc" }}
+{{- else }}
+{{- .Values.sapiVersion }}
+{{- end }}
+{{- end }}
+
+
 {{/*
 Create a default fully qualified app name.
 We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).

.helm/siibra-api/templates/deployment-server.yaml

@@ -58,7 +58,7 @@ spec:
                 name: siibra-api-common
           env:
             - name: SIIBRA_CACHEDIR
-              value: "/siibra-api-volume/{{ .Values.sapiVersion }}"
+              value: "/siibra-api-volume/{{ include "siibra-api.cache-dir" . }}"
             - name: SIIBRA_API_NAMESPACE
               value: {{ $.Values.sapiFlavor }}
           resources:

.helm/siibra-api/templates/deployment-worker.yaml

@@ -52,7 +52,7 @@ spec:
           - "api.worker.app"
           - "worker"
           - "-Q"
-          - "{{ $.Values.sapiVersion }}.siibraapi.{{ . }}"
+          - "{{ $.Values.sapiVersion }}.{{ $.Values.sapiFlavor }}.{{ . }}"
           - "-O"
           - "fair"
           livenessProbe:
@@ -74,7 +74,7 @@ spec:
                 name: siibra-api-common
           env:
             - name: SIIBRA_CACHEDIR
-              value: "/siibra-api-volume/{{ $.Values.sapiVersion }}"
+              value: "/siibra-api-volume/{{ include "siibra-api.cache-dir" $ }}"
             - name: SIIBRA_API_NAMESPACE
               value: {{ $.Values.sapiFlavor }}
           resources:

api/common/data_handlers/compounds/download.py

@@ -64,8 +64,8 @@ def download_all(space_id: str, parcellation_id: str, region_id: str=None, featu
             try:
                 path_to_feature_export = Path(SIIBRA_API_SHARED_DIR, f"export-{feature_id}.zip")
                 if not path_to_feature_export.exists():
-                    feature = siibra.features.Feature.get_instance_by_id(feature_id)
-                    feature.export(path_to_feature_export)
+                    feature = siibra.features.Feature._get_instance_by_id(feature_id)
+                    feature.to_zip(path_to_feature_export)
                 zipfile.write(path_to_feature_export, f"export-{feature_id}.zip")
             except Exception as e:
                 zipfile.writestr(f"{feature_id}.error.txt", f"Feature exporting failed: {str(e)}")

api/common/data_handlers/features/types.py

@@ -110,15 +110,9 @@ def get_single_feature_download_zip_path(feature_id: str, **kwargs):
     except Exception as e:
         general_logger.error(f"Error finding single feature {feature_id=}, {str(e)}")
         raise NotFound from e
-    try:
-        feat.export(str(full_filename))
-        return str(full_filename)
-    except Exception as e:
-        general_logger.error(f"Error export single feature {feature_id=}, {str(e)}")
-        error_filename = full_filename.with_suffix(".error.zip")
-        with ZipFile(error_filename, "w") as zf:
-            zf.writestr("error.txt", f"Error exporting file for feature_id: {feature_id}: {str(e)}")
-        return str(error_filename)
+
+    feat.to_zip(str(full_filename))
+    return str(full_filename)
 
 
 def extract_concept(*, space_id: str=None, parcellation_id: str=None, region_id: str=None, bbox: str=None, **kwargs):

requirements/siibra.txt

@@ -1,3 +1,3 @@
 # siibra==1.0a5
-git+https://github.com/FZJ-INM1-BDA/siibra-python.git@50f038f
+git+https://github.com/FZJ-INM1-BDA/siibra-python.git@6135e6e8b4158676a9c80e64510495a5900f3912
 plotly