diff --git a/.github/workflows/deploy-helm.yml b/.github/workflows/deploy-helm.yml index e29c994..3219d35 100644 --- a/.github/workflows/deploy-helm.yml +++ b/.github/workflows/deploy-helm.yml @@ -27,10 +27,19 @@ jobs: if [[ $helm_status = "0" ]] then echo "tag ${{ inputs.DEPLOYMENT_NAME }} found. Update" - helm --set sapiVersion=${version} --kubeconfig=$kubecfg_path upgrade ${{ inputs.DEPLOYMENT_NAME }} .helm/siibra-api/ + helm --set sapiVersion=${version} \ + --kubeconfig=$kubecfg_path \ + --history-max=3 \ + upgrade \ + ${{ inputs.DEPLOYMENT_NAME }} \ + .helm/siibra-api/ else echo "tag ${{ inputs.DEPLOYMENT_NAME }} not found. Install" - helm --set sapiVersion=${version} --kubeconfig=$kubecfg_path install ${{ inputs.DEPLOYMENT_NAME }} .helm/siibra-api/ + helm --set sapiVersion=${version} \ + --kubeconfig=$kubecfg_path \ + install \ + ${{ inputs.DEPLOYMENT_NAME }} \ + .helm/siibra-api/ fi rm $kubecfg_path diff --git a/.github/workflows/docker-img.yml b/.github/workflows/docker-img.yml index 7b34965..4ae7f4c 100644 --- a/.github/workflows/docker-img.yml +++ b/.github/workflows/docker-img.yml @@ -196,7 +196,68 @@ jobs: secrets: okd_token: ${{ matrix.deploy-site == 'jsc' && secrets.OKD_JSC_SECRET || secrets.OKD_PROD_SECRET }} + warmup-rc-at-helm: + needs: setup-envvar + if: ${{ github.event_name == 'release' && contains(github.ref, 'rc') }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - timeout-minutes: 15 # should not take more than 15 minutes to warmup cache + run: | + kubecfg_path=${{ runner.temp }}/.kube_config + echo "${{ secrets.KUBECONFIG }}" > $kubecfg_path + + kubectl delete pod/warmup-pod + + # TODO Flaky + # see .helm/siibra-api/templates/_helpers.tpl how siibra-api.cache-dir is defined + + SIIBRA_CACHEDIR=/siibra-api-volume/${{ needs.setup-envvar.outputs.version }}-rc/ + WARM_CACHE_YML=$(envsubst < .helm/adhoc/warm-cache.yaml) + echo WARM_CACHE_YML: $WARM_CACHE_YML + + echo $WARM_CACHE_YML | kubectl apply -f - + + while true + do + sleep 10 + POD_PHASE=$(kubectl get pod busybox -o json | jq -r '.status.phase') + + echo Possible phases: Pending, Running, Succeeded, Failed, Unknown + echo Found phase: $POD_PHASE + + if [[ "$POD_PHASE" == "Failed" ]] || [[ "$POD_PHASE" == "Unknown" ]] + then + exit 1 + fi + + if [[ "$POD_PHASE" == "Succeeded" ]] + then + exit 0 + fi + done + + clear-rc-redis-cache: + runs-on: 'ubuntu' + timeout-minutes: 1 # should not take more than 1 minute to clear the cache + needs: + - warmup-rc-at-helm + - setup-envvar + steps: + - run: | + REDIS_POD=$(kubectl get pod -l app=cache-redis | grep Running | awk '{print $1}') + echo kubectl exec $REDIS_POD -- /bin/ash -c "redis-cli redis-cli --scan --pattern "*${{ needs.setup-envvar.outputs.version }}*" | while IFS= read -r line; do redis-cli del "$line"; done" + kubectl exec $REDIS_POD -- /bin/ash -c "redis-cli redis-cli --scan --pattern "*${{ needs.setup-envvar.outputs.version }}*" | while IFS= read -r line; do redis-cli del "$line"; done" + deploy-rc-via-helm: + needs: warmup-rc-at-helm + if: ${{ github.event_name == 'release' && contains(github.ref, 'rc') }} + uses: ./.github/workflows/deploy-helm.yml + with: + DEPLOYMENT_NAME: rc + secrets: + KUBECONFIG: ${{ secrets.KUBECONFIG }} + deploy-prod-via-helm: needs: setup-envvar if: ${{ github.event_name == 'release' && !contains(github.ref, 'rc') }} diff --git a/.helm/adhoc/warm-cache.yaml b/.helm/adhoc/warm-cache.yaml new file mode 100644 index 0000000..f78260f --- /dev/null +++ b/.helm/adhoc/warm-cache.yaml @@ -0,0 +1,35 @@ +apiVersion: v1 +kind: Pod +metadata: + labels: + app: warmup-pod + name: warmup-pod +spec: + containers: + - name: warmup-container + image: docker-registry.ebrains.eu/siibra/siibra-api:rc-worker + command: + - python + args: + - -c + - import siibra; siibra.cache.clear(); siibra.warm_cache() + resources: + limits: + cpu: 900m + memory: 1Gi + requests: + cpu: 500m + memory: 1Gi + volumeMounts: + - mountPath: /siibra-api-volume + name: data-volume + env: + - name: SIIBRA_CACHEDIR + value: "$SIIBRA_CACHEDIR" + - name: SKIP_CACHEINIT_MAINTENANCE + value: "1" + restartPolicy: Never + volumes: + - name: data-volume + persistentVolumeClaim: + claimName: data-volume-claim diff --git a/.helm/siibra-api/templates/_helpers.tpl b/.helm/siibra-api/templates/_helpers.tpl index 9d97ece..8be824e 100644 --- a/.helm/siibra-api/templates/_helpers.tpl +++ b/.helm/siibra-api/templates/_helpers.tpl @@ -5,6 +5,9 @@ Expand the name of the chart. {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} {{- end }} +{{/* +Define root image +*/}} {{- define "siibra-api.root-img" -}} {{- if eq .Values.sapiFlavor "rc" }} {{- "rc" }} @@ -13,6 +16,25 @@ Expand the name of the chart. {{- end }} {{- end }} + +{{/* +Define cache-dir. append -rc if is rc +This is because, on deploy staging it will rm -rf cache-dir. +This should prevent misconfiguration from deleting prod cache +*/}} +{{- define "siibra-api.cache-dir" -}} +{{- if eq .Values.sapiFlavor "rc" }} +{{/* +N.B. *any* update here *needs* to be reflected in +.github/workflows/docker-img.yml#jobs>warmup-rc-at-helm +*/}} +{{- printf "%s-%s" .Values.sapiVersion "rc" }} +{{- else }} +{{- .Values.sapiVersion }} +{{- end }} +{{- end }} + + {{/* Create a default fully qualified app name. We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). diff --git a/.helm/siibra-api/templates/deployment-server.yaml b/.helm/siibra-api/templates/deployment-server.yaml index 0e4b880..3a6d842 100644 --- a/.helm/siibra-api/templates/deployment-server.yaml +++ b/.helm/siibra-api/templates/deployment-server.yaml @@ -58,7 +58,7 @@ spec: name: siibra-api-common env: - name: SIIBRA_CACHEDIR - value: "/siibra-api-volume/{{ .Values.sapiVersion }}" + value: "/siibra-api-volume/{{ include "siibra-api.cache-dir" . }}" - name: SIIBRA_API_NAMESPACE value: {{ $.Values.sapiFlavor }} resources: diff --git a/.helm/siibra-api/templates/deployment-worker.yaml b/.helm/siibra-api/templates/deployment-worker.yaml index 6f32ce2..2fee1a6 100644 --- a/.helm/siibra-api/templates/deployment-worker.yaml +++ b/.helm/siibra-api/templates/deployment-worker.yaml @@ -52,7 +52,7 @@ spec: - "api.worker.app" - "worker" - "-Q" - - "{{ $.Values.sapiVersion }}.siibraapi.{{ . }}" + - "{{ $.Values.sapiVersion }}.{{ $.Values.sapiFlavor }}.{{ . }}" - "-O" - "fair" livenessProbe: @@ -74,7 +74,7 @@ spec: name: siibra-api-common env: - name: SIIBRA_CACHEDIR - value: "/siibra-api-volume/{{ $.Values.sapiVersion }}" + value: "/siibra-api-volume/{{ include "siibra-api.cache-dir" $ }}" - name: SIIBRA_API_NAMESPACE value: {{ $.Values.sapiFlavor }} resources: diff --git a/api/common/data_handlers/compounds/download.py b/api/common/data_handlers/compounds/download.py index 4626c41..e4f8e77 100644 --- a/api/common/data_handlers/compounds/download.py +++ b/api/common/data_handlers/compounds/download.py @@ -64,8 +64,8 @@ def download_all(space_id: str, parcellation_id: str, region_id: str=None, featu try: path_to_feature_export = Path(SIIBRA_API_SHARED_DIR, f"export-{feature_id}.zip") if not path_to_feature_export.exists(): - feature = siibra.features.Feature.get_instance_by_id(feature_id) - feature.export(path_to_feature_export) + feature = siibra.features.Feature._get_instance_by_id(feature_id) + feature.to_zip(path_to_feature_export) zipfile.write(path_to_feature_export, f"export-{feature_id}.zip") except Exception as e: zipfile.writestr(f"{feature_id}.error.txt", f"Feature exporting failed: {str(e)}") diff --git a/api/common/data_handlers/features/types.py b/api/common/data_handlers/features/types.py index bdac586..ad9e13b 100644 --- a/api/common/data_handlers/features/types.py +++ b/api/common/data_handlers/features/types.py @@ -110,15 +110,9 @@ def get_single_feature_download_zip_path(feature_id: str, **kwargs): except Exception as e: general_logger.error(f"Error finding single feature {feature_id=}, {str(e)}") raise NotFound from e - try: - feat.export(str(full_filename)) - return str(full_filename) - except Exception as e: - general_logger.error(f"Error export single feature {feature_id=}, {str(e)}") - error_filename = full_filename.with_suffix(".error.zip") - with ZipFile(error_filename, "w") as zf: - zf.writestr("error.txt", f"Error exporting file for feature_id: {feature_id}: {str(e)}") - return str(error_filename) + + feat.to_zip(str(full_filename)) + return str(full_filename) def extract_concept(*, space_id: str=None, parcellation_id: str=None, region_id: str=None, bbox: str=None, **kwargs): diff --git a/requirements/siibra.txt b/requirements/siibra.txt index 8adff86..c1d3fd4 100644 --- a/requirements/siibra.txt +++ b/requirements/siibra.txt @@ -1,3 +1,3 @@ # siibra==1.0a5 -git+https://github.com/FZJ-INM1-BDA/siibra-python.git@50f038f +git+https://github.com/FZJ-INM1-BDA/siibra-python.git@6135e6e8b4158676a9c80e64510495a5900f3912 plotly