Merge pull request #146 from FZJ-INM1-BDA/feat_compoundFt

feat: compound feat

.github/workflows/deploy-helm.yml

@@ -18,7 +18,7 @@ jobs:
     - name: 'Deploy'
       run: |
         kubecfg_path=${{ runner.temp }}/.kube_config
-        version=$(echo api/VERSION)
+        version=$(cat api/VERSION)
         
         echo "${{ secrets.KUBECONFIG }}" > $kubecfg_path
         helm --kubeconfig=$kubecfg_path status ${{ inputs.DEPLOYMENT_NAME }}

.github/workflows/github-actions.yml

@@ -71,6 +71,6 @@ jobs:
       - name: Install test dependecies
         run: pip install pytest pytest-asyncio httpx mock coverage pytest-cov
       - name: Run tests
-        run: pytest
+        run: pytest test
       - name: Run e2e
         run: pytest e2e_test

.helm/README.md

@@ -1,6 +1,6 @@
 # Deployment
 
-This document is intended for a documentation on how siibra-explorer can be deployed to a [kubernetes (k8s)](https://kubernetes.io/) cluster via [helm](https://helm.sh/).
+This document is intended for a documentation on how siibra-api can be deployed to a [kubernetes (k8s)](https://kubernetes.io/) cluster via [helm](https://helm.sh/).
 
 ## Active deployments
 

.helm/siibra-api/templates/deployment-worker.yaml

@@ -16,7 +16,6 @@ spec:
     matchLabels:
       role: worker
       queuename: {{ . }}
-      sapiVersion: {{ $.Values.sapiVersion }}
       sapiFlavor: {{ $.Values.sapiFlavor }}
       {{- include "siibra-api.selectorLabels" $ | nindent 6 }}
   template:
@@ -28,7 +27,6 @@ spec:
       labels:
         role: worker
         queuename: {{ . }}
-        sapiVersion: {{ $.Values.sapiVersion }}
         sapiFlavor: {{ $.Values.sapiFlavor }}
         {{- include "siibra-api.labels" $ | nindent 8 }}
 	{{- with $.Values.podLabels }}

.helm/siibra-api/templates/hpa.yaml

@@ -1,32 +1,37 @@
 {{- if .Values.autoscaling.enabled }}
+{{- range .Values.sapiWorkerQueues }}
+---
 apiVersion: autoscaling/v2
 kind: HorizontalPodAutoscaler
 metadata:
-  name: {{ include "siibra-api.fullname" . }}
+  name: {{ include "siibra-api.fullname" $ }}-worker-hpa-{{ . }}
   labels:
-    {{- include "siibra-api.labels" . | nindent 4 }}
+    queuename: {{ . }}
+    {{- include "siibra-api.labels" $ | nindent 4 }}
 spec:
   scaleTargetRef:
     apiVersion: apps/v1
     kind: Deployment
-    name: {{ include "siibra-api.fullname" . }}
-  minReplicas: {{ .Values.autoscaling.minReplicas }}
-  maxReplicas: {{ .Values.autoscaling.maxReplicas }}
+    name: {{ include "siibra-api.fullname" $ }}-worker-{{ . }}
+  minReplicas: {{ $.Values.autoscaling.minReplicas }}
+  maxReplicas: {{ $.Values.autoscaling.maxReplicas }}
   metrics:
-    {{- if .Values.autoscaling.targetCPUUtilizationPercentage }}
+    {{- if $.Values.autoscaling.targetCPUUtilizationPercentage }}
     - type: Resource
       resource:
         name: cpu
         target:
           type: Utilization
-          averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }}
+          averageUtilization: {{ $.Values.autoscaling.targetCPUUtilizationPercentage }}
     {{- end }}
-    {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }}
+    {{- if $.Values.autoscaling.targetMemoryUtilizationPercentage }}
     - type: Resource
       resource:
         name: memory
         target:
           type: Utilization
-          averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }}
+          averageUtilization: {{ $.Values.autoscaling.targetCPUUtilizationPercentage }}
     {{- end }}
+
+{{- end }}
 {{- end }}

.helm/siibra-api/values.yaml

@@ -92,7 +92,7 @@ resourcesWorkerPod:
 autoscaling:
   enabled: true
   minReplicas: 1
-  maxReplicas: 100
+  maxReplicas: 10
   targetCPUUtilizationPercentage: 80
   # targetMemoryUtilizationPercentage: 80
 

api/VERSION

@@ -1 +1 @@
-0.3.16
+0.3.17

api/common/__init__.py

@@ -1,5 +1,5 @@
 from .decorators import data_decorator, router_decorator, async_router_decorator, name_to_fns_map
 from .siibra_api_typing import ROLE_TYPE
-from .logger import logger, access_logger
+from .logger import logger as general_logger, access_logger
 from .exceptions import *
 from .storage import get_filename

api/common/data_handlers/__init__.py

@@ -3,7 +3,7 @@
 from . import volumes
 from . import compounds
 
-from ...siibra_api_config import ROLE
+from api.siibra_api_config import ROLE
 
 if ROLE == "all" or ROLE == "worker":
     import siibra

api/common/data_handlers/features/types.py

@@ -1,9 +1,10 @@
-from api.common import data_decorator, InsufficientParameters, NotFound, AmbiguousParameters, logger
+from api.common import data_decorator, InsufficientParameters, NotFound, AmbiguousParameters, general_logger
 from api.siibra_api_config import ROLE, SIIBRA_API_SHARED_DIR
 from typing import List, Type, Any, Dict
 from hashlib import md5
 from pathlib import Path
 from zipfile import ZipFile
+import json
 
 @data_decorator(ROLE)
 def all_feature_types() -> List[Dict[str, str]]:
@@ -32,15 +33,15 @@ def get_hierarchy_type(Cls: Type[Any]) -> str:
         {
             'name': get_hierarchy_type(Cls),
             'category': Cls.category,
-        } for Cls in Feature.SUBCLASSES
+        } for Cls in Feature._SUBCLASSES
     ]
 
 @data_decorator(ROLE)
 def get_single_feature_from_id(feature_id: str, **kwargs):
     import siibra
     from api.serialization.util import instance_to_model
     try:
-        feature = siibra.features.Feature.get_instance_by_id(feature_id)
+        feature = siibra.features.Feature._get_instance_by_id(feature_id)
     except Exception as e:
         raise NotFound(str(e))
     else:
@@ -53,18 +54,44 @@ def get_single_feature_plot_from_id(feature_id: str, template="plotly", **kwargs
     import json
 
     try:
-        feature = siibra.features.Feature.get_instance_by_id(feature_id)
+        feature = siibra.features.Feature._get_instance_by_id(feature_id)
     except Exception as e:
         raise NotFound from e
 
     try:
-        plotly_fig = feature.plot(backend="plotly", template=template)
+        plotly_fig = feature.plot(backend="plotly", template=template, **kwargs)
         json_str = to_json(plotly_fig)
         return json.loads(json_str)
 
     except NotImplementedError:
         raise NotFound(f"feature with id {feature_id} is found, but the plot function has not been implemented")
 
+@data_decorator(ROLE)
+def get_single_feature_intents_from_id(feature_id: str, **kwargs):
+    import siibra
+    from api.serialization.util.siibra import RegionalConnectivity
+    from api.serialization.util import instance_to_model
+    from api.models.intents.colorization import RegionMapping, ColorizationIntent
+
+    try:
+        feature = siibra.features.Feature._get_instance_by_id(feature_id)
+    except Exception as e:
+        raise NotFound from e
+
+    if not isinstance(feature, RegionalConnectivity):
+        return []
+
+    region_mappings = [RegionMapping(
+        region=instance_to_model(reg),
+        rgb=rgb,
+    ) for reg, rgb in feature.get_profile_colorscale(**kwargs)]
+
+    return [
+        ColorizationIntent(
+            region_mappings=region_mappings
+        ).dict()
+    ]
+
 @data_decorator(ROLE)
 def get_single_feature_download_zip_path(feature_id: str, **kwargs):
     assert feature_id, f"feature_id must be defined"
@@ -79,36 +106,36 @@ def get_single_feature_download_zip_path(feature_id: str, **kwargs):
         return str(full_filename)
     import siibra
     try:
-        feat = siibra.features.Feature.get_instance_by_id(feature_id)
+        feat = siibra.features.Feature._get_instance_by_id(feature_id)
     except Exception as e:
-        logger.error(f"Error finding single feature {feature_id=}, {str(e)}")
+        general_logger.error(f"Error finding single feature {feature_id=}, {str(e)}")
         raise NotFound from e
     try:
         feat.export(str(full_filename))
         return str(full_filename)
     except Exception as e:
-        logger.error(f"Error export single feature {feature_id=}, {str(e)}")
+        general_logger.error(f"Error export single feature {feature_id=}, {str(e)}")
         error_filename = full_filename.with_suffix(".error.zip")
         with ZipFile(error_filename, "w") as zf:
             zf.writestr("error.txt", f"Error exporting file for feature_id: {feature_id}: {str(e)}")
         return str(error_filename)
 
 
-def extract_concept(*, space_id: str=None, parcellation_id: str=None, region_id: str=None, **kwargs):
-    import siibra
+def extract_concept(*, space_id: str=None, parcellation_id: str=None, region_id: str=None, bbox: str=None, **kwargs):
+    from api.serialization.util.siibra import Region, Parcellation, BoundingBox, siibra
     if region_id is not None:
         if parcellation_id is None:
             raise InsufficientParameters(f"if region_id is defined, parcellation_id must also be defined!")
-        region: siibra.core.parcellation.region.Region = siibra.get_region(parcellation_id, region_id)
+        region: Region = siibra.get_region(parcellation_id, region_id)
         return region
 
     if parcellation_id:
-        parcellation: siibra._parcellation.Parcellation = siibra.parcellations[parcellation_id]
+        parcellation: Parcellation = siibra.parcellations[parcellation_id]
         return parcellation
 
     if space_id:
-        space: siibra._space.Space = siibra.spaces[space_id]
-        return space
+        assert bbox, f"If space_id is defined, bbox must be defined!"
+        return BoundingBox(*json.loads(bbox), space_id)
     raise InsufficientParameters(f"concept cannot be extracted")
 
 @data_decorator(ROLE)
@@ -126,35 +153,22 @@ def get_all_all_features(*, space_id: str=None, parcellation_id: str=None, regio
             )
         except Exception as e:
             err_str = str(e).replace('\n', ' ')
-            logger.warning(f"feature failed to be serialized. Params: space_id={space_id}, parcellation_id={parcellation_id}, region_id={region_id}. feature id: {f.id}. error: {err_str}")
+            general_logger.warning(f"feature failed to be serialized. Params: space_id={space_id}, parcellation_id={parcellation_id}, region_id={region_id}. feature id: {f.id}. error: {err_str}")
     return re_features
 
 
 def _get_all_features(*, space_id: str, parcellation_id: str, region_id: str, type: str, bbox: str=None, **kwargs):
     import siibra
-    from siibra.features.image.image import Image
-    import json
     if type is None:
         raise InsufficientParameters(f"type is a required kwarg")
 
     *_, type = type.split(".")
 
-    concept = extract_concept(space_id=space_id, parcellation_id=parcellation_id, region_id=region_id)
+    concept = extract_concept(space_id=space_id, parcellation_id=parcellation_id, region_id=region_id, bbox=bbox)
 
     if concept is None:
         raise InsufficientParameters(f"at least one of space_id, parcellation_id and/or region_id must be defined.")
-
-    features = siibra.features.get(concept, type, **kwargs)
-
-    if bbox:
-        assert isinstance(concept, siibra.core.space.Space)
-        bounding_box = concept.get_bounding_box(*json.loads(bbox))
-        features = [f
-                for f in features
-                if isinstance(f, Image) and
-                f.anchor.location.intersects(bounding_box)]
-
-    return features
+    return siibra.features.get(concept, type, **kwargs)
 
 @data_decorator(ROLE)
 def all_features(*, space_id: str, parcellation_id: str, region_id: str, type: str, **kwargs):
@@ -170,7 +184,7 @@ def all_features(*, space_id: str, parcellation_id: str, region_id: str, type: s
             )
         except Exception as e:
             err_str = str(e).replace('\n', ' ')
-            logger.warning(f"feature failed to be serialized. Params: space_id={space_id}, parcellation_id={parcellation_id}, region_id={region_id}. feature id: {f.id}, error: {err_str}")
+            general_logger.warning(f"feature failed to be serialized. Params: space_id={space_id}, parcellation_id={parcellation_id}, region_id={region_id}. feature id: {f.id}, error: {err_str}")
     return re_features
 
 

api/common/decorators.py

@@ -1,13 +1,14 @@
-from .siibra_api_typing import ROLE_TYPE
-from api.common import logger
 from functools import wraps, partial
 import inspect
 import asyncio
 import time
-from api.common.exceptions import (
+from typing import Dict, Callable, Tuple
+
+from .siibra_api_typing import ROLE_TYPE
+from .logger import logger as general_logger
+from .exceptions import (
     FaultyRoleException,
 )
-from typing import Dict, Callable, Tuple
 
 name_to_fns_map: Dict[str, Tuple[Callable, Callable]] = {}
 
@@ -32,14 +33,16 @@ def outer_wrapper(fn):
                 from api.worker import app
 
                 def celery_task_wrapper(self, *args, **kwargs):
+                    if role == "worker":
+                        general_logger.info(f"Task Received: {fn.__name__=}, {args=}, {kwargs=}")
                     return fn(*args, **kwargs)
 
                 return app.task(bind=True)(
                     wraps(fn)(celery_task_wrapper)
                 )
             except ImportError as e:
                 errmsg = f"For worker role, celery must be installed as a dep"
-                logger.critical(errmsg)
+                general_logger.critical(errmsg)
                 raise ImportError(errmsg) from e
         raise FaultyRoleException(f"role must be 'all', 'server' or 'worker', but it is {role}")
     return outer_wrapper

api/common/logger.py

@@ -58,6 +58,9 @@
     import socket
     filename = log_dir + f"{socket.gethostname()}.general.log"
     warn_fh = TimedRotatingFileHandler(filename, when="d", encoding="utf-8")
-    warn_fh.setLevel(logging.INFO)
-    warn_fh.setFormatter(formatter)
-    logger.addHandler(warn_fh)
+else:
+    warn_fh = logging.StreamHandler()
+
+warn_fh.setLevel(logging.INFO)
+warn_fh.setFormatter(formatter)
+logger.addHandler(warn_fh)

api/models/_commons.py

@@ -108,6 +108,11 @@ class DataFrameModel(ConfigBaseModel):
         ]]
     ]
 
+class TimedeltaModel(ConfigBaseModel):
+    """TimedeltaModel"""
+    total_seconds: float
+
+
 class TaskIdResp(BaseModel):
     """TaskIdResp"""
     task_id: str

api/models/features/_basetypes/feature.py

@@ -1,7 +1,8 @@
 from api.models._commons import ConfigBaseModel
 from api.models._retrieval.datasets import EbrainsDatasetModel
 from api.models.features.anchor import SiibraAnchorModel
-from typing import List, Optional
+from api.models.locations.point import CoordinatePointModel
+from typing import List, Optional, Union
 from abc import ABC
 
 class _FeatureModel(ConfigBaseModel, ABC, type="feature"):
@@ -20,3 +21,11 @@ class _FeatureModel(ConfigBaseModel, ABC, type="feature"):
 class FeatureModel(_FeatureModel):
     """FeatureModel"""
     pass
+
+class SubfeatureModel(ConfigBaseModel):
+    id: str
+    index: Union[str, CoordinatePointModel]
+    name: str
+
+class CompoundFeatureModel(_FeatureModel, type="compoundfeature"):
+    indices: List[SubfeatureModel]

api/models/features/_basetypes/regional_connectivity.py

@@ -5,5 +5,6 @@
 class SiibraRegionalConnectivityModel(_FeatureModel, type="regional_connectivity"):
     """SiibraRegionalConnectivityModel"""
     cohort: str
-    subjects: List[str]
-    matrices: Optional[Dict[str, DataFrameModel]]
+    subject: str
+    feature: Optional[str]
+    matrix: Optional[DataFrameModel]

api/models/intents/base.py

@@ -0,0 +1,5 @@
+from abc import ABC
+from api.models._commons import ConfigBaseModel
+
+class _BaseIntent(ConfigBaseModel, ABC, type="intent"):
+    pass