From e7fd63dbd8e4d304716995a33d62c09eb4b9fa61 Mon Sep 17 00:00:00 2001 From: Xiao Gui Date: Tue, 16 Jan 2024 16:30:16 +0100 Subject: [PATCH 1/3] feat: fixed doc --- docs/develop.example.adding_serialization.md | 86 +++++++++++++++----- 1 file changed, 67 insertions(+), 19 deletions(-) diff --git a/docs/develop.example.adding_serialization.md b/docs/develop.example.adding_serialization.md index 645dff2..20e9125 100644 --- a/docs/develop.example.adding_serialization.md +++ b/docs/develop.example.adding_serialization.md @@ -4,12 +4,12 @@ siibra v0.5 introduced compound features. New serialization strategies and model `api.serialization.util.siibra` Serves as the one and single entrypoint to siibra package. To access the new class `CompoundFeature`, import it here. -```python +```diff # in api.serialization.util.siibra # ... trimmed for brevity -from siibra.feature.feature import CompoundFeature ++ from siibra.feature.feature import CompoundFeature ``` ## Adding serialization model @@ -20,38 +20,86 @@ Since `CompoundFeature` subclasses `Feature`, it is quite natural to have the mo Here, we also added the property `subfeature_keys`, which is the additional property of `CompoundFeatureModel` compared to `FeatureModel` see [more detail](../api.models/#api.models._commons.ConfigBaseModel.__init_subclass__) -```python +```diff # in api.models.features._basetypes.feature # ... trimmed for brevity -class CompoundFeatureModel(_FeatureModel, type="compound_feature"): - """CompoundFeatureModel""" - subfeature_keys: List[str] +from api.models.features.anchor import SiibraAnchorModel +- from typing import List, Optional ++ from api.models.locations.point import CoordinatePointModel ++ from typing import List, Optional, Union +from abc import ABC +# ... trimmed for brevity + + ++ class SubfeatureModel(ConfigBaseModel): ++ id: str ++ index: Union[str, CoordinatePointModel] ++ name: str ++ ++ class CompoundFeatureModel(_FeatureModel, type="compoundfeature"): ++ indices: List[SubfeatureModel] ``` ## Adding serialization strategy We then add the logic of serialization. -```python +```diff # in api.serialization.features.feature -from api.serialization.util.siibra import CompoundFeature -from api.models.features._basetypes.feature import CompoundFeatureModel +# ... trimmed for brevity + +- from api.serialization.util.siibra import Feature ++ from api.serialization.util.siibra import Feature, CompoundFeature +from api.serialization.util import serialize, instance_to_model +- from api.models.features._basetypes.feature import FeatureModel ++ from api.models.features._basetypes.feature import FeatureModel, CompoundFeatureModel, SubfeatureModel ++ ++ @serialize(CompoundFeature, pass_super_model=True) ++ def cmpdfeature_to_model(cf: CompoundFeature, detail: bool=False, super_model_dict={}, **kwargs): ++ return CompoundFeatureModel( ++ **super_model_dict, ++ indices=[SubfeatureModel(id=ft.id, index=instance_to_model(idx), name=ft.name) for (idx, ft) in zip(cf.indices, cf.elements)] ++ ) ++ + + +``` + +## Updating response models + +Whilst the update to serve compound feature does not require the addition of additional REST endpoints, several existing endpoints needs to be updated, as their response would contain compound feature model *in addition* to the previous models. + +```diff +# api.server.features.__init__ + +# ... trimmed for brevity + ++ from api.models.features._basetypes.feature import ( ++ CompoundFeatureModel ++ ) + +# ... trimmed for brevity + +# Regional Connectivity +- RegionalConnectivityModels = SiibraRegionalConnectivityModel ++ RegionalConnectivityModels = Union[SiibraRegionalConnectivityModel, CompoundFeatureModel] + +# ... trimmed for brevity + +# Cortical Profiles +- CortialProfileModels = SiibraCorticalProfileModel ++ CortialProfileModels = Union[SiibraCorticalProfileModel, CompoundFeatureModel] + +# ... trimmed for brevity -from api.serialization.util import ( - serialize, - instance_to_model -) +# Tabular +- TabularModels = Union[SiibraCorticalProfileModel, SiibraReceptorDensityFp, SiibraTabularModel] ++ TabularModels = Union[CompoundFeatureModel, SiibraCorticalProfileModel, SiibraReceptorDensityFp, SiibraTabularModel] -@serialize(CompoundFeature, pass_super_model=True) -def serialize_cf(cf: CompoundFeature, detail=False, super_model_dict={}, **kwargs) -> CompoundFeatureModel: - return CompoundFeatureModel( - **super_model_dict, - subfeature_keys=cf.subfeature_keys - ) ``` From ee5c542845684358f8619281853893e46993b255 Mon Sep 17 00:00:00 2001 From: Xiao Gui Date: Tue, 16 Jan 2024 16:35:02 +0100 Subject: [PATCH 2/3] feat: add /gene endpoint --- .helm/siibra-api/values.yaml | 2 +- api/common/data_handlers/__init__.py | 1 + .../data_handlers/vocabularies/__init__.py | 1 + api/common/data_handlers/vocabularies/gene.py | 24 +++ api/models/_commons.py | 2 +- api/models/vocabularies/__init__.py | 0 api/models/vocabularies/base.py | 7 + api/models/vocabularies/genes.py | 5 + api/serialization/__init__.py | 1 + api/serialization/util/siibra.py | 2 + api/server/api.py | 2 + api/server/volcabularies/__init__.py | 25 +++ api/siibra_api_config.py | 1 + bla.py | 24 +++ ...elop.example.adding_vocabulary_endpoint.md | 196 ++++++++++++++++++ e2e_test/vocabularies/__init__.py | 0 e2e_test/vocabularies/test_genes.py | 22 ++ mkdocs.yml | 1 + 18 files changed, 314 insertions(+), 2 deletions(-) create mode 100644 api/common/data_handlers/vocabularies/__init__.py create mode 100644 api/common/data_handlers/vocabularies/gene.py create mode 100644 api/models/vocabularies/__init__.py create mode 100644 api/models/vocabularies/base.py create mode 100644 api/models/vocabularies/genes.py create mode 100644 api/server/volcabularies/__init__.py create mode 100644 bla.py create mode 100644 docs/develop.example.adding_vocabulary_endpoint.md create mode 100644 e2e_test/vocabularies/__init__.py create mode 100644 e2e_test/vocabularies/test_genes.py diff --git a/.helm/siibra-api/values.yaml b/.helm/siibra-api/values.yaml index af496ed..b02a92a 100644 --- a/.helm/siibra-api/values.yaml +++ b/.helm/siibra-api/values.yaml @@ -5,7 +5,7 @@ replicaCount: 1 sapiVersion: "0.3.15" # "latest" or "0.3.15" -sapiWorkerQueues: ["core", "features", "volumes", "compounds"] +sapiWorkerQueues: ["core", "features", "volumes", "compounds", "vocabularies"] sapiFlavor: "prod" # could be prod, rc, latest, etc image: diff --git a/api/common/data_handlers/__init__.py b/api/common/data_handlers/__init__.py index 8a0c0c5..3ffdcb0 100644 --- a/api/common/data_handlers/__init__.py +++ b/api/common/data_handlers/__init__.py @@ -2,6 +2,7 @@ from . import features from . import volumes from . import compounds +from . import vocabularies from api.siibra_api_config import ROLE diff --git a/api/common/data_handlers/vocabularies/__init__.py b/api/common/data_handlers/vocabularies/__init__.py new file mode 100644 index 0000000..3914ca0 --- /dev/null +++ b/api/common/data_handlers/vocabularies/__init__.py @@ -0,0 +1 @@ +from . import gene \ No newline at end of file diff --git a/api/common/data_handlers/vocabularies/gene.py b/api/common/data_handlers/vocabularies/gene.py new file mode 100644 index 0000000..1b986ae --- /dev/null +++ b/api/common/data_handlers/vocabularies/gene.py @@ -0,0 +1,24 @@ +from api.common import data_decorator +from api.siibra_api_config import ROLE +from api.models.vocabularies.genes import GeneModel + +@data_decorator(ROLE) +def get_genes(find:str=None): + """Get all genes + + Args: + string to find in vocabularies + + Returns: + List of the genes.""" + from api.serialization.util.siibra import GENE_NAMES + + if find == None: + return_list = [v for v in GENE_NAMES] + else: + return_list = GENE_NAMES.find(find) + + return [GeneModel( + symbol=v.get("symbol"), + description=v.get("description") + ).dict() for v in return_list] diff --git a/api/models/_commons.py b/api/models/_commons.py index 358651e..0752f90 100644 --- a/api/models/_commons.py +++ b/api/models/_commons.py @@ -3,7 +3,7 @@ from pydantic import Field, BaseModel from abc import ABC -SIIBRA_PYTHON_VERSION = "0.4" +SIIBRA_PYTHON_VERSION = "1.0" ignore_cls=( "BrainAtlasVersionModel", diff --git a/api/models/vocabularies/__init__.py b/api/models/vocabularies/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/api/models/vocabularies/base.py b/api/models/vocabularies/base.py new file mode 100644 index 0000000..fef0c2f --- /dev/null +++ b/api/models/vocabularies/base.py @@ -0,0 +1,7 @@ +from api.models._commons import ( + ConfigBaseModel, +) +from abc import ABC + +class _VocabBaseModel(ConfigBaseModel, ABC, type="vocabulary"): + ... diff --git a/api/models/vocabularies/genes.py b/api/models/vocabularies/genes.py new file mode 100644 index 0000000..01082ad --- /dev/null +++ b/api/models/vocabularies/genes.py @@ -0,0 +1,5 @@ +from .base import _VocabBaseModel + +class GeneModel(_VocabBaseModel, type="gene"): + symbol: str + description: str diff --git a/api/serialization/__init__.py b/api/serialization/__init__.py index 7aa5fbd..13520a3 100644 --- a/api/serialization/__init__.py +++ b/api/serialization/__init__.py @@ -10,5 +10,6 @@ _retrieval, _common, locations, + # vocabularies, # For now, there are no serialization to be done on vocabolaries ) diff --git a/api/serialization/util/siibra.py b/api/serialization/util/siibra.py index d6e6139..50b001f 100644 --- a/api/serialization/util/siibra.py +++ b/api/serialization/util/siibra.py @@ -45,4 +45,6 @@ from siibra import parcellations, spaces, atlases +from siibra.vocabularies import GENE_NAMES + import siibra diff --git a/api/server/api.py b/api/server/api.py index a542329..165ebda 100644 --- a/api/server/api.py +++ b/api/server/api.py @@ -19,6 +19,7 @@ from .volumes import prefixed_routers as volume_prefixed_routers from .compounds import prefixed_routers as compound_prefixed_routers from .features import router as feature_router +from .volcabularies import router as vocabolaries_router from .metrics import prom_metrics_resp, on_startup as metrics_on_startup, on_terminate as metrics_on_terminate from .code_snippet import get_sourcecode @@ -37,6 +38,7 @@ siibra_api.include_router(prefix_router.router, prefix=prefix_router.prefix) siibra_api.include_router(feature_router, prefix="/feature") +siibra_api.include_router(vocabolaries_router, prefix="/vocabularies") add_pagination(siibra_api) diff --git a/api/server/volcabularies/__init__.py b/api/server/volcabularies/__init__.py new file mode 100644 index 0000000..e556513 --- /dev/null +++ b/api/server/volcabularies/__init__.py @@ -0,0 +1,25 @@ +from fastapi import APIRouter, HTTPException +from fastapi_pagination import paginate, Page +from fastapi_versioning import version + +from api.server.util import SapiCustomRoute +from api.server import FASTAPI_VERSION +from api.siibra_api_config import ROLE +from api.common import router_decorator +from api.models.vocabularies.genes import GeneModel +from api.common.data_handlers.vocabularies.gene import get_genes + +TAGS= ["vocabularies"] +"""HTTP vocabularies tags""" + +router = APIRouter(route_class=SapiCustomRoute, tags=TAGS) +"""HTTP vocabularies router""" + +@router.get("/genes", response_model=Page[GeneModel]) +@version(*FASTAPI_VERSION) +@router_decorator(ROLE, func=get_genes) +def get_genes(find:str=None, func=None): + """HTTP get (filtered) genes""" + if func is None: + raise HTTPException(500, "func: None passed") + return paginate(func(find=find)) diff --git a/api/siibra_api_config.py b/api/siibra_api_config.py index 6840e90..aaac863 100644 --- a/api/siibra_api_config.py +++ b/api/siibra_api_config.py @@ -69,6 +69,7 @@ def get_config_dir_short_hash(path_to_config: str): "features", "volumes", "compounds", + "vocabularies", ] class CELERY_CONFIG: diff --git a/bla.py b/bla.py new file mode 100644 index 0000000..9766f13 --- /dev/null +++ b/bla.py @@ -0,0 +1,24 @@ +feature_types = [ + ((), "FunctionalConnectivity",) + ((), "ReceptorDensityFingerprint",) + ((), "ReceptorDensityProfile",) + ((), "MRIVolumeOfInterest",) + ((), "BlockfaceVolumeOfInterest",) + ((), "RegionalBOLD",) + ((), "PLIVolumeOfInterest",) + ((), "DTIVolumeOfInterest",) + ((), "TracingConnectivity",) + ((), "StreamlineLengths",) + ((), "StreamlineCounts",) + ((), "AnatomoFunctionalConnectivity",) +] + +import os +from subprocess import run +MONITOR_FIRSTLVL_DIR = os.getenv("MONITOR_FIRSTLVL_DIR") +dirs = os.listdir(MONITOR_FIRSTLVL_DIR) +for dir in dirs: + result = run(["du", "-s", f"{MONITOR_FIRSTLVL_DIR}/{dir}"], capture_output=True, text=True) + size_b, *_ = result.stdout.split("\t") + print(size_b) + print("dir", size_b, int(size_b)) \ No newline at end of file diff --git a/docs/develop.example.adding_vocabulary_endpoint.md b/docs/develop.example.adding_vocabulary_endpoint.md new file mode 100644 index 0000000..1cc54b8 --- /dev/null +++ b/docs/develop.example.adding_vocabulary_endpoint.md @@ -0,0 +1,196 @@ +This document will record the process to reimplement the `/genes` endpoint, which was briefly removed during the refactor of `/v2_0` to `/v3_0`. This should serve as an informative document, the process adds a top level module, which needs to be included for the server/worker architecture to work properly. + +## Adding the new import + +`api.serialization.util.siibra` Serves as the one and single entrypoint to siibra package. To access the new variable `GENE_NAMES`, import it here. + +```diff +# in api.serialization.util.siibra + +# ... trimmed for brevity + ++ from siibra.vocabularies import GENE_NAMES +``` + +## Adding serialization model + +Next, we define the shape of serialized instance of gene. As this is the first model to be added in vocabulary module, it is useful to add the base class for vocabulary. + +```diff +# api.models.vocabularies.__init__ +``` + +```diff +# api.models.vocabularies.base ++ from api.models._commons import ( ++ ConfigBaseModel, ++ ) ++ from abc import ABC ++ ++ class _VocabBaseModel(ConfigBaseModel, ABC, type="vocabulary"): ++ ... + +``` + +```diff +# api.models.vocabularies.gene ++ from .base import _VocabBaseModel ++ ++ class GeneModel(_VocabBaseModel, type="gene"): ++ symbol: str ++ description: str + +``` + +The advantage of constructing the inheritence model `ConfigBaseModel` -> `_VocabBaseModel` -> `GeneModel` is so that initialized model will automatically have the `@type` attribute set to `siibra-1.0/vocabulary/gene`. + + +## Adding serialization strategy + +For serializing `GENE_NAMES`, this step can be skipped, as the variable is already a dictionary. + +This is, however, a double edged sword. An unshaped dictionary can change the key/shape without warning. More so than ever, adding tests would be crucial to ensure future versions of siibra does not break siibra-api. + +## Adding query logic + +Here, we define, given arguments as string primitives, how to retrieve siibra objects. In our case, given optional string argument `find`, how to get a list of `GeneModel`. + +```diff +# api.common.data_handlers.vocabularies.__init__ ++ from . import gene +``` + +```diff +# api.common.data_handlers.vocabularies.gene ++ from api.common import data_decorator ++ from api.siibra_api_config import ROLE ++ from api.models.vocabularies.genes import GeneModel ++ ++ @data_decorator(ROLE) ++ def get_genes(find:str=None): ++ """Get all genes ++ ++ Args: ++ string to find in vocabularies ++ ++ Returns: ++ List of the genes.""" ++ from api.serialization.util.siibra import GENE_NAMES ++ ++ if find == None: ++ return_list = [v for v in GENE_NAMES] ++ else: ++ return_list = GENE_NAMES.find(find) ++ ++ return [GeneModel( ++ symbol=v.get("symbol"), ++ description=v.get("description") ++ ).dict() for v in return_list] + +``` + +Important to note here is, since this is a new module, it *must* be imported into the parent module: + +```diff +# api.common.data_handlers.__init__ + +from . import core +from . import features +from . import volumes +from . import compounds ++ from . import vocabularies + +from api.siibra_api_config import ROLE + +if ROLE == "all" or ROLE == "worker": + import siibra + siibra.warm_cache() + +``` + +## Adding the route + +Finally, add the route to FastAPI + +```diff +# api.server.vocabularies.__init__ ++ from fastapi import APIRouter, HTTPException ++ from fastapi_pagination import paginate, Page ++ from fastapi_versioning import version ++ ++ from api.server.util import SapiCustomRoute ++ from api.server import FASTAPI_VERSION ++ from api.siibra_api_config import ROLE ++ from api.common import router_decorator ++ from api.models.vocabularies.genes import GeneModel ++ from api.common.data_handlers.vocabularies.gene import get_genes + ++ TAGS= ["vocabularies"] ++ """HTTP vocabularies tags""" ++ ++ router = APIRouter(route_class=SapiCustomRoute, tags=TAGS) ++ """HTTP vocabularies router""" + ++ @router.get("/genes", response_model=Page[GeneModel]) ++ @version(*FASTAPI_VERSION) ++ @router_decorator(ROLE, func=get_genes) ++ def get_genes(find:str=None, func=None): ++ """HTTP get (filtered) genes""" ++ if func is None: ++ raise HTTPException(500, "func: None passed") ++ return paginate(func(find=find)) + +``` + +```diff +# api.server.api + +# ... truncated for brevity + +from .features import router as feature_router ++ from .volcabularies import router as vocabolaries_router +from .metrics import prom_metrics_resp, on_startup as metrics_on_startup, on_terminate as metrics_on_terminate + +# ... truncated for brevity + +siibra_api.include_router(feature_router, prefix="/feature") ++ siibra_api.include_router(vocabolaries_router, prefix="/vocabularies") + +add_pagination(siibra_api) + +# ... truncated for brevity +``` + +## Configure workers + +Since a new top level module was introduced, a new queue is automatically created. New workers must be configured to take jobs from this queue, or HTTP requests related to this queue will hang forever. + +```diff +# api.siibra_api_config + +# ... truncated for brevity + +_queues = [ + "core", + "features", + "volumes", + "compounds", ++ "vocabularies", +] + +# ... truncated for brevity + +``` + +```diff +# .helm/siibra-api.values.yaml + +# ... truncated for brevity + +sapiVersion: "0.3.15" # "latest" or "0.3.15" +- sapiWorkerQueues: ["core", "features", "volumes", "compounds"] ++ sapiWorkerQueues: ["core", "features", "volumes", "compounds", "vocabularies"] +sapiFlavor: "prod" # could be prod, rc, latest, etc + +# ... truncated for brevity +``` \ No newline at end of file diff --git a/e2e_test/vocabularies/__init__.py b/e2e_test/vocabularies/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/e2e_test/vocabularies/test_genes.py b/e2e_test/vocabularies/test_genes.py new file mode 100644 index 0000000..07df7aa --- /dev/null +++ b/e2e_test/vocabularies/test_genes.py @@ -0,0 +1,22 @@ +import pytest +from fastapi.testclient import TestClient +from api.server import api + +client = TestClient(api) + +@pytest.mark.parametrize('find, expect_no', [ + (None, 50), + ("MA", 50), + ("MAO", 2), + ("MAOC", 0), +]) +def test_genes(find, expect_no): + response = client.get(f"/v3_0/vocabularies/genes", params={ + "find": find + }) + assert response.status_code == 200 + resp_json = response.json() + items = resp_json.get("items", []) + + assert len(items) == expect_no + \ No newline at end of file diff --git a/mkdocs.yml b/mkdocs.yml index b6dfde3..8b4d13f 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -61,6 +61,7 @@ nav: - 'start developer server': 'develop.md' - 'example: adding CompoundFeature serialization': 'develop.example.adding_serialization.md' - 'example: adding related region endpoint': 'develop.example.adding_related_regions.md' + - 'example: adding /gene endpoint': 'develop.example.adding_vocabulary_endpoint.md' - api references: - api.siibra_api_config: 'api.siibra_api_config.md' - api.server: 'api.server.md' From da22096427ef1975dcf31c604624ca7cd311816f Mon Sep 17 00:00:00 2001 From: Xiao Gui Date: Tue, 16 Jan 2024 16:41:01 +0100 Subject: [PATCH 3/3] feat: added link to commit --- docs/develop.example.adding_serialization.md | 2 ++ docs/develop.example.adding_vocabulary_endpoint.md | 2 ++ 2 files changed, 4 insertions(+) diff --git a/docs/develop.example.adding_serialization.md b/docs/develop.example.adding_serialization.md index 20e9125..0f6fc12 100644 --- a/docs/develop.example.adding_serialization.md +++ b/docs/develop.example.adding_serialization.md @@ -1,5 +1,7 @@ siibra v0.5 introduced compound features. New serialization strategies and models need to be introduced in order to take full advantage of the introduction of compound features. +See the [commit](https://github.com/FZJ-INM1-BDA/siibra-api/commit/6723cfa3e612de66022150b6d5e645caddbc3e7a) + ## Import the new class `api.serialization.util.siibra` Serves as the one and single entrypoint to siibra package. To access the new class `CompoundFeature`, import it here. diff --git a/docs/develop.example.adding_vocabulary_endpoint.md b/docs/develop.example.adding_vocabulary_endpoint.md index 1cc54b8..59749c1 100644 --- a/docs/develop.example.adding_vocabulary_endpoint.md +++ b/docs/develop.example.adding_vocabulary_endpoint.md @@ -1,5 +1,7 @@ This document will record the process to reimplement the `/genes` endpoint, which was briefly removed during the refactor of `/v2_0` to `/v3_0`. This should serve as an informative document, the process adds a top level module, which needs to be included for the server/worker architecture to work properly. +See [commit](https://github.com/FZJ-INM1-BDA/siibra-api/commit/ee5c542845684358f8619281853893e46993b255) + ## Adding the new import `api.serialization.util.siibra` Serves as the one and single entrypoint to siibra package. To access the new variable `GENE_NAMES`, import it here.