diff --git a/.helm/siibra-api/values.yaml b/.helm/siibra-api/values.yaml index af496ed..b02a92a 100644 --- a/.helm/siibra-api/values.yaml +++ b/.helm/siibra-api/values.yaml @@ -5,7 +5,7 @@ replicaCount: 1 sapiVersion: "0.3.15" # "latest" or "0.3.15" -sapiWorkerQueues: ["core", "features", "volumes", "compounds"] +sapiWorkerQueues: ["core", "features", "volumes", "compounds", "vocabularies"] sapiFlavor: "prod" # could be prod, rc, latest, etc image: diff --git a/api/common/data_handlers/__init__.py b/api/common/data_handlers/__init__.py index 8a0c0c5..3ffdcb0 100644 --- a/api/common/data_handlers/__init__.py +++ b/api/common/data_handlers/__init__.py @@ -2,6 +2,7 @@ from . import features from . import volumes from . import compounds +from . import vocabularies from api.siibra_api_config import ROLE diff --git a/api/common/data_handlers/vocabularies/__init__.py b/api/common/data_handlers/vocabularies/__init__.py new file mode 100644 index 0000000..3914ca0 --- /dev/null +++ b/api/common/data_handlers/vocabularies/__init__.py @@ -0,0 +1 @@ +from . import gene \ No newline at end of file diff --git a/api/common/data_handlers/vocabularies/gene.py b/api/common/data_handlers/vocabularies/gene.py new file mode 100644 index 0000000..1b986ae --- /dev/null +++ b/api/common/data_handlers/vocabularies/gene.py @@ -0,0 +1,24 @@ +from api.common import data_decorator +from api.siibra_api_config import ROLE +from api.models.vocabularies.genes import GeneModel + +@data_decorator(ROLE) +def get_genes(find:str=None): + """Get all genes + + Args: + string to find in vocabularies + + Returns: + List of the genes.""" + from api.serialization.util.siibra import GENE_NAMES + + if find == None: + return_list = [v for v in GENE_NAMES] + else: + return_list = GENE_NAMES.find(find) + + return [GeneModel( + symbol=v.get("symbol"), + description=v.get("description") + ).dict() for v in return_list] diff --git a/api/models/_commons.py b/api/models/_commons.py index 358651e..0752f90 100644 --- a/api/models/_commons.py +++ b/api/models/_commons.py @@ -3,7 +3,7 @@ from pydantic import Field, BaseModel from abc import ABC -SIIBRA_PYTHON_VERSION = "0.4" +SIIBRA_PYTHON_VERSION = "1.0" ignore_cls=( "BrainAtlasVersionModel", diff --git a/api/models/vocabularies/__init__.py b/api/models/vocabularies/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/api/models/vocabularies/base.py b/api/models/vocabularies/base.py new file mode 100644 index 0000000..fef0c2f --- /dev/null +++ b/api/models/vocabularies/base.py @@ -0,0 +1,7 @@ +from api.models._commons import ( + ConfigBaseModel, +) +from abc import ABC + +class _VocabBaseModel(ConfigBaseModel, ABC, type="vocabulary"): + ... diff --git a/api/models/vocabularies/genes.py b/api/models/vocabularies/genes.py new file mode 100644 index 0000000..01082ad --- /dev/null +++ b/api/models/vocabularies/genes.py @@ -0,0 +1,5 @@ +from .base import _VocabBaseModel + +class GeneModel(_VocabBaseModel, type="gene"): + symbol: str + description: str diff --git a/api/serialization/__init__.py b/api/serialization/__init__.py index 7aa5fbd..13520a3 100644 --- a/api/serialization/__init__.py +++ b/api/serialization/__init__.py @@ -10,5 +10,6 @@ _retrieval, _common, locations, + # vocabularies, # For now, there are no serialization to be done on vocabolaries ) diff --git a/api/serialization/util/siibra.py b/api/serialization/util/siibra.py index d6e6139..50b001f 100644 --- a/api/serialization/util/siibra.py +++ b/api/serialization/util/siibra.py @@ -45,4 +45,6 @@ from siibra import parcellations, spaces, atlases +from siibra.vocabularies import GENE_NAMES + import siibra diff --git a/api/server/api.py b/api/server/api.py index a542329..165ebda 100644 --- a/api/server/api.py +++ b/api/server/api.py @@ -19,6 +19,7 @@ from .volumes import prefixed_routers as volume_prefixed_routers from .compounds import prefixed_routers as compound_prefixed_routers from .features import router as feature_router +from .volcabularies import router as vocabolaries_router from .metrics import prom_metrics_resp, on_startup as metrics_on_startup, on_terminate as metrics_on_terminate from .code_snippet import get_sourcecode @@ -37,6 +38,7 @@ siibra_api.include_router(prefix_router.router, prefix=prefix_router.prefix) siibra_api.include_router(feature_router, prefix="/feature") +siibra_api.include_router(vocabolaries_router, prefix="/vocabularies") add_pagination(siibra_api) diff --git a/api/server/volcabularies/__init__.py b/api/server/volcabularies/__init__.py new file mode 100644 index 0000000..e556513 --- /dev/null +++ b/api/server/volcabularies/__init__.py @@ -0,0 +1,25 @@ +from fastapi import APIRouter, HTTPException +from fastapi_pagination import paginate, Page +from fastapi_versioning import version + +from api.server.util import SapiCustomRoute +from api.server import FASTAPI_VERSION +from api.siibra_api_config import ROLE +from api.common import router_decorator +from api.models.vocabularies.genes import GeneModel +from api.common.data_handlers.vocabularies.gene import get_genes + +TAGS= ["vocabularies"] +"""HTTP vocabularies tags""" + +router = APIRouter(route_class=SapiCustomRoute, tags=TAGS) +"""HTTP vocabularies router""" + +@router.get("/genes", response_model=Page[GeneModel]) +@version(*FASTAPI_VERSION) +@router_decorator(ROLE, func=get_genes) +def get_genes(find:str=None, func=None): + """HTTP get (filtered) genes""" + if func is None: + raise HTTPException(500, "func: None passed") + return paginate(func(find=find)) diff --git a/api/siibra_api_config.py b/api/siibra_api_config.py index 6840e90..aaac863 100644 --- a/api/siibra_api_config.py +++ b/api/siibra_api_config.py @@ -69,6 +69,7 @@ def get_config_dir_short_hash(path_to_config: str): "features", "volumes", "compounds", + "vocabularies", ] class CELERY_CONFIG: diff --git a/bla.py b/bla.py new file mode 100644 index 0000000..9766f13 --- /dev/null +++ b/bla.py @@ -0,0 +1,24 @@ +feature_types = [ + ((), "FunctionalConnectivity",) + ((), "ReceptorDensityFingerprint",) + ((), "ReceptorDensityProfile",) + ((), "MRIVolumeOfInterest",) + ((), "BlockfaceVolumeOfInterest",) + ((), "RegionalBOLD",) + ((), "PLIVolumeOfInterest",) + ((), "DTIVolumeOfInterest",) + ((), "TracingConnectivity",) + ((), "StreamlineLengths",) + ((), "StreamlineCounts",) + ((), "AnatomoFunctionalConnectivity",) +] + +import os +from subprocess import run +MONITOR_FIRSTLVL_DIR = os.getenv("MONITOR_FIRSTLVL_DIR") +dirs = os.listdir(MONITOR_FIRSTLVL_DIR) +for dir in dirs: + result = run(["du", "-s", f"{MONITOR_FIRSTLVL_DIR}/{dir}"], capture_output=True, text=True) + size_b, *_ = result.stdout.split("\t") + print(size_b) + print("dir", size_b, int(size_b)) \ No newline at end of file diff --git a/docs/develop.example.adding_vocabulary_endpoint.md b/docs/develop.example.adding_vocabulary_endpoint.md new file mode 100644 index 0000000..1cc54b8 --- /dev/null +++ b/docs/develop.example.adding_vocabulary_endpoint.md @@ -0,0 +1,196 @@ +This document will record the process to reimplement the `/genes` endpoint, which was briefly removed during the refactor of `/v2_0` to `/v3_0`. This should serve as an informative document, the process adds a top level module, which needs to be included for the server/worker architecture to work properly. + +## Adding the new import + +`api.serialization.util.siibra` Serves as the one and single entrypoint to siibra package. To access the new variable `GENE_NAMES`, import it here. + +```diff +# in api.serialization.util.siibra + +# ... trimmed for brevity + ++ from siibra.vocabularies import GENE_NAMES +``` + +## Adding serialization model + +Next, we define the shape of serialized instance of gene. As this is the first model to be added in vocabulary module, it is useful to add the base class for vocabulary. + +```diff +# api.models.vocabularies.__init__ +``` + +```diff +# api.models.vocabularies.base ++ from api.models._commons import ( ++ ConfigBaseModel, ++ ) ++ from abc import ABC ++ ++ class _VocabBaseModel(ConfigBaseModel, ABC, type="vocabulary"): ++ ... + +``` + +```diff +# api.models.vocabularies.gene ++ from .base import _VocabBaseModel ++ ++ class GeneModel(_VocabBaseModel, type="gene"): ++ symbol: str ++ description: str + +``` + +The advantage of constructing the inheritence model `ConfigBaseModel` -> `_VocabBaseModel` -> `GeneModel` is so that initialized model will automatically have the `@type` attribute set to `siibra-1.0/vocabulary/gene`. + + +## Adding serialization strategy + +For serializing `GENE_NAMES`, this step can be skipped, as the variable is already a dictionary. + +This is, however, a double edged sword. An unshaped dictionary can change the key/shape without warning. More so than ever, adding tests would be crucial to ensure future versions of siibra does not break siibra-api. + +## Adding query logic + +Here, we define, given arguments as string primitives, how to retrieve siibra objects. In our case, given optional string argument `find`, how to get a list of `GeneModel`. + +```diff +# api.common.data_handlers.vocabularies.__init__ ++ from . import gene +``` + +```diff +# api.common.data_handlers.vocabularies.gene ++ from api.common import data_decorator ++ from api.siibra_api_config import ROLE ++ from api.models.vocabularies.genes import GeneModel ++ ++ @data_decorator(ROLE) ++ def get_genes(find:str=None): ++ """Get all genes ++ ++ Args: ++ string to find in vocabularies ++ ++ Returns: ++ List of the genes.""" ++ from api.serialization.util.siibra import GENE_NAMES ++ ++ if find == None: ++ return_list = [v for v in GENE_NAMES] ++ else: ++ return_list = GENE_NAMES.find(find) ++ ++ return [GeneModel( ++ symbol=v.get("symbol"), ++ description=v.get("description") ++ ).dict() for v in return_list] + +``` + +Important to note here is, since this is a new module, it *must* be imported into the parent module: + +```diff +# api.common.data_handlers.__init__ + +from . import core +from . import features +from . import volumes +from . import compounds ++ from . import vocabularies + +from api.siibra_api_config import ROLE + +if ROLE == "all" or ROLE == "worker": + import siibra + siibra.warm_cache() + +``` + +## Adding the route + +Finally, add the route to FastAPI + +```diff +# api.server.vocabularies.__init__ ++ from fastapi import APIRouter, HTTPException ++ from fastapi_pagination import paginate, Page ++ from fastapi_versioning import version ++ ++ from api.server.util import SapiCustomRoute ++ from api.server import FASTAPI_VERSION ++ from api.siibra_api_config import ROLE ++ from api.common import router_decorator ++ from api.models.vocabularies.genes import GeneModel ++ from api.common.data_handlers.vocabularies.gene import get_genes + ++ TAGS= ["vocabularies"] ++ """HTTP vocabularies tags""" ++ ++ router = APIRouter(route_class=SapiCustomRoute, tags=TAGS) ++ """HTTP vocabularies router""" + ++ @router.get("/genes", response_model=Page[GeneModel]) ++ @version(*FASTAPI_VERSION) ++ @router_decorator(ROLE, func=get_genes) ++ def get_genes(find:str=None, func=None): ++ """HTTP get (filtered) genes""" ++ if func is None: ++ raise HTTPException(500, "func: None passed") ++ return paginate(func(find=find)) + +``` + +```diff +# api.server.api + +# ... truncated for brevity + +from .features import router as feature_router ++ from .volcabularies import router as vocabolaries_router +from .metrics import prom_metrics_resp, on_startup as metrics_on_startup, on_terminate as metrics_on_terminate + +# ... truncated for brevity + +siibra_api.include_router(feature_router, prefix="/feature") ++ siibra_api.include_router(vocabolaries_router, prefix="/vocabularies") + +add_pagination(siibra_api) + +# ... truncated for brevity +``` + +## Configure workers + +Since a new top level module was introduced, a new queue is automatically created. New workers must be configured to take jobs from this queue, or HTTP requests related to this queue will hang forever. + +```diff +# api.siibra_api_config + +# ... truncated for brevity + +_queues = [ + "core", + "features", + "volumes", + "compounds", ++ "vocabularies", +] + +# ... truncated for brevity + +``` + +```diff +# .helm/siibra-api.values.yaml + +# ... truncated for brevity + +sapiVersion: "0.3.15" # "latest" or "0.3.15" +- sapiWorkerQueues: ["core", "features", "volumes", "compounds"] ++ sapiWorkerQueues: ["core", "features", "volumes", "compounds", "vocabularies"] +sapiFlavor: "prod" # could be prod, rc, latest, etc + +# ... truncated for brevity +``` \ No newline at end of file diff --git a/e2e_test/vocabularies/__init__.py b/e2e_test/vocabularies/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/e2e_test/vocabularies/test_genes.py b/e2e_test/vocabularies/test_genes.py new file mode 100644 index 0000000..07df7aa --- /dev/null +++ b/e2e_test/vocabularies/test_genes.py @@ -0,0 +1,22 @@ +import pytest +from fastapi.testclient import TestClient +from api.server import api + +client = TestClient(api) + +@pytest.mark.parametrize('find, expect_no', [ + (None, 50), + ("MA", 50), + ("MAO", 2), + ("MAOC", 0), +]) +def test_genes(find, expect_no): + response = client.get(f"/v3_0/vocabularies/genes", params={ + "find": find + }) + assert response.status_code == 200 + resp_json = response.json() + items = resp_json.get("items", []) + + assert len(items) == expect_no + \ No newline at end of file diff --git a/mkdocs.yml b/mkdocs.yml index b6dfde3..8b4d13f 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -61,6 +61,7 @@ nav: - 'start developer server': 'develop.md' - 'example: adding CompoundFeature serialization': 'develop.example.adding_serialization.md' - 'example: adding related region endpoint': 'develop.example.adding_related_regions.md' + - 'example: adding /gene endpoint': 'develop.example.adding_vocabulary_endpoint.md' - api references: - api.siibra_api_config: 'api.siibra_api_config.md' - api.server: 'api.server.md'