Merge pull request #9 from spapa013/main

reorganize api.config, add api.schemas, change how base imports api

python/microns-morphology-api/microns_morphology_api/__init__.py

@@ -10,4 +10,4 @@
     )
 )
 
-check_latest_version_from_github = version_utils.latest_github_version_checker(owner='cajal', repo='microns-morphology')
+check_latest_version_from_github = version_utils.latest_github_version_checker(owner='cajal', repo='microns-morphology')

python/microns-morphology-api/microns_morphology_api/config/__init__.py

@@ -1,95 +1,37 @@
 """
 Configuration package/module for microns-morphology.
 """
-
-import inspect
-import traceback
-from enum import Enum
+import datajoint.datajoint_plus as djp
+from microns_utils.config_utils import SchemaConfig
 from . import adapters
 from . import externals
-from . import bases
-try:
-    import datajoint as dj
-except:
-    traceback.print_exc()
-    raise ImportError('DataJoint package not found.')
-from microns_utils import config_utils
-
-
-config_utils.enable_datajoint_flags()
-
-
-def register_externals(schema_name:str):
-    """
-    Registers the external stores for a schema_name in this module.
-    """
-    external_stores = config_mapping[SCHEMAS(schema_name)]["externals"]
-
-    if external_stores is not None:
-        config_utils.register_externals(external_stores)
-
-
-def register_adapters(schema_name:str, context=None):
-    """
-    Imports the adapters for a schema_name into the global namespace.
-    """
-    adapter_objects = config_mapping[SCHEMAS(schema_name)]["adapters"]
-
-    if adapter_objects is not None:
-        config_utils.register_adapters(adapter_objects, context=context)
-
-
-def register_bases(schema_name:str, module):
-    """
-    Maps base classes to DataJoint tables.
-    """
-    bases = config_mapping[SCHEMAS(schema_name)]["bases"]
-
-    if bases is not None:
-        for base in bases:
-            config_utils.register_bases(base, module)
-        return module
-
-
-def create_vm(schema_name:str):
-    """
-    Creates a virtual module after registering the external stores, adapter objects, DatajointPlus and base classes.
-    """
-    schema = SCHEMAS(schema_name)
-    vm = config_utils._create_vm(schema.value, external_stores=config_mapping[schema]["externals"], adapter_objects=config_mapping[schema]["adapters"])
-    config_utils.add_datajoint_plus(vm)
-    register_bases(schema_name, vm)
-    return vm
-
-
-class SCHEMAS(Enum):
-    H01_MORPHOLOGY = "microns_h01_morphology"
-    MINNIE65_AUTO_PROOFREADING = "microns_minnie65_auto_proofreading"
-    H01_AUTO_PROOFREADING = "microns_h01_auto_proofreading"
-    MINNIE65_MORPHOLOGY = "microns_minnie65_morphology"
-
-
-config_mapping = {
-    SCHEMAS.H01_MORPHOLOGY: {
-        "externals": externals.h01_morphology,
-        "adapters": adapters.h01_morphology_adapter_objects,
-        "bases": None
-    },
-    SCHEMAS.H01_AUTO_PROOFREADING: {
-        "externals": externals.h01_auto_proofreading,
-        "adapters": adapters.h01_auto_proofreading_adapter_objects,
-        "bases": None
-    },
-    SCHEMAS.MINNIE65_AUTO_PROOFREADING: {
-        "externals": externals.minnie65_auto_proofreading,
-        "adapters": adapters.minnie65_auto_proofreading_adapter_objects,
-        "bases": None
-    },
-    SCHEMAS.MINNIE65_MORPHOLOGY: {
-        "externals": externals.minnie65_morphology,
-        "adapters": adapters.minnie65_morphology_adapter_objects,
-        "bases": None
-
-    },
 
-}
+djp.enable_datajoint_flags()
+
+h01_morphology_config = SchemaConfig(
+    module_name='h01_morphology',
+    schema_name='microns_h01_morphology',
+    externals=externals.h01_morphology,
+    adapters=adapters.h01_morphology
+)
+
+minnie65_auto_proofreading_config = SchemaConfig(
+    module_name='minnie65_auto_proofreading',
+    schema_name='microns_minnie65_auto_proofreading',
+    externals=externals.minnie65_auto_proofreading,
+    adapters=adapters.minnie65_auto_proofreading
+)
+
+h01_auto_proofreading_config = SchemaConfig(
+    module_name='h01_auto_proofreading',
+    schema_name='microns_h01_auto_proofreading',
+    externals=externals.h01_auto_proofreading,
+    adapters=adapters.h01_auto_proofreading
+)
+
+minnie65_morphology_config = SchemaConfig(
+    module_name='minnie65_morphology',
+    schema_name='microns_minnie65_morphology',
+    externals=externals.minnie65_morphology,
+    adapters=adapters.minnie65_morphology
+)

python/microns-morphology-api/microns_morphology_api/config/adapters.py

@@ -243,28 +243,28 @@ def get(self, filepath):
 minnie65_somas = SomasAdapter('filepath@minnie65_somas')
 
 # also store in one object for ease of use with virtual modules
-h01_auto_proofreading_adapter_objects = {
+h01_auto_proofreading = {
     'h01_meshes': h01_meshes,
     'h01_faces': h01_faces,
     'h01_skeletons': h01_skeletons,
     'h01_decomposition': h01_decomposition
 }
 
-h01_morphology_adapter_objects = {
+h01_morphology = {
     'h01_meshes': h01_meshes,
     'h01_skeletons': h01_skeletons,
     'h01_somas':h01_somas,
     'h01_faces':h01_faces,
 }
 
-minnie65_auto_proofreading_adapter_objects = {
+minnie65_auto_proofreading = {
     'minnie65_meshes': minnie65_meshes,
     'minnie65_faces': minnie65_faces,
     'minnie65_skeletons': minnie65_skeletons,
     'minnie65_decomposition': minnie65_decomposition
 }
 
-minnie65_morphology_adapter_objects = {
+minnie65_morphology = {
     'minnie65_meshes': minnie65_meshes,
     'minnie65_skeletons': minnie65_skeletons,
     'minnie65_somas':minnie65_somas,

python/microns-morphology-api/microns_morphology_api/config/externals.py

@@ -3,7 +3,7 @@
 """
 
 from pathlib import Path
-from microns_utils import config_utils
+import datajoint.datajoint_plus as djp
 
 '''
 minnie_em_external_stack_path = Path() / '/mnt' / 'dj-stor01' / 'microns'/ 'minnie' / 'stacks'
@@ -21,48 +21,44 @@
 h01_morphology_external_meshes_path = base_path / 'h01' / 'meshes'
 h01_morphology_external_faces_path = base_path / 'h01' / 'faces'
 h01_morphology = {
-    'h01_somas': config_utils.make_store_dict(h01_morphology_external_somas_path),
-    'h01_skeletons': config_utils.make_store_dict(h01_morphology_external_skeletons_path),
-    'h01_meshes': config_utils.make_store_dict(h01_morphology_external_meshes_path),
-    'h01_faces': config_utils.make_store_dict(h01_morphology_external_faces_path),
-
-    }
+    'h01_somas': djp.make_store_dict(h01_morphology_external_somas_path),
+    'h01_skeletons': djp.make_store_dict(h01_morphology_external_skeletons_path),
+    'h01_meshes': djp.make_store_dict(h01_morphology_external_meshes_path),
+    'h01_faces': djp.make_store_dict(h01_morphology_external_faces_path),
+}
 
 #h01_auto_proofreading
 h01_auto_proofreading_external_decomposition_path = base_path / 'h01' / 'decomposition'
 h01_auto_proofreading_external_faces_path = base_path / 'h01' / 'faces'
 h01_auto_proofreading_external_skeletons_path = base_path / 'h01' / 'skeletons'
 h01_auto_proofreading_external_meshes_path = base_path / 'h01' / 'meshes'
 h01_auto_proofreading = {
-    'h01_decomposition': config_utils.make_store_dict(h01_auto_proofreading_external_decomposition_path),
-    'h01_faces': config_utils.make_store_dict(h01_auto_proofreading_external_faces_path),
-    'h01_skeletons': config_utils.make_store_dict(h01_auto_proofreading_external_skeletons_path),
-    'h01_meshes': config_utils.make_store_dict(h01_auto_proofreading_external_meshes_path),
-
-    }
+    'h01_decomposition': djp.make_store_dict(h01_auto_proofreading_external_decomposition_path),
+    'h01_faces': djp.make_store_dict(h01_auto_proofreading_external_faces_path),
+    'h01_skeletons': djp.make_store_dict(h01_auto_proofreading_external_skeletons_path),
+    'h01_meshes': djp.make_store_dict(h01_auto_proofreading_external_meshes_path),
+}
 
 #minnie65_morphology
 minnie65_morphology_external_somas_path = base_path / 'minnie' / 'somas'
 minnie65_morphology_external_skeletons_path = base_path / 'minnie' / 'skeletons'
 minnie65_morphology_external_meshes_path = base_path / 'minnie' / 'meshes'
 minnie65_morphology_external_faces_path = base_path / 'minnie' / 'faces'
 minnie65_morphology = {
-    'minnie65_somas': config_utils.make_store_dict(minnie65_morphology_external_somas_path),
-    'minnie65_skeletons': config_utils.make_store_dict(minnie65_morphology_external_skeletons_path),
-    'minnie65_meshes': config_utils.make_store_dict(minnie65_morphology_external_meshes_path),
-    'minnie65_faces': config_utils.make_store_dict(minnie65_morphology_external_faces_path),
-
-    }
+    'minnie65_somas': djp.make_store_dict(minnie65_morphology_external_somas_path),
+    'minnie65_skeletons': djp.make_store_dict(minnie65_morphology_external_skeletons_path),
+    'minnie65_meshes': djp.make_store_dict(minnie65_morphology_external_meshes_path),
+    'minnie65_faces': djp.make_store_dict(minnie65_morphology_external_faces_path),
+}
 
 #minnie65_auto_proofreading
 minnie65_auto_proofreading_external_decomposition_path = base_path / 'minnie' / 'decomposition'
 minnie65_auto_proofreading_external_faces_path = base_path / 'minnie' / 'faces'
 minnie65_auto_proofreading_external_skeletons_path = base_path / 'minnie' / 'skeletons'
 minnie65_auto_proofreading_external_meshes_path = base_path / 'minnie' / 'meshes'
 minnie65_auto_proofreading = {
-    'minnie65_decomposition': config_utils.make_store_dict(minnie65_auto_proofreading_external_decomposition_path),
-    'minnie65_faces': config_utils.make_store_dict(minnie65_auto_proofreading_external_faces_path),
-    'minnie65_skeletons': config_utils.make_store_dict(minnie65_auto_proofreading_external_skeletons_path),
-    'minnie65_meshes': config_utils.make_store_dict(minnie65_auto_proofreading_external_meshes_path),
-
-    }
+    'minnie65_decomposition': djp.make_store_dict(minnie65_auto_proofreading_external_decomposition_path),
+    'minnie65_faces': djp.make_store_dict(minnie65_auto_proofreading_external_faces_path),
+    'minnie65_skeletons': djp.make_store_dict(minnie65_auto_proofreading_external_skeletons_path),
+    'minnie65_meshes': djp.make_store_dict(minnie65_auto_proofreading_external_meshes_path),
+}

python/microns-morphology-api/microns_morphology_api/schemas/h01_auto_proofreading.py

@@ -0,0 +1,13 @@
+import datajoint as dj
+from datajoint import datajoint_plus as djp
+
+from ..config import h01_auto_proofreading_config
+
+h01_auto_proofreading_config.register_adapters(context=locals())
+h01_auto_proofreading_config.register_externals()
+
+schema = dj.schema(h01_auto_proofreading_config.schema_name)
+
+schema.spawn_missing_classes()
+schema.connection.dependencies.load()
+

python/microns-morphology-api/microns_morphology_api/schemas/h01_morphology.py

@@ -0,0 +1,12 @@
+import datajoint as dj
+from datajoint import datajoint_plus as djp
+
+from ..config import h01_morphology_config
+
+h01_morphology_config.register_adapters(context=locals())
+h01_morphology_config.register_externals()
+
+schema = dj.schema(h01_morphology_config.schema_name)
+
+schema.spawn_missing_classes()
+schema.connection.dependencies.load()

python/microns-morphology-api/microns_morphology_api/schemas/minnie65_auto_proofreading.py

@@ -0,0 +1,12 @@
+import datajoint as dj
+from datajoint import datajoint_plus as djp
+
+from ..config import minnie65_auto_proofreading_config
+
+minnie65_auto_proofreading_config.register_adapters(context=locals())
+minnie65_auto_proofreading_config.register_externals()
+
+schema = dj.schema(minnie65_auto_proofreading_config.schema_name)
+
+schema.spawn_missing_classes()
+schema.connection.dependencies.load()

python/microns-morphology-api/microns_morphology_api/schemas/minnie65_morphology.py

@@ -0,0 +1,12 @@
+import datajoint as dj
+from datajoint import datajoint_plus as djp
+
+from ..config import minnie65_morphology_config
+
+minnie65_morphology_config.register_adapters(context=locals())
+minnie65_morphology_config.register_externals()
+
+schema = dj.schema(minnie65_morphology_config.schema_name)
+
+schema.spawn_missing_classes()
+schema.connection.dependencies.load()

python/microns-morphology/microns_morphology/__init__.py

@@ -10,4 +10,4 @@
     )
 )
 
-check_latest_version_from_github = version_utils.latest_github_version_checker(owner='cajal', repo='microns-morphology')
+check_latest_version_from_github = version_utils.latest_github_version_checker(owner='cajal', repo='microns-morphology')

python/microns-morphology/microns_morphology/auto_proofreading/__init__.py

@@ -1,5 +1,5 @@
-from microns_morphology_api import config
+import datajoint.datajoint_plus as djp
 from . import h01_auto_proofreading, minnie65_auto_proofreading
 
-config.register_bases(config.SCHEMAS.H01_AUTO_PROOFREADING, h01_auto_proofreading)
-config.register_bases(config.SCHEMAS.MINNIE65_AUTO_PROOFREADING, minnie65_auto_proofreading)
+djp.reassign_master_attribute(h01_auto_proofreading)
+djp.reassign_master_attribute(minnie65_auto_proofreading)

python/microns-morphology/microns_morphology/auto_proofreading/h01_auto_proofreading.py

@@ -1,12 +1,5 @@
 import datajoint as dj
 from datajoint import datajoint_plus as djp
 
-from microns_morphology_api import config
-schema_obj = config.SCHEMAS.H01_AUTO_PROOFREADING
-
-config.register_adapters(schema_obj, context=locals())
-config.register_externals(schema_obj)
-
-schema = dj.schema(schema_obj.value)
-schema.spawn_missing_classes()
+from microns_morphology_api.schemas import h01_auto_proofreading
 

python/microns-morphology/microns_morphology/auto_proofreading/minnie65_auto_proofreading.py

@@ -1,12 +1,4 @@
 import datajoint as dj
 from datajoint import datajoint_plus as djp
 
-from microns_morphology_api import config
-schema_obj = config.SCHEMAS.MINNIE65_AUTO_PROOFREADING
-
-config.register_adapters(schema_obj, context=locals())
-config.register_externals(schema_obj)
-
-schema = dj.schema(schema_obj.value)
-schema.spawn_missing_classes()
-
+from microns_morphology_api.schemas import minnie65_auto_proofreading

python/microns-morphology/microns_morphology/h01_morphology/__init__.py

@@ -1,4 +1,4 @@
-from microns_morphology_api import config
+import datajoint.datajoint_plus as djp
 from . import h01_morphology
 
-config.register_bases(config.SCHEMAS.H01_MORPHOLOGY, h01_morphology)
+djp.reassign_master_attribute(h01_morphology)

python/microns-morphology/microns_morphology/h01_morphology/h01_morphology.py

@@ -1,11 +1,4 @@
 import datajoint as dj
 from datajoint import datajoint_plus as djp
 
-from microns_morphology_api import config
-schema_obj = config.SCHEMAS.H01_MORPHOLOGY
-
-config.register_adapters(schema_obj, context=locals())
-config.register_externals(schema_obj)
-
-schema = dj.schema(schema_obj.value)
-schema.spawn_missing_classes()
+from microns_morphology_api.schemas import h01_morphology

python/microns-morphology/microns_morphology/minnie_morphology/__init__.py

@@ -1,4 +1,4 @@
-from microns_morphology_api import config
+import datajoint.datajoint_plus as djp
 from . import minnie65_morphology
 
-config.register_bases(config.SCHEMAS.MINNIE65_MORPHOLOGY, minnie65_morphology)
+djp.reassign_master_attribute(minnie65_morphology)

python/microns-morphology/microns_morphology/minnie_morphology/minnie65_morphology.py

@@ -1,11 +1,4 @@
 import datajoint as dj
 from datajoint import datajoint_plus as djp
 
-from microns_morphology_api import config
-schema_obj = config.SCHEMAS.MINNIE65_MORPHOLOGY
-
-config.register_adapters(schema_obj, context=locals())
-config.register_externals(schema_obj)
-
-schema = dj.schema(schema_obj.value)
-schema.spawn_missing_classes()
+from microns_morphology_api.schemas import minnie65_morphology