From 80dfab4edde7134c26f26db9024f1678f1058e29 Mon Sep 17 00:00:00 2001 From: spapa013 Date: Tue, 21 Dec 2021 14:48:03 -0600 Subject: [PATCH 01/12] add newline --- .../microns_materialization_api/__init__.py | 2 +- .../microns-materialization/microns_materialization/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/python/microns-materialization-api/microns_materialization_api/__init__.py b/python/microns-materialization-api/microns_materialization_api/__init__.py index df8e120..bff272c 100644 --- a/python/microns-materialization-api/microns_materialization_api/__init__.py +++ b/python/microns-materialization-api/microns_materialization_api/__init__.py @@ -10,4 +10,4 @@ ) ) -check_latest_version_from_github = version_utils.latest_github_version_checker(owner='cajal', repo='microns-materialization') \ No newline at end of file +check_latest_version_from_github = version_utils.latest_github_version_checker(owner='cajal', repo='microns-materialization') diff --git a/python/microns-materialization/microns_materialization/__init__.py b/python/microns-materialization/microns_materialization/__init__.py index a8f0c16..6667695 100644 --- a/python/microns-materialization/microns_materialization/__init__.py +++ b/python/microns-materialization/microns_materialization/__init__.py @@ -10,4 +10,4 @@ ) ) -check_latest_version_from_github = version_utils.latest_github_version_checker(owner='cajal', repo='microns-materialization') \ No newline at end of file +check_latest_version_from_github = version_utils.latest_github_version_checker(owner='cajal', repo='microns-materialization') From 8e413e09426e690ba3aba3bce218cdf1c69ba98c Mon Sep 17 00:00:00 2001 From: spapa013 Date: Tue, 21 Dec 2021 14:48:26 -0600 Subject: [PATCH 02/12] init api.schemas --- .../microns_materialization_api/schemas/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 python/microns-materialization-api/microns_materialization_api/schemas/__init__.py diff --git a/python/microns-materialization-api/microns_materialization_api/schemas/__init__.py b/python/microns-materialization-api/microns_materialization_api/schemas/__init__.py new file mode 100644 index 0000000..e69de29 From 0474da9805fa93d2b7eb13604b52410cf16812f8 Mon Sep 17 00:00:00 2001 From: spapa013 Date: Tue, 21 Dec 2021 14:48:58 -0600 Subject: [PATCH 03/12] use SchemaConfig for api.config --- .../config/__init__.py | 93 ++++--------------- 1 file changed, 17 insertions(+), 76 deletions(-) diff --git a/python/microns-materialization-api/microns_materialization_api/config/__init__.py b/python/microns-materialization-api/microns_materialization_api/config/__init__.py index 914575a..4be6ebb 100644 --- a/python/microns-materialization-api/microns_materialization_api/config/__init__.py +++ b/python/microns-materialization-api/microns_materialization_api/config/__init__.py @@ -1,82 +1,23 @@ """ Configuration package/module for microns-materialization. """ - -import inspect -import traceback -from enum import Enum +import datajoint.datajoint_plus as djp +from microns_utils.config_utils import SchemaConfig from . import adapters from . import externals -from . import bases -try: - import datajoint as dj -except: - traceback.print_exc() - raise ImportError('DataJoint package not found.') -from microns_utils import config_utils - - -config_utils.enable_datajoint_flags() - - -def register_externals(schema_name:str): - """ - Registers the external stores for a schema_name in this module. - """ - external_stores = config_mapping[SCHEMAS(schema_name)]["externals"] - - if external_stores is not None: - config_utils.register_externals(external_stores) - - -def register_adapters(schema_name:str, context=None): - """ - Imports the adapters for a schema_name into the global namespace. - """ - adapter_objects = config_mapping[SCHEMAS(schema_name)]["adapters"] - - if adapter_objects is not None: - config_utils.register_adapters(adapter_objects, context=context) - - -def register_bases(schema_name:str, module): - """ - Maps base classes to DataJoint tables. - """ - bases = config_mapping[SCHEMAS(schema_name)]["bases"] - - if bases is not None: - for base in bases: - config_utils.register_bases(base, module) - return module - - -def create_vm(schema_name:str): - """ - Creates a virtual module after registering the external stores, adapter objects, DatajointPlus and base classes. - """ - schema = SCHEMAS(schema_name) - vm = config_utils._create_vm(schema.value, external_stores=config_mapping[schema]["externals"], adapter_objects=config_mapping[schema]["adapters"]) - config_utils.add_datajoint_plus(vm) - register_bases(schema_name, vm) - return vm - - -class SCHEMAS(Enum): - H01_MATERIALIZATION = "microns_h01_materialization" - MINNIE65_MATERIALIZATION = "microns_minnie65_materialization" - -config_mapping = { - SCHEMAS.H01_MATERIALIZATION: { - "externals": externals.h01_materialization, - "adapters": adapters.h01_materialization_adapter_objects, - "bases": None - }, - SCHEMAS.MINNIE65_MATERIALIZATION: { - "externals": externals.minnie65_materialization, - "adapters": adapters.minnie65_materialization_adapter_objects, - "bases": None - }, - -} +djp.enable_datajoint_flags() + +h01_materialization_config = SchemaConfig( + module_name='h01_materialization', + schema_name='microns_h01_materialization', + externals=externals.h01_materialization, + adapters=adapters.h01_materialization +) + +minnie65_materialization_config = SchemaConfig( + module_name='minnie65_materialization', + schema_name='microns_minnie65_materialization', + externals=externals.minnie65_materialization, + adapters=adapters.minnie65_materialization +) From 5c84661542c80ce5994c50fe766d9fac27204a24 Mon Sep 17 00:00:00 2001 From: spapa013 Date: Tue, 21 Dec 2021 14:49:14 -0600 Subject: [PATCH 04/12] modify naming --- .../microns_materialization_api/config/adapters.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/microns-materialization-api/microns_materialization_api/config/adapters.py b/python/microns-materialization-api/microns_materialization_api/config/adapters.py index c0c87e4..f8cd127 100644 --- a/python/microns-materialization-api/microns_materialization_api/config/adapters.py +++ b/python/microns-materialization-api/microns_materialization_api/config/adapters.py @@ -46,10 +46,10 @@ def get(self, filepath): # also store in one object for ease of use with virtual modules -h01_materialization_adapter_objects = { +h01_materialization = { 'h01_meshes': h01_meshes, } -minnie65_materialization_adapter_objects = { +minnie65_materialization = { 'minnie65_meshes': minnie65_meshes, } From a742502d4aa9c621101abe7d0bc6a964d5fd3576 Mon Sep 17 00:00:00 2001 From: spapa013 Date: Tue, 21 Dec 2021 14:49:26 -0600 Subject: [PATCH 05/12] make_store_dict with djp --- .../microns_materialization_api/config/externals.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/python/microns-materialization-api/microns_materialization_api/config/externals.py b/python/microns-materialization-api/microns_materialization_api/config/externals.py index b89929c..e1422ea 100644 --- a/python/microns-materialization-api/microns_materialization_api/config/externals.py +++ b/python/microns-materialization-api/microns_materialization_api/config/externals.py @@ -1,22 +1,19 @@ """ Externals for DataJoint tables. """ - +import datajoint.datajoint_plus as djp from pathlib import Path -from microns_utils import config_utils base_path = Path() / '/mnt' / 'dj-stor01' / 'microns' #h01 materialization h01_materialization_external_meshes_path = base_path / 'h01' / 'meshes' h01_materialization = { - 'h01_meshes': config_utils.make_store_dict(h01_materialization_external_meshes_path), - - } + 'h01_meshes': djp.make_store_dict(h01_materialization_external_meshes_path), +} #minnie65_materialization minnie65_materialization_external_meshes_path = base_path / 'minnie' / 'meshes' minnie65_materialization = { - 'minnie65_meshes': config_utils.make_store_dict(minnie65_materialization_external_meshes_path), - - } + 'minnie65_meshes': djp.make_store_dict(minnie65_materialization_external_meshes_path), +} From 1cebab62dd0813bceab23f15e94b124739e140f9 Mon Sep 17 00:00:00 2001 From: spapa013 Date: Tue, 21 Dec 2021 14:49:38 -0600 Subject: [PATCH 06/12] delete bases.py --- .../microns_materialization_api/config/bases.py | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 python/microns-materialization-api/microns_materialization_api/config/bases.py diff --git a/python/microns-materialization-api/microns_materialization_api/config/bases.py b/python/microns-materialization-api/microns_materialization_api/config/bases.py deleted file mode 100644 index 1b1d909..0000000 --- a/python/microns-materialization-api/microns_materialization_api/config/bases.py +++ /dev/null @@ -1,3 +0,0 @@ -""" -Base classes for DataJoint tables. -""" \ No newline at end of file From 705ab16b8458e691597e093a8f5291455f6cca91 Mon Sep 17 00:00:00 2001 From: spapa013 Date: Tue, 21 Dec 2021 14:50:18 -0600 Subject: [PATCH 07/12] remove register_bases and add reassign_master_attribute --- .../microns_materialization/h01_materialization/__init__.py | 4 ++-- .../minnie_materialization/__init__.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/python/microns-materialization/microns_materialization/h01_materialization/__init__.py b/python/microns-materialization/microns_materialization/h01_materialization/__init__.py index 7dc34a2..f23b7d3 100644 --- a/python/microns-materialization/microns_materialization/h01_materialization/__init__.py +++ b/python/microns-materialization/microns_materialization/h01_materialization/__init__.py @@ -1,4 +1,4 @@ -from microns_materialization_api import config +import datajoint.datajoint_plus as djp from . import h01_materialization -config.register_bases(config.SCHEMAS.H01_MATERIALIZATION, h01_materialization) \ No newline at end of file +djp.reassign_master_attribute(h01_materialization) diff --git a/python/microns-materialization/microns_materialization/minnie_materialization/__init__.py b/python/microns-materialization/microns_materialization/minnie_materialization/__init__.py index 8744d4a..1712eb0 100644 --- a/python/microns-materialization/microns_materialization/minnie_materialization/__init__.py +++ b/python/microns-materialization/microns_materialization/minnie_materialization/__init__.py @@ -1,4 +1,4 @@ -from microns_materialization_api import config +import datajoint.datajoint_plus as djp from . import minnie65_materialization -config.register_bases(config.SCHEMAS.MINNIE65_MATERIALIZATION, minnie65_materialization) \ No newline at end of file +djp.reassign_master_attribute(minnie65_materialization) From 411f4ff56cd43ffafaef7a9f01a2866d2994730a Mon Sep 17 00:00:00 2001 From: spapa013 Date: Tue, 21 Dec 2021 14:50:46 -0600 Subject: [PATCH 08/12] move definitions to api.schemas --- .../minnie65_materialization.py | 250 ++---------------- 1 file changed, 29 insertions(+), 221 deletions(-) diff --git a/python/microns-materialization/microns_materialization/minnie_materialization/minnie65_materialization.py b/python/microns-materialization/microns_materialization/minnie_materialization/minnie65_materialization.py index da6d7c7..eee0255 100644 --- a/python/microns-materialization/microns_materialization/minnie_materialization/minnie65_materialization.py +++ b/python/microns-materialization/microns_materialization/minnie_materialization/minnie65_materialization.py @@ -17,43 +17,13 @@ from tqdm import tqdm # Schema creation -from microns_materialization_api import config -schema_obj = config.SCHEMAS.MINNIE65_MATERIALIZATION +from microns_materialization_api.schemas import minnie65_materialization as m65mat -config.register_adapters(schema_obj, context=locals()) -config.register_externals(schema_obj) - -# Schema creation -schema = dj.schema(schema_obj.value) -schema.spawn_missing_classes() -schema.connection.dependencies.load() - -@schema -class Materialization(dj.Manual): - definition = """ - # version and timestamp of Minnie65 materialization - ver : DECIMAL(6,2) # materialization version - --- - valid=1 : tinyint # marks whether the materialization is valid. Defaults to 1. - timestamp : timestamp # marks the time at which the materialization service was started - """ +class Materialization(m65mat.Materialization): - class CurrentVersion(dj.Part): - definition = """ - # version and timestamp of Minnie65 materialization - kind : varchar(16) # selection criteria for setting the current materialization - --- - -> master - description=NULL : varchar(256) # description of the materialization and source - """ + class CurrentVersion(m65mat.Materialization.CurrentVersion): pass - class Meta(dj.Part): - definition = """ - -> master - --- - description=NULL : varchar(256) # description of the materialization and source - ts_inserted=CURRENT_TIMESTAMP : timestamp # timestamp that data was inserted into this DataJoint table. - """ + class Meta(m65mat.Materialization.Meta): pass @classmethod def fill(cls, client, ver, description=None, update_latest_tag=True): @@ -82,11 +52,11 @@ def fill(cls, client, ver, description=None, update_latest_tag=True): print(f'Updated Materialization.CurrentVersion "latest" to materialization version {client.materialize.version:.2f}') -stable = Materialization.CurrentVersion & {'kind': 'stable'} -latest = Materialization.CurrentVersion & {'kind': 'latest'} -MICrONS_Mar_2021 = Materialization.CurrentVersion & {'kind': 'MICrONS_Mar_2021'} -MICrONS_Jul_2021 = Materialization.CurrentVersion & {'kind': 'MICrONS_Jul_2021'} -releaseJun2021 = Materialization.CurrentVersion & {'kind': 'release_Jun2021'} +stable = m65mat.stable +latest = m65mat.latest +MICrONS_Mar_2021 = m65mat.MICrONS_Mar_2021 +MICrONS_Jul_2021 = m65mat.MICrONS_Jul_2021 +releaseJun2021 = m65mat.releaseJun2021 def _version_mgr(client, ver, table=Materialization): """ Modify the client to the user specified materialization version and check if the version is already in the DataJoint table. @@ -127,35 +97,12 @@ def _version_mgr(client, ver, table=Materialization): return code, client -@schema -class Nucleus(dj.Manual): - definition = """ - # Nucleus detection from version 0 of the Allen Institute. Table name: 'nucleus_detection_v0' - nucleus_id : int unsigned # id of nucleus from the flat segmentation Equivalent to Allen: 'id'. - --- - """ + +class Nucleus(m65mat.Nucleus): - class Info(dj.Part): - definition = """ - # Detailed information from each nucleus_id - -> Materialization - -> master - segment_id : bigint unsigned # id of the segment under the nucleus centroid. Equivalent to Allen 'pt_root_id'. - --- - nucleus_x : int unsigned # x coordinate of nucleus centroid in EM voxels (x: 4nm, y: 4nm, z: 40nm) - nucleus_y : int unsigned # y coordinate of nucleus centroid in EM voxels (x: 4nm, y: 4nm, z: 40nm) - nucleus_z : int unsigned # z coordinate of nucleus centroid in EM voxels (x: 4nm, y: 4nm, z: 40nm) - supervoxel_id : bigint unsigned # id of the supervoxel under the nucleus centroid. Equivalent to Allen: 'pt_supervoxel_id'. - volume=NULL : float # volume of the nucleus in um^3 - """ - - class Meta(dj.Part): - definition = """ - ->Materialization - --- - description=NULL : varchar(256) # description of the table version - ts_inserted=CURRENT_TIMESTAMP : timestamp # timestamp that data was inserted into this DataJoint table. - """ + class Info(m65mat.Nucleus.Info): pass + + class Meta(m65mat.Nucleus.Meta): pass @classmethod def fetch_df(cls, client): @@ -201,30 +148,10 @@ def fill(cls, client, description=None): print(f'Successfully inserted nucleus information for materialization version: {client.materialize.version:.2f}') -@schema -class FunctionalCoreg(dj.Manual): - definition = """ - # ID's of cells from the table 'functional_coreg' - ->Materialization - ->Nucleus - segment_id : bigint unsigned # id of the segment under the nucleus centroid. Equivalent to Allen 'pt_root_id'. - scan_session : smallint # session index for the mouse - scan_idx : smallint # number of TIFF stack file - unit_id : int # unit id from ScanSet.Unit - --- - centroid_x : int unsigned # x coordinate of centroid in EM voxels (x: 4nm, y: 4nm, z: 40nm) - centroid_y : int unsigned # y coordinate of centroid in EM voxels (x: 4nm, y: 4nm, z: 40nm) - centroid_z : int unsigned # z coordinate of centroid in EM voxels (x: 4nm, y: 4nm, z: 40nm) - supervoxel_id : bigint unsigned # id of the supervoxel under the nucleus centroid. Equivalent to Allen: 'pt_supervoxel_id'. - """ + +class FunctionalCoreg(m65mat.FunctionalCoreg): - class Meta(dj.Part): - definition = """ - ->Materialization - --- - description=NULL : varchar(256) # description of the table version - ts_inserted=CURRENT_TIMESTAMP : timestamp # timestamp that data was inserted into this DataJoint table. - """ + class Meta(m65mat.FunctionalCoreg.Meta): pass @classmethod def fetch_df(cls, client): @@ -264,21 +191,9 @@ def fill(cls, client, description=None): print(f'Successfully inserted functional coregistration for materialization version: {client.materialize.version:.2f}') -@schema -class ProofreadSegment(dj.Manual): - definition = """ - # Segment ID's of manually proofread neurons from 'proofreading_functional_coreg_v2' - ->Materialization - segment_id : bigint unsigned # id of the segment under the nucleus centroid. Equivalent to Allen 'pt_root_id'. - """ +class ProofreadSegment(m65mat.ProofreadSegment): - class Meta(dj.Part): - definition = """ - ->Materialization - --- - description=NULL : varchar(256) # description of the table version - ts_inserted=CURRENT_TIMESTAMP : timestamp # timestamp that data was inserted into this DataJoint table. - """ + class Meta(m65mat.ProofreadSegment): pass @classmethod def fetch_df(cls, client): @@ -305,30 +220,9 @@ def fill(cls, client, description=None): print(f'Successfully inserted proofread segments for materialization version: {client.materialize.version:.2f}') -@schema -class ProofreadFunctionalCoregV2(dj.Manual): - definition = """ - # ID's of cells from the table 'proofreading_functional_coreg_v2' - ->Materialization - ->Nucleus - segment_id : bigint unsigned # id of the segment under the nucleus centroid. Equivalent to Allen 'pt_root_id'. - scan_session : smallint # session index for the mouse - scan_idx : smallint # number of TIFF stack file - unit_id : int # unit id from ScanSet.Unit - --- - centroid_x : int unsigned # x coordinate of centroid in EM voxels (x: 4nm, y: 4nm, z: 40nm) - centroid_y : int unsigned # y coordinate of centroid in EM voxels (x: 4nm, y: 4nm, z: 40nm) - centroid_z : int unsigned # z coordinate of centroid in EM voxels (x: 4nm, y: 4nm, z: 40nm) - supervoxel_id : bigint unsigned # id of the supervoxel under the nucleus centroid. Equivalent to Allen: 'pt_supervoxel_id'. - """ +class ProofreadFunctionalCoregV2(m65mat.ProofreadFunctionalCoregV2): - class Meta(dj.Part): - definition = """ - ->Materialization - --- - description=NULL : varchar(256) # description of the table version - ts_inserted=CURRENT_TIMESTAMP : timestamp # timestamp that data was inserted into this DataJoint table. - """ + class Meta(m65mat.ProofreadFunctionalCoregV2.Meta): pass @classmethod def fetch_df(cls, client): @@ -353,30 +247,10 @@ def fetch_df(cls, client): return fc_nuc_df -@schema -class SynapseSegmentSource(dj.Manual): - definition = """ - segment_id : bigint unsigned # id of the segment under the nucleus centroid. Equivalent to Allen 'pt_root_id'. - --- - include=1 : tinyint # 1 if included in synapse source, 0 if excluded from synapse source - """ +class SynapseSegmentSource(m65mat.SynapseSegmentSource): pass -@schema -class Synapse(dj.Computed): - definition = """ - # Synapses from the table 'synapses_pni_2' - ->SynapseSegmentSource.proj(primary_seg_id='segment_id') - secondary_seg_id : bigint unsigned # id of the segment that is synaptically paired to primary_segment_id. - synapse_id : bigint unsigned # synapse index within the segmentation - --- - prepost : varchar(16) # whether the primary_seg_id is "presyn" or "postsyn" - synapse_x : int unsigned # x coordinate of synapse centroid in EM voxels (x: 4nm, y: 4nm, z: 40nm). From Allen 'ctr_pt_position'. - synapse_y : int unsigned # y coordinate of centroid in EM voxels (x: 4nm, y: 4nm, z: 40nm). From Allen 'ctr_pt_position'. - synapse_z : int unsigned # z coordinate of centroid in EM voxels (x: 4nm, y: 4nm, z: 40nm). From Allen 'ctr_pt_position'. - synapse_size : int unsigned # (EM voxels) scaled by (4x4x40) - ts_inserted=CURRENT_TIMESTAMP : timestamp # timestamp that data was inserted into this DataJoint table. - """ +class Synapse(m65mat.Synapse): client = None @@ -493,31 +367,9 @@ def fetch_materialization(ver=-1, update_latest_tag=True): ## ADDITIONAL TABLES -@schema -class AllenV1ColumnTypesSlanted(dj.Manual): - definition = """ - # ID's of cells from the table 'allen_v1_column_types_slanted' - ->Materialization - ->Nucleus - segment_id : bigint unsigned # id of the segment under the nucleus centroid. Equivalent to Allen 'pt_root_id'. - --- - valid : tinyint # 1 = valid entry, 0 = invalid entry - classification_system : varchar(128) # method for classification - cell_type : varchar(128) # manually classified cell type - n_nuc : smallint # number of nuclei associated with segment_id (single somas n_nuc = 1) - centroid_x : int unsigned # x coordinate of centroid in EM voxels (x: 4nm, y: 4nm, z: 40nm) - centroid_y : int unsigned # y coordinate of centroid in EM voxels (x: 4nm, y: 4nm, z: 40nm) - centroid_z : int unsigned # z coordinate of centroid in EM voxels (x: 4nm, y: 4nm, z: 40nm) - supervoxel_id : bigint unsigned # id of the supervoxel under the nucleus centroid. Equivalent to Allen: 'pt_supervoxel_id'. - """ +class AllenV1ColumnTypesSlanted(m65mat.AllenV1ColumnTypesSlanted): - class Meta(dj.Part): - definition = """ - ->Materialization - --- - description=NULL : varchar(256) # description of the table version - ts_inserted=CURRENT_TIMESTAMP : timestamp # timestamp that data was inserted into this DataJoint table. - """ + class Meta(m65mat.AllenV1ColumnTypesSlanted.Meta): pass @classmethod def fetch_df(cls, client): @@ -560,31 +412,9 @@ def fill(cls, client, description=None): print(f'Successfully inserted data for materialization version: {client.materialize.version:.2f}') -@schema -class AllenSomaCourseCellClassModelV1(dj.Manual): - definition = """ - # ID's of cells from the table 'allen_soma_coarse_cell_class_model_v1' - ->Materialization - ->Nucleus - segment_id : bigint unsigned # id of the segment under the nucleus centroid. Equivalent to Allen 'pt_root_id'. - --- - valid : tinyint # 1 = valid entry, 0 = invalid entry - classification_system : varchar(128) # method for classification - cell_type : varchar(128) # manually classified cell type - n_nuc : smallint # number of nuclei associated with segment_id (single somas n_nuc = 1) - centroid_x : int unsigned # x coordinate of centroid in EM voxels (x: 4nm, y: 4nm, z: 40nm) - centroid_y : int unsigned # y coordinate of centroid in EM voxels (x: 4nm, y: 4nm, z: 40nm) - centroid_z : int unsigned # z coordinate of centroid in EM voxels (x: 4nm, y: 4nm, z: 40nm) - supervoxel_id : bigint unsigned # id of the supervoxel under the nucleus centroid. Equivalent to Allen: 'pt_supervoxel_id'. - """ +class AllenSomaCourseCellClassModelV1(m65mat.AllenSomaCourseCellClassModelV1): - class Meta(dj.Part): - definition = """ - ->Materialization - --- - description=NULL : varchar(256) # description of the table version - ts_inserted=CURRENT_TIMESTAMP : timestamp # timestamp that data was inserted into this DataJoint table. - """ + class Meta(m65mat.AllenSomaCourseCellClassModelV1.Meta): pass @classmethod def fetch_df(cls, client): @@ -627,31 +457,9 @@ def fill(cls, client, description=None): print(f'Successfully inserted data for materialization version: {client.materialize.version:.2f}') -@schema -class AllenSomaCourseCellClassModelV2(dj.Manual): - definition = """ - # ID's of cells from the table 'allen_soma_coarse_cell_class_model_v2' - ->Materialization - ->Nucleus - segment_id : bigint unsigned # id of the segment under the nucleus centroid. Equivalent to Allen 'pt_root_id'. - --- - valid : tinyint # 1 = valid entry, 0 = invalid entry - classification_system : varchar(128) # method for classification - cell_type : varchar(128) # manually classified cell type - n_nuc : smallint # number of nuclei associated with segment_id (single somas n_nuc = 1) - centroid_x : int unsigned # x coordinate of centroid in EM voxels (x: 4nm, y: 4nm, z: 40nm) - centroid_y : int unsigned # y coordinate of centroid in EM voxels (x: 4nm, y: 4nm, z: 40nm) - centroid_z : int unsigned # z coordinate of centroid in EM voxels (x: 4nm, y: 4nm, z: 40nm) - supervoxel_id : bigint unsigned # id of the supervoxel under the nucleus centroid. Equivalent to Allen: 'pt_supervoxel_id'. - """ +class AllenSomaCourseCellClassModelV2(m65mat.AllenSomaCourseCellClassModelV2): - class Meta(dj.Part): - definition = """ - ->Materialization - --- - description=NULL : varchar(256) # description of the table version - ts_inserted=CURRENT_TIMESTAMP : timestamp # timestamp that data was inserted into this DataJoint table. - """ + class Meta(m65mat.AllenSomaCourseCellClassModelV2.Meta): pass @classmethod def fetch_df(cls, client): From d62114f5613b96b6e4960a1a5792c725432d0a68 Mon Sep 17 00:00:00 2001 From: spapa013 Date: Tue, 21 Dec 2021 14:51:00 -0600 Subject: [PATCH 09/12] add minnie65_materialization to api.schemas --- .../schemas/minnie65_materialization.py | 253 ++++++++++++++++++ 1 file changed, 253 insertions(+) create mode 100644 python/microns-materialization-api/microns_materialization_api/schemas/minnie65_materialization.py diff --git a/python/microns-materialization-api/microns_materialization_api/schemas/minnie65_materialization.py b/python/microns-materialization-api/microns_materialization_api/schemas/minnie65_materialization.py new file mode 100644 index 0000000..7338cb8 --- /dev/null +++ b/python/microns-materialization-api/microns_materialization_api/schemas/minnie65_materialization.py @@ -0,0 +1,253 @@ +""" +DataJoint tables for importing minnie65 from CAVE. +""" +import datajoint as dj +import datajoint.datajoint_plus as djp + +from ..config import minnie65_materialization_config + +minnie65_materialization_config.register_externals() +minnie65_materialization_config.register_adapters(context=locals()) + +schema = dj.schema(minnie65_materialization_config.schema_name, create_schema=True) + +@schema +class Materialization(djp.Manual): + definition = """ + # version and timestamp of Minnie65 materialization + ver : DECIMAL(6,2) # materialization version + --- + valid=1 : tinyint # marks whether the materialization is valid. Defaults to 1. + timestamp : timestamp # marks the time at which the materialization service was started + """ + + class CurrentVersion(djp.Part): + definition = """ + # version and timestamp of Minnie65 materialization + kind : varchar(16) # selection criteria for setting the current materialization + --- + -> master + description=NULL : varchar(256) # description of the materialization and source + """ + + class Meta(djp.Part): + definition = """ + -> master + --- + description=NULL : varchar(256) # description of the materialization and source + ts_inserted=CURRENT_TIMESTAMP : timestamp # timestamp that data was inserted into this DataJoint table. + """ + + +stable = Materialization.CurrentVersion & {'kind': 'stable'} +latest = Materialization.CurrentVersion & {'kind': 'latest'} +MICrONS_Mar_2021 = Materialization.CurrentVersion & {'kind': 'MICrONS_Mar_2021'} +MICrONS_Jul_2021 = Materialization.CurrentVersion & {'kind': 'MICrONS_Jul_2021'} +releaseJun2021 = Materialization.CurrentVersion & {'kind': 'release_Jun2021'} + + +@schema +class Nucleus(djp.Manual): + definition = """ + # Nucleus detection from version 0 of the Allen Institute. Table name: 'nucleus_detection_v0' + nucleus_id : int unsigned # id of nucleus from the flat segmentation Equivalent to Allen: 'id'. + --- + """ + + class Info(djp.Part): + definition = """ + # Detailed information from each nucleus_id + -> Materialization + -> master + segment_id : bigint unsigned # id of the segment under the nucleus centroid. Equivalent to Allen 'pt_root_id'. + --- + nucleus_x : int unsigned # x coordinate of nucleus centroid in EM voxels (x: 4nm, y: 4nm, z: 40nm) + nucleus_y : int unsigned # y coordinate of nucleus centroid in EM voxels (x: 4nm, y: 4nm, z: 40nm) + nucleus_z : int unsigned # z coordinate of nucleus centroid in EM voxels (x: 4nm, y: 4nm, z: 40nm) + supervoxel_id : bigint unsigned # id of the supervoxel under the nucleus centroid. Equivalent to Allen: 'pt_supervoxel_id'. + volume=NULL : float # volume of the nucleus in um^3 + """ + + class Meta(djp.Part): + definition = """ + ->Materialization + --- + description=NULL : varchar(256) # description of the table version + ts_inserted=CURRENT_TIMESTAMP : timestamp # timestamp that data was inserted into this DataJoint table. + """ + + +@schema +class FunctionalCoreg(dj.Manual): + definition = """ + # ID's of cells from the table 'functional_coreg' + ->Materialization + ->Nucleus + segment_id : bigint unsigned # id of the segment under the nucleus centroid. Equivalent to Allen 'pt_root_id'. + scan_session : smallint # session index for the mouse + scan_idx : smallint # number of TIFF stack file + unit_id : int # unit id from ScanSet.Unit + --- + centroid_x : int unsigned # x coordinate of centroid in EM voxels (x: 4nm, y: 4nm, z: 40nm) + centroid_y : int unsigned # y coordinate of centroid in EM voxels (x: 4nm, y: 4nm, z: 40nm) + centroid_z : int unsigned # z coordinate of centroid in EM voxels (x: 4nm, y: 4nm, z: 40nm) + supervoxel_id : bigint unsigned # id of the supervoxel under the nucleus centroid. Equivalent to Allen: 'pt_supervoxel_id'. + """ + + class Meta(dj.Part): + definition = """ + ->Materialization + --- + description=NULL : varchar(256) # description of the table version + ts_inserted=CURRENT_TIMESTAMP : timestamp # timestamp that data was inserted into this DataJoint table. + """ + + +@schema +class ProofreadSegment(dj.Manual): + definition = """ + # Segment ID's of manually proofread neurons from 'proofreading_functional_coreg_v2' + ->Materialization + segment_id : bigint unsigned # id of the segment under the nucleus centroid. Equivalent to Allen 'pt_root_id'. + """ + + class Meta(dj.Part): + definition = """ + ->Materialization + --- + description=NULL : varchar(256) # description of the table version + ts_inserted=CURRENT_TIMESTAMP : timestamp # timestamp that data was inserted into this DataJoint table. + """ + + +@schema +class ProofreadFunctionalCoregV2(dj.Manual): + definition = """ + # ID's of cells from the table 'proofreading_functional_coreg_v2' + ->Materialization + ->Nucleus + segment_id : bigint unsigned # id of the segment under the nucleus centroid. Equivalent to Allen 'pt_root_id'. + scan_session : smallint # session index for the mouse + scan_idx : smallint # number of TIFF stack file + unit_id : int # unit id from ScanSet.Unit + --- + centroid_x : int unsigned # x coordinate of centroid in EM voxels (x: 4nm, y: 4nm, z: 40nm) + centroid_y : int unsigned # y coordinate of centroid in EM voxels (x: 4nm, y: 4nm, z: 40nm) + centroid_z : int unsigned # z coordinate of centroid in EM voxels (x: 4nm, y: 4nm, z: 40nm) + supervoxel_id : bigint unsigned # id of the supervoxel under the nucleus centroid. Equivalent to Allen: 'pt_supervoxel_id'. + """ + + class Meta(dj.Part): + definition = """ + ->Materialization + --- + description=NULL : varchar(256) # description of the table version + ts_inserted=CURRENT_TIMESTAMP : timestamp # timestamp that data was inserted into this DataJoint table. + """ + + +@schema +class SynapseSegmentSource(dj.Manual): + definition = """ + segment_id : bigint unsigned # id of the segment under the nucleus centroid. Equivalent to Allen 'pt_root_id'. + --- + include=1 : tinyint # 1 if included in synapse source, 0 if excluded from synapse source + """ + + +@schema +class Synapse(dj.Computed): + definition = """ + # Synapses from the table 'synapses_pni_2' + ->SynapseSegmentSource.proj(primary_seg_id='segment_id') + secondary_seg_id : bigint unsigned # id of the segment that is synaptically paired to primary_segment_id. + synapse_id : bigint unsigned # synapse index within the segmentation + --- + prepost : varchar(16) # whether the primary_seg_id is "presyn" or "postsyn" + synapse_x : int unsigned # x coordinate of synapse centroid in EM voxels (x: 4nm, y: 4nm, z: 40nm). From Allen 'ctr_pt_position'. + synapse_y : int unsigned # y coordinate of centroid in EM voxels (x: 4nm, y: 4nm, z: 40nm). From Allen 'ctr_pt_position'. + synapse_z : int unsigned # z coordinate of centroid in EM voxels (x: 4nm, y: 4nm, z: 40nm). From Allen 'ctr_pt_position'. + synapse_size : int unsigned # (EM voxels) scaled by (4x4x40) + ts_inserted=CURRENT_TIMESTAMP : timestamp # timestamp that data was inserted into this DataJoint table. + """ + + +@schema +class AllenV1ColumnTypesSlanted(dj.Manual): + definition = """ + # ID's of cells from the table 'allen_v1_column_types_slanted' + ->Materialization + ->Nucleus + segment_id : bigint unsigned # id of the segment under the nucleus centroid. Equivalent to Allen 'pt_root_id'. + --- + valid : tinyint # 1 = valid entry, 0 = invalid entry + classification_system : varchar(128) # method for classification + cell_type : varchar(128) # manually classified cell type + n_nuc : smallint # number of nuclei associated with segment_id (single somas n_nuc = 1) + centroid_x : int unsigned # x coordinate of centroid in EM voxels (x: 4nm, y: 4nm, z: 40nm) + centroid_y : int unsigned # y coordinate of centroid in EM voxels (x: 4nm, y: 4nm, z: 40nm) + centroid_z : int unsigned # z coordinate of centroid in EM voxels (x: 4nm, y: 4nm, z: 40nm) + supervoxel_id : bigint unsigned # id of the supervoxel under the nucleus centroid. Equivalent to Allen: 'pt_supervoxel_id'. + """ + + class Meta(dj.Part): + definition = """ + ->Materialization + --- + description=NULL : varchar(256) # description of the table version + ts_inserted=CURRENT_TIMESTAMP : timestamp # timestamp that data was inserted into this DataJoint table. + """ + + +@schema +class AllenSomaCourseCellClassModelV1(dj.Manual): + definition = """ + # ID's of cells from the table 'allen_soma_coarse_cell_class_model_v1' + ->Materialization + ->Nucleus + segment_id : bigint unsigned # id of the segment under the nucleus centroid. Equivalent to Allen 'pt_root_id'. + --- + valid : tinyint # 1 = valid entry, 0 = invalid entry + classification_system : varchar(128) # method for classification + cell_type : varchar(128) # manually classified cell type + n_nuc : smallint # number of nuclei associated with segment_id (single somas n_nuc = 1) + centroid_x : int unsigned # x coordinate of centroid in EM voxels (x: 4nm, y: 4nm, z: 40nm) + centroid_y : int unsigned # y coordinate of centroid in EM voxels (x: 4nm, y: 4nm, z: 40nm) + centroid_z : int unsigned # z coordinate of centroid in EM voxels (x: 4nm, y: 4nm, z: 40nm) + supervoxel_id : bigint unsigned # id of the supervoxel under the nucleus centroid. Equivalent to Allen: 'pt_supervoxel_id'. + """ + + class Meta(dj.Part): + definition = """ + ->Materialization + --- + description=NULL : varchar(256) # description of the table version + ts_inserted=CURRENT_TIMESTAMP : timestamp # timestamp that data was inserted into this DataJoint table. + """ + + +@schema +class AllenSomaCourseCellClassModelV2(dj.Manual): + definition = """ + # ID's of cells from the table 'allen_soma_coarse_cell_class_model_v2' + ->Materialization + ->Nucleus + segment_id : bigint unsigned # id of the segment under the nucleus centroid. Equivalent to Allen 'pt_root_id'. + --- + valid : tinyint # 1 = valid entry, 0 = invalid entry + classification_system : varchar(128) # method for classification + cell_type : varchar(128) # manually classified cell type + n_nuc : smallint # number of nuclei associated with segment_id (single somas n_nuc = 1) + centroid_x : int unsigned # x coordinate of centroid in EM voxels (x: 4nm, y: 4nm, z: 40nm) + centroid_y : int unsigned # y coordinate of centroid in EM voxels (x: 4nm, y: 4nm, z: 40nm) + centroid_z : int unsigned # z coordinate of centroid in EM voxels (x: 4nm, y: 4nm, z: 40nm) + supervoxel_id : bigint unsigned # id of the supervoxel under the nucleus centroid. Equivalent to Allen: 'pt_supervoxel_id'. + """ + + class Meta(dj.Part): + definition = """ + ->Materialization + --- + description=NULL : varchar(256) # description of the table version + ts_inserted=CURRENT_TIMESTAMP : timestamp # timestamp that data was inserted into this DataJoint table. + """ \ No newline at end of file From 4ee1c814fcd0d5b106a9439e8a69df0242110d30 Mon Sep 17 00:00:00 2001 From: spapa013 Date: Tue, 21 Dec 2021 14:51:24 -0600 Subject: [PATCH 10/12] add h01_materialization to api.schemas and change import to reflect new structure --- .../schemas/h01_materialization.py | 12 ++++++++++++ .../h01_materialization/h01_materialization.py | 11 +---------- 2 files changed, 13 insertions(+), 10 deletions(-) create mode 100644 python/microns-materialization-api/microns_materialization_api/schemas/h01_materialization.py diff --git a/python/microns-materialization-api/microns_materialization_api/schemas/h01_materialization.py b/python/microns-materialization-api/microns_materialization_api/schemas/h01_materialization.py new file mode 100644 index 0000000..05c5402 --- /dev/null +++ b/python/microns-materialization-api/microns_materialization_api/schemas/h01_materialization.py @@ -0,0 +1,12 @@ +""" +DataJoint tables for importing h01 from Jeff Lichtman group. +""" +import datajoint as dj +import datajoint.datajoint_plus as djp + +from ..config import h01_materialization_config + +h01_materialization_config.register_externals() +h01_materialization_config.register_adapters(context=locals()) + +schema = dj.schema(h01_materialization_config.schema_name, create_schema=True) diff --git a/python/microns-materialization/microns_materialization/h01_materialization/h01_materialization.py b/python/microns-materialization/microns_materialization/h01_materialization/h01_materialization.py index 5930220..8d68bf2 100644 --- a/python/microns-materialization/microns_materialization/h01_materialization/h01_materialization.py +++ b/python/microns-materialization/microns_materialization/h01_materialization/h01_materialization.py @@ -16,13 +16,4 @@ else: from tqdm import tqdm -from microns_materialization_api import config -schema_obj = config.SCHEMAS.H01_MATERIALIZATION - -config.register_adapters(schema_obj, context=locals()) -config.register_externals(schema_obj) - -# Schema creation -schema = dj.schema(schema_obj.value) -schema.spawn_missing_classes() -schema.connection.dependencies.load() \ No newline at end of file +from microns_materialization_api.schemas import h01_materialization as h01mat \ No newline at end of file From d6a27a48e08f7cf6e202efb682a9190939279a9d Mon Sep 17 00:00:00 2001 From: spapa013 Date: Tue, 21 Dec 2021 14:51:33 -0600 Subject: [PATCH 11/12] update version --- python/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/version.py b/python/version.py index d85b28e..7d53ea3 100644 --- a/python/version.py +++ b/python/version.py @@ -1 +1 @@ -__version__ = "0.0.6" \ No newline at end of file +__version__ = "0.0.7" \ No newline at end of file From 161f3c0e3041094fa268b9086303803409b10dce Mon Sep 17 00:00:00 2001 From: spapa013 Date: Tue, 21 Dec 2021 14:55:35 -0600 Subject: [PATCH 12/12] add spawn_missing_classes and load_dependencies --- .../schemas/h01_materialization.py | 3 +++ .../schemas/minnie65_materialization.py | 6 +++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/python/microns-materialization-api/microns_materialization_api/schemas/h01_materialization.py b/python/microns-materialization-api/microns_materialization_api/schemas/h01_materialization.py index 05c5402..6b5553e 100644 --- a/python/microns-materialization-api/microns_materialization_api/schemas/h01_materialization.py +++ b/python/microns-materialization-api/microns_materialization_api/schemas/h01_materialization.py @@ -10,3 +10,6 @@ h01_materialization_config.register_adapters(context=locals()) schema = dj.schema(h01_materialization_config.schema_name, create_schema=True) + +schema.spawn_missing_classes() +schema.connection.dependencies.load() \ No newline at end of file diff --git a/python/microns-materialization-api/microns_materialization_api/schemas/minnie65_materialization.py b/python/microns-materialization-api/microns_materialization_api/schemas/minnie65_materialization.py index 7338cb8..3615b94 100644 --- a/python/microns-materialization-api/microns_materialization_api/schemas/minnie65_materialization.py +++ b/python/microns-materialization-api/microns_materialization_api/schemas/minnie65_materialization.py @@ -250,4 +250,8 @@ class Meta(dj.Part): --- description=NULL : varchar(256) # description of the table version ts_inserted=CURRENT_TIMESTAMP : timestamp # timestamp that data was inserted into this DataJoint table. - """ \ No newline at end of file + """ + + +schema.spawn_missing_classes() +schema.connection.dependencies.load() \ No newline at end of file