Merge pull request #15 from spapa013/main

v0.0.10 - microns_materialization_v3

Dockerfile

@@ -3,6 +3,7 @@ LABEL maintainer="Stelios Papadopoulos <[email protected]>"
 
 RUN pip3 install \
         meshparty \
+        pcg-skel \
         cloud-volume \
         analysisdatalink\
         caveclient \
@@ -14,5 +15,5 @@ RUN mkdir -p .cloudvolume/secrets
 RUN echo "{\"token\": \"${CLOUDVOLUME_TOKEN:-}\"}" > .cloudvolume/secrets/cave-secret.json
 
 COPY . /src/microns-materialization
-RUN pip3 install -e /src/microns-materialization/python/microns-materialization
-RUN pip3 install -e /src/microns-materialization/python/microns-materialization-api
+RUN pip3 install --prefix=$(python -m site --user-base) -e /src/microns-materialization/python/microns-materialization
+RUN pip3 install --prefix=$(python -m site --user-base) -e /src/microns-materialization/python/microns-materialization-api

deploy/docker-compose.yml

@@ -15,6 +15,7 @@ x-shared: &common
     - /mnt:/mnt
   env_file:
     - .env
+  container_name: "microns-materialization"
 
 services:
   notebook:

deploy/kubernetes/minnie65_download_materialization.py

@@ -0,0 +1,6 @@
+import os
+
+if __name__ == 'main':
+    from microns_materialization.minnie_materialization.minnie65_materialization import download_materialization
+    download_materialization(ver=int(os.getenv('MICRONS_MAT_VER_TO_DL')), download_meshes=True, download_synapses=True, loglevel=os.getenv('MICRONS_LOGLEVEL'))
+

deploy/kubernetes/minnie65_download_meshes.py

@@ -0,0 +1,5 @@
+import os
+
+if __name__ == 'main':
+    from microns_materialization.minnie_materialization.minnie65_materialization import download_materialization
+    download_materialization(ver=int(os.getenv('MICRONS_MAT_VER_TO_DL')), download_meshes=True, download_synapses=False, loglevel=os.getenv('MICRONS_LOGLEVEL'))

deploy/kubernetes/minnie65_download_meshwork_objs.py

@@ -0,0 +1,5 @@
+import os
+
+if __name__ == 'main':
+    from microns_materialization.minnie_materialization.minnie65_materialization import Queue, download_meshwork_objects
+    download_meshwork_objects(Queue.PCGMeshwork, loglevel=os.getenv('MICRONS_LOGLEVEL'))

deploy/kubernetes/minnie65_download_pcg_skeletons.py

@@ -0,0 +1,5 @@
+import os
+
+if __name__ == 'main':
+    from microns_materialization.minnie_materialization.minnie65_materialization import Queue, download_pcg_skeletons
+    download_pcg_skeletons(Queue.PCGSkeleton, loglevel=os.getenv('MICRONS_LOGLEVEL'))

deploy/kubernetes/minnie65_download_synapses.py

@@ -0,0 +1,5 @@
+import os
+
+if __name__ == 'main':
+    from microns_materialization.minnie_materialization.minnie65_materialization import download_materialization
+    download_materialization(ver=int(os.getenv('MICRONS_MAT_VER_TO_DL')), download_meshes=False, download_synapses=True, loglevel=os.getenv('MICRONS_LOGLEVEL'))

python/microns-materialization-api/microns_materialization_api/config/__init__.py

@@ -17,7 +17,7 @@
 
 minnie65_materialization_config = SchemaConfig(
     module_name='minnie65_materialization',
-    schema_name='microns_minnie65_materialization',
+    schema_name='microns_minnie65_materialization_v3',
     externals=externals.minnie65_materialization,
     adapters=adapters.minnie65_materialization
 )

python/microns-materialization-api/microns_materialization_api/config/adapters.py

@@ -2,54 +2,53 @@
 Adapters for DataJoint tables.
 """
 
-import datajoint as dj
-import numpy as np
+import json
+
 import h5py
-import os
 import trimesh
-
-from collections import namedtuple
+from meshparty import meshwork, skeleton
+from microns_utils.adapter_utils import FilePathAdapter, adapt_mesh_hdf5
 
 
-class MeshAdapter(dj.AttributeAdapter):
-    # Initialize the correct attribute type (allows for use with multiple stores)
-    def __init__(self, attribute_type):
-        self.attribute_type = attribute_type
-        super().__init__()
-
-    attribute_type = '' # this is how the attribute will be declared
+class TrimeshAdapter(FilePathAdapter):
+    def get(self, filepath):
+        filepath = super().get(filepath)
+        mesh = adapt_mesh_hdf5(filepath, parse_filepath_stem=False, return_type='namedtuple')
+        return trimesh.Trimesh(vertices=mesh.vertices, faces=mesh.faces)
 
-    TriangularMesh = namedtuple('TriangularMesh', ['segment_id', 'vertices', 'faces'])
-
-    def put(self, filepath):
-        # save the filepath to the mesh
-        filepath = os.path.abspath(filepath)
-        assert os.path.exists(filepath)
-        return filepath
 
+class MeshworkAdapter(FilePathAdapter):
     def get(self, filepath):
-        # access the h5 file and return a mesh
-        assert os.path.exists(filepath)
+        filepath = super().get(filepath)
+        return meshwork.load_meshwork(filepath)
 
-        with h5py.File(filepath, 'r') as hf:
-            vertices = hf['vertices'][()].astype(np.float64)
-            faces = hf['faces'][()].reshape(-1, 3).astype(np.uint32)
-
-        segment_id = os.path.splitext(os.path.basename(filepath))[0]
-
-        return trimesh.Trimesh(vertices = vertices,faces=faces)
 
+class PCGSkelAdapter(FilePathAdapter):
+    def get(self, filepath):
+        filepath = super().get(filepath)
+        with h5py.File(filepath, 'r') as f:
+            vertices = f['vertices'][()]
+            edges = f['edges'][()]
+            mesh_to_skel_map = f['mesh_to_skel_map'][()]
+            root = f['root'][()]
+            meta = json.loads(f['meta'][()])
+            skel = skeleton.Skeleton(vertices=vertices, edges=edges, mesh_to_skel_map=mesh_to_skel_map, root=root, meta=meta)
+        return skel
+
+# M65
+minnie65_meshes = TrimeshAdapter('filepath@minnie65_meshes')
+minnie65_meshwork = MeshworkAdapter('filepath@minnie65_meshwork')
+minnie65_pcg_skeletons = PCGSkelAdapter('filepath@minnie65_pcg_skeletons')
 
-# instantiate for use as a datajoint type
-h01_meshes = MeshAdapter('filepath@h01_meshes')
-minnie65_meshes = MeshAdapter('filepath@minnie65_meshes')
+minnie65_materialization = {
+    'minnie65_meshes': minnie65_meshes,
+    'minnie65_meshwork': minnie65_meshwork,
+    'minnie65_pcg_skeletons': minnie65_pcg_skeletons
+}
 
-# also store in one object for ease of use with virtual modules
+# H01
+h01_meshes = TrimeshAdapter('filepath@h01_meshes')
 
 h01_materialization = {
     'h01_meshes': h01_meshes,
 }
-
-minnie65_materialization = {
-    'minnie65_meshes': minnie65_meshes,
-}

python/microns-materialization-api/microns_materialization_api/config/externals.py

@@ -1,19 +1,26 @@
 """
 Externals for DataJoint tables.
 """
-import datajoint.datajoint_plus as djp
 from pathlib import Path
 
+import datajoint.datajoint_plus as djp
+
 base_path = Path() / '/mnt' / 'dj-stor01' / 'microns'
 
 #h01 materialization
 h01_materialization_external_meshes_path = base_path / 'h01' / 'meshes'
+
 h01_materialization = {
     'h01_meshes': djp.make_store_dict(h01_materialization_external_meshes_path),
 }
 
 #minnie65_materialization
-minnie65_materialization_external_meshes_path = base_path / 'minnie' / 'meshes'
+minnie65_materialization_external_meshes_path = base_path / 'minnie65' / 'meshes'
+minnie65_materialization_external_meshwork_path = base_path / 'minnie65' / 'meshwork'
+minnie65_materialization_external_pcg_skeletons_path = base_path / 'minnie65' / 'pcg_skeletons'
+
 minnie65_materialization = {
     'minnie65_meshes': djp.make_store_dict(minnie65_materialization_external_meshes_path),
+    'minnie65_meshwork': djp.make_store_dict(minnie65_materialization_external_meshwork_path),
+    'minnie65_pcg_skeletons': djp.make_store_dict(minnie65_materialization_external_pcg_skeletons_path),
 }