Merge pull request #3 from spapa013/main

v0.0.3

Dockerfile

@@ -1,4 +1,4 @@
-FROM at-docker:5000/microns-base:cuda11.4.0-python3.8
+FROM at-docker:5000/microns-base:cuda11.8.0-python3.8
 
 LABEL mantainer="Stelios Papadopoulos <[email protected]>"
 
@@ -14,5 +14,5 @@ RUN echo "{\"token\": \"${CLOUDVOLUME_TOKEN:-}\"}" > .cloudvolume/secrets/cave-s
 
 # CURRENT PACKAGE
 COPY . /src/microns-proximities
-RUN python -m pip install --no-cache-dir /src/microns-proximities/python/microns-proximities
-RUN python -m pip install --no-cache-dir /src/microns-proximities/python/microns-proximities-api
+RUN python -m pip install -e /src/microns-proximities/python/microns-proximities
+RUN python -m pip install -e /src/microns-proximities/python/microns-proximities-api

deploy/kubernetes/minnie65_compute_proximities.py

@@ -1,4 +1,4 @@
 if __name__ == '__main__':
     from microns_proximities.minnie_proximities import minnie_proximities as mp
-    mp.ProximityMaker.SkeletonProcessedSetChunk.populate({'skeleton_prc_set_chunk': '8fe31e81'}, reserve_jobs=True, order='random')
+    mp.ProximityMaker.SkeletonProcessedSetChunk.populate(reserve_jobs=True, order='random', suppress_errors=True)
 

python/microns-proximities-api/microns_proximities_api/config/adapters.py

@@ -2,5 +2,5 @@
 
 minnie_proximities = {
     'minnie65_proximity_skeletons': NumpyAdapter('filepath@minnie65_proximity_skeletons'),
-    'minnie65_proximity_results': NumpyAdapter('filepath@minnie65_proximity_results'),
+    'minnie65_proximities': NumpyAdapter('filepath@minnie65_proximities'),
 }

python/microns-proximities-api/microns_proximities_api/config/externals.py

@@ -6,8 +6,9 @@
 import datajoint_plus as djp
 
 base_path = Path() / "/mnt" / "dj-stor01" / "microns" / "minnie" / "proximities"
+base_path2 = Path() / "/mnt" / "scratch09" / "microns" / "minnie" / "proximities"
 
 minnie_proximities = {
     "minnie65_proximity_skeletons": djp.make_store_dict(base_path / "minnie65_proximity_skeletons"),
-    "minnie65_proximity_results": djp.make_store_dict(base_path / "minnie65_proximity_results"),
+    "minnie65_proximities": djp.make_store_dict(base_path2 / "minnie65_proximities"),
 }

python/microns-proximities-api/microns_proximities_api/schemas/minnie_proximities.py

@@ -1,10 +1,12 @@
 """
 DataJoint tables for proximities.
 """
+from pathlib import Path
 import datajoint as dj
 import datajoint_plus as djp
 
 import microns_utils.datajoint_utils as dju
+from microns_utils.misc_utils import classproperty
 
 from microns_manual_proofreading_api.schemas import minnie65_manual_proofreading as m65manprf
 from microns_materialization_api.schemas import minnie65_materialization as m65mat
@@ -345,7 +347,7 @@ class SkeletonProcessedSetChunk(djp.Part):
 
 
 @schema
-class Proximity(djp.Lookup):
+class Proximity2(djp.Lookup):
     enable_hashing = True
     hash_name = 'prx_id'
     hashed_attrs = ProximityMethod.hash_name, 'skeleton_prc_id_axon', 'skeleton_prc_id_dend'
@@ -357,12 +359,55 @@ class Proximity(djp.Lookup):
     ---
     axon_len :                  float                                               # the skeletal length of the axon in the proximity
     dend_len :                  float                                               # the skeletal length of the dendrite in the proximity
-    data :                      <minnie65_proximity_results>                        # data products computed during proximity
-    compute_time :              float # time to compute (seconds)
+    data :                      <minnie65_proximities>                        # data products computed during proximity
+    prx_chunk_hash : varchar(6) # hash of proximity chunk from ProximityMaker
     """
+    @classproperty
+    def base_path(cls):
+        return Path(config.externals['minnie65_proximities']['location'])
 
+    @classmethod
+    def make_filepath(cls, proximity_method, skeleton_prc_id_axon, skeleton_prc_id_dend, prx_id, suffix='.npz', **kwargs):
+        return cls.base_path.joinpath(f'{proximity_method}_{skeleton_prc_id_axon}_{skeleton_prc_id_dend}_{prx_id}_proximity').with_suffix(suffix)
+
     @classmethod
     def restrict_with(cls, axon_source, dend_source, axon_key=None, dend_key=None, skeleton_prc_set='4c396d17', proximity_method='0ab4bc'):
+        """
+        Restrict the Proximity table with additional filtering criteria.
+
+        Parameters:
+        ----------
+        axon_source : str
+            Source of the axon data, can be either 'manual' or 'auto'. If 'manual', it relates with
+            the `SkeletonProcessed.MeshworkAxonDendrite`, and if 'auto', it relates with 
+            `SkeletonProcessed.AutoProofreadNeuron`.
+
+        dend_source : str
+            Source of the dendrite data, similar to `axon_source`.
+
+        axon_key : dict, optional
+            Filtering relation for axon data. Defaults to an empty dictionary if None.
+
+        dend_key : dict, optional
+            Filtering relation for dendrite data. Defaults to an empty dictionary if None.
+
+        skeleton_prc_set : str, optional
+            Hash for the skeleton processed set relation `SkeletonProcessedSet.r1pwh()`. 
+            Defaults to '4c396d17'.
+
+        proximity_method : str, optional
+            Proximity method hash. Defaults to '0ab4bc'.
+
+        Returns:
+        -------
+        DataJoint relation
+            Restricted relation based on provided criteria and sources for axon and dendrite data.
+
+        Raises:
+        ------
+        AttributeError
+            If the axon or dendrite source is neither 'manual' nor 'auto'.
+        """
         def resolve_source(source):
             if source == 'manual':
                 source_rel = SkeletonProcessed.MeshworkAxonDendrite
@@ -389,6 +434,22 @@ def resolve_source(source):
 
         return prx_rel * axon_rel * dend_rel
 
+    @classmethod
+    def make_filepaths_from_rel(cls, rel):
+        """
+        Make filepaths from a DataJoint relation. Relation must contain the 'data' attribute containing the filepath hash.
+        """
+        fns = (rel.proj(hash='data') * schema.external['minnie65_proximities']).fetch('filepath')
+        fps = []
+        for fn in fns:
+            fps.append(cls.base_path.joinpath(fn))
+        return fps
+
+
+class Proximity(Proximity2):
+    def __new__(cls):
+        return Proximity2()
+
 
 @schema
 class ProximityMaker(djp.Lookup):
@@ -405,6 +466,9 @@ class SkeletonProcessedSetChunk(djp.Part, dj.Computed):
         -> master
         -> ProximityKeySource.SkeletonProcessedSetChunk
         ---
-        duration :         float     # total time to finish chunk (seconds)
+        load_time : float       # time to load data (seconds)
+        compute_time : float    # time to compute (seconds)
+        save_time : float       # time to save data (seconds)
+        total_time : float      # total time (seconds)
         ts_inserted=CURRENT_TIMESTAMP : timestamp
         """

python/microns-proximities/microns_proximities/minnie_proximities/minnie_proximities.py

@@ -606,24 +606,24 @@ def fill(cls, proximity_method_hash, skeleton_prc_set_chunk_hash):
 
 
 class Proximity(mp.Proximity):
-    @classproperty
-    def base_path(cls):
-        return Path(mp.config.externals['minnie65_proximity_results']['location'])
+    pass
 
-    @classmethod
-    def make_filepath(cls, proximity_method, skeleton_prc_id_axon, skeleton_prc_id_dend, prx_id, suffix='.npz', **kwargs):
-        return cls.base_path.joinpath(f'{proximity_method}_{skeleton_prc_id_axon}_{skeleton_prc_id_dend}_{prx_id}_proximity').with_suffix(suffix)
+
+class Proximity2(mp.Proximity2):
+    pass
 
 
 class ProximityMaker(mp.ProximityMaker):
 
     class SkeletonProcessedSetChunk(mp.ProximityMaker.SkeletonProcessedSetChunk):
         @classproperty
         def key_source(cls):
-            return ProximityKeySource.SkeletonProcessedSetChunk & ['axon_chunk_id=0', 'dend_chunk_id=0'] # only chunks that contain the manually proofread skeletons
+            return ProximityKeySource.SkeletonProcessedSetChunk & {'prx_key_src': '15be8db2'} #& ['axon_chunk_id=0', 'dend_chunk_id=0', 'axon_chunk_id=1', 'dend_chunk_id=1'] # only chunks that contain the manually proofread skeletons
 
         def make(self, key):
-            t0 = time.time()
+            start_ts = time.time()
+            key['prx_chunk_hash'] = self.hash1(key)
+
             logger.info('--> LOAD SKELETONS')
             logger.info('Getting skeleton chunks...')
             axon_chunk_key = {
@@ -653,7 +653,10 @@ def make(self, key):
             axons = {k: np.load(v) for k, v in axon_fps.items()}
             dends = {k: np.load(v) for k, v in dend_fps.items()}
             logger.info('Skeletons loaded.')
-
+            load_ts = time.time()
+            load_time = np.round(load_ts - start_ts, decimals=3)
+            logger.info(f'Skeleton loading time: {load_time} seconds.')
+
             logger.info('--> COMPUTE PROXIMITIES')
             skeleton_pairs = list(product(axon_fps.keys(), dend_fps.keys()))
 
@@ -665,7 +668,6 @@ def make(self, key):
             # iterate
             results = []
             for sk_pair in tqdm(skeleton_pairs):
-                tc0 = time.time()
                 axon_id, dend_id = sk_pair
                 axon_verts = axons[axon_id]['vertices']
                 axon_edges = axons[axon_id]['edges']
@@ -687,25 +689,34 @@ def make(self, key):
                     result['proximity_method'] = proximity_method
                     result['skeleton_prc_id_axon'] = axon_id
                     result['skeleton_prc_id_dend'] = dend_id
-                    result['prx_id'] = Proximity.hash1(result)
+                    result['prx_id'] = Proximity2.hash1(result)
                     data['edges1_prx'] = sku.filter_edges(axon_edges, vertices_inds_subset=np.unique(data['verts1_inds_prx']))
                     data['edges2_prx'] = sku.filter_edges(dend_edges, vertices_inds_subset=np.unique(data['verts2_inds_prx']))
                     result['axon_len'] = sku.compute_skeletal_length(axon_verts, data['edges1_prx'])
                     result['dend_len'] = sku.compute_skeletal_length(dend_verts, data['edges2_prx'])
                     result['data'] = data
-                    result['compute_time'] = np.round(time.time() - tc0, decimals=3)
+                    result['prx_chunk_hash'] = key['prx_chunk_hash']
                     results.append(result)
             logger.info('Compute proximities completed.')
-
+            compute_ts = time.time()
+            compute_time = np.round(compute_ts - load_ts, decimals=3)
+            logger.info(f'Total compute time: {compute_time} seconds.')
+
             logger.info('--> SAVE FILEPATHS')
             for result in results:
-                fp = Proximity.make_filepath(**result)
-                np.savez(fp, **result['data'])
+                fp = Proximity2.make_filepath(**result)
+                np.savez_compressed(fp, **result['data'])
                 result['data'] = fp
             logger.info('Save completed.')
-
+            save_time = np.round(time.time() - compute_ts, decimals=3)
+            logger.info(f'Total save time: {save_time} seconds.')
+
             logger.info('--> INSERT TO DATAJOINT')
-            Proximity.insert(results, ignore_extra_fields=True, skip_hashing=True)
-            key['duration'] = np.round(time.time() - t0, decimals=3)
-            self.insert(key, insert_to_master=True)
+
+            key['load_time'] = load_time
+            key['compute_time'] = compute_time
+            key['save_time'] = save_time
+            key['total_time'] = np.round(time.time() - start_ts, decimals=3)
+            Proximity2.insert(results, ignore_extra_fields=True, skip_hashing=True, skip_duplicates=True)
+            self.insert1(key, insert_to_master=True, skip_hashing=True)
             logger.info('Insert completed.')

python/version.py

@@ -1 +1 @@
-__version__ = '0.0.2'
+__version__ = '0.0.3'