NWB revisions - PR 2 of 3

element_array_ephys/ephys_acute.py

@@ -4,6 +4,7 @@
 import numpy as np
 import inspect
 import importlib
+import gc
 from decimal import Decimal
 
 from element_interface.utils import find_root_directory, find_full_path, dict_to_uuid
@@ -295,6 +296,9 @@ def make(self, key):
                 raise FileNotFoundError(
                     'No Open Ephys data found for probe insertion: {}'.format(key))
 
+            if not probe_data.ap_meta:
+                raise IOError('No analog signals found - check "structure.oebin" file or "continuous" directory')
+
             if probe_data.probe_model in supported_probe_types:
                 probe_type = probe_data.probe_model
                 electrode_query = probe.ProbeType.Electrode & {'probe_type': probe_type}
@@ -323,6 +327,10 @@ def make(self, key):
             self.EphysFile.insert([{**key,
                                     'file_path': fp.relative_to(root_dir).as_posix()}
                                    for fp in probe_data.recording_info['recording_files']])
+            # explicitly garbage collect "dataset"
+            # as these may have large memory footprint and may not be cleared fast enough
+            del probe_data, dataset
+            gc.collect()
         else:
             raise NotImplementedError(f'Processing ephys files from'
                                       f' acquisition software of type {acq_software} is'
@@ -621,7 +629,7 @@ def make(self, key):
                             ks_output_dir=kilosort_dir,
                             params=params,
                             KS2ver=f'{Decimal(clustering_method.replace("kilosort", "")):.1f}',
-                            run_CatGT=False)
+                            run_CatGT=True)
                         run_kilosort.run_modules()
                 elif acq_software == 'Open Ephys':
                     oe_probe = get_openephys_probe_data(key)
@@ -882,9 +890,14 @@ def yield_unit_waveforms():
 
 def get_spikeglx_meta_filepath(ephys_recording_key):
     # attempt to retrieve from EphysRecording.EphysFile
-    spikeglx_meta_filepath = (EphysRecording.EphysFile & ephys_recording_key
-                              & 'file_path LIKE "%.ap.meta"').fetch1('file_path')
-
+    spikeglx_meta_filepath = pathlib.Path(
+        (
+            EphysRecording.EphysFile
+            & ephys_recording_key
+            & 'file_path LIKE "%.ap.meta"'
+        ).fetch1("file_path")
+    )
+
     try:
         spikeglx_meta_filepath = find_full_path(get_ephys_root_data_dir(),
                                                 spikeglx_meta_filepath)
@@ -916,7 +929,14 @@ def get_openephys_probe_data(ephys_recording_key):
     session_dir = find_full_path(get_ephys_root_data_dir(),
                               get_session_directory(ephys_recording_key))
     loaded_oe = openephys.OpenEphys(session_dir)
-    return loaded_oe.probes[inserted_probe_serial_number]
+    probe_data = loaded_oe.probes[inserted_probe_serial_number]
+
+    # explicitly garbage collect "loaded_oe"
+    # as these may have large memory footprint and may not be cleared fast enough
+    del loaded_oe
+    gc.collect()
+
+    return probe_data
 
 
 def get_neuropixels_channel2electrode_map(ephys_recording_key, acq_software):

element_array_ephys/ephys_chronic.py

@@ -4,6 +4,7 @@
 import numpy as np
 import inspect
 import importlib
+import gc
 from decimal import Decimal
 
 from element_interface.utils import find_root_directory, find_full_path, dict_to_uuid
@@ -242,6 +243,9 @@ def make(self, key):
                 raise FileNotFoundError(
                     'No Open Ephys data found for probe insertion: {}'.format(key))
 
+            if not probe_data.ap_meta:
+                raise IOError('No analog signals found - check "structure.oebin" file or "continuous" directory')
+
             if probe_data.probe_model in supported_probe_types:
                 probe_type = probe_data.probe_model
                 electrode_query = probe.ProbeType.Electrode & {'probe_type': probe_type}
@@ -270,6 +274,10 @@ def make(self, key):
             self.EphysFile.insert([{**key,
                                     'file_path': fp.relative_to(root_dir).as_posix()}
                                    for fp in probe_data.recording_info['recording_files']])
+            # explicitly garbage collect "dataset"
+            # as these may have large memory footprint and may not be cleared fast enough
+            del probe_data, dataset
+            gc.collect()
         else:
             raise NotImplementedError(f'Processing ephys files from'
                                       f' acquisition software of type {acq_software} is'
@@ -568,7 +576,7 @@ def make(self, key):
                             ks_output_dir=kilosort_dir,
                             params=params,
                             KS2ver=f'{Decimal(clustering_method.replace("kilosort", "")):.1f}',
-                            run_CatGT=False)
+                            run_CatGT=True)
                         run_kilosort.run_modules()
                 elif acq_software == 'Open Ephys':
                     oe_probe = get_openephys_probe_data(key)
@@ -827,8 +835,13 @@ def yield_unit_waveforms():
 
 def get_spikeglx_meta_filepath(ephys_recording_key):
     # attempt to retrieve from EphysRecording.EphysFile
-    spikeglx_meta_filepath = (EphysRecording.EphysFile & ephys_recording_key
-                              & 'file_path LIKE "%.ap.meta"').fetch1('file_path')
+    spikeglx_meta_filepath = pathlib.Path(
+        (
+            EphysRecording.EphysFile
+            & ephys_recording_key
+            & 'file_path LIKE "%.ap.meta"'
+        ).fetch1("file_path")
+    )
 
     try:
         spikeglx_meta_filepath = find_full_path(get_ephys_root_data_dir(),
@@ -859,10 +872,17 @@ def get_spikeglx_meta_filepath(ephys_recording_key):
 def get_openephys_probe_data(ephys_recording_key):
     inserted_probe_serial_number = (ProbeInsertion * probe.Probe
                                     & ephys_recording_key).fetch1('probe')
-    sess_dir = find_full_path(get_ephys_root_data_dir(),
+    session_dir = find_full_path(get_ephys_root_data_dir(),
                               get_session_directory(ephys_recording_key))
-    loaded_oe = openephys.OpenEphys(sess_dir)
-    return loaded_oe.probes[inserted_probe_serial_number]
+    loaded_oe = openephys.OpenEphys(session_dir)
+    probe_data = loaded_oe.probes[inserted_probe_serial_number]
+
+    # explicitly garbage collect "loaded_oe"
+    # as these may have large memory footprint and may not be cleared fast enough
+    del loaded_oe
+    gc.collect()
+
+    return probe_data
 
 
 def get_neuropixels_channel2electrode_map(ephys_recording_key, acq_software):

element_array_ephys/ephys_no_curation.py

@@ -4,6 +4,7 @@
 import numpy as np
 import inspect
 import importlib
+import gc
 from decimal import Decimal
 
 from element_interface.utils import find_root_directory, find_full_path, dict_to_uuid
@@ -293,6 +294,9 @@ def make(self, key):
                 raise FileNotFoundError(
                     'No Open Ephys data found for probe insertion: {}'.format(key))
 
+            if not probe_data.ap_meta:
+                raise IOError('No analog signals found - check "structure.oebin" file or "continuous" directory')
+
             if probe_data.probe_model in supported_probe_types:
                 probe_type = probe_data.probe_model
                 electrode_query = probe.ProbeType.Electrode & {'probe_type': probe_type}
@@ -322,6 +326,10 @@ def make(self, key):
             self.EphysFile.insert([{**key,
                                     'file_path': fp.relative_to(root_dir).as_posix()}
                                    for fp in probe_data.recording_info['recording_files']])
+            # explicitly garbage collect "dataset"
+            # as these may have large memory footprint and may not be cleared fast enough
+            del probe_data, dataset
+            gc.collect()
         else:
             raise NotImplementedError(f'Processing ephys files from'
                                       f' acquisition software of type {acq_software} is'
@@ -620,7 +628,7 @@ def make(self, key):
                             ks_output_dir=kilosort_dir,
                             params=params,
                             KS2ver=f'{Decimal(clustering_method.replace("kilosort", "")):.1f}',
-                            run_CatGT=False)
+                            run_CatGT=True)
                         run_kilosort.run_modules()
                 elif acq_software == 'Open Ephys':
                     oe_probe = get_openephys_probe_data(key)
@@ -841,8 +849,13 @@ def yield_unit_waveforms():
 
 def get_spikeglx_meta_filepath(ephys_recording_key):
     # attempt to retrieve from EphysRecording.EphysFile
-    spikeglx_meta_filepath = (EphysRecording.EphysFile & ephys_recording_key
-                              & 'file_path LIKE "%.ap.meta"').fetch1('file_path')
+    spikeglx_meta_filepath = pathlib.Path(
+        (
+            EphysRecording.EphysFile
+            & ephys_recording_key
+            & 'file_path LIKE "%.ap.meta"'
+        ).fetch1("file_path")
+    )
 
     try:
         spikeglx_meta_filepath = find_full_path(get_ephys_root_data_dir(),
@@ -874,7 +887,14 @@ def get_openephys_probe_data(ephys_recording_key):
     session_dir = find_full_path(get_ephys_root_data_dir(),
                               get_session_directory(ephys_recording_key))
     loaded_oe = openephys.OpenEphys(session_dir)
-    return loaded_oe.probes[inserted_probe_serial_number]
+    probe_data = loaded_oe.probes[inserted_probe_serial_number]
+
+    # explicitly garbage collect "loaded_oe"
+    # as these may have large memory footprint and may not be cleared fast enough
+    del loaded_oe
+    gc.collect()
+
+    return probe_data
 
 
 def get_neuropixels_channel2electrode_map(ephys_recording_key, acq_software):