fix trialization

.gitignore

@@ -40,7 +40,7 @@ docs/source/gen_modules/*
 *.log
 
 # test output
-_test
+_test*
 
 # space for users to save custom scripts
 scripts/user/*

docs/source/nsds_lab_to_nwb/common.rst

@@ -11,6 +11,11 @@ Common API
    :undoc-members:
    :show-inheritance:
 
+.. automodule:: nsds_lab_to_nwb.common.rec_manager
+  :members:
+  :undoc-members:
+  :show-inheritance:
+
 .. automodule:: nsds_lab_to_nwb.common.time
   :members:
   :undoc-members:

docs/source/nsds_lab_to_nwb/components/stimulus.rst

@@ -21,11 +21,6 @@ components.stimulus API
    :undoc-members:
    :show-inheritance:
 
-.. automodule:: nsds_lab_to_nwb.components.stimulus.stim_value_extractor
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
 .. automodule:: nsds_lab_to_nwb.components.stimulus.tokenizers.base_tokenizer
    :members:
    :undoc-members:

nsds_lab_to_nwb/common/rec_manager.py

@@ -0,0 +1,54 @@
+import logging
+
+from nsds_lab_to_nwb.tools.htk.htk_reader import HTKReader
+from nsds_lab_to_nwb.tools.htk.readers.htkfile import HTKFile
+from nsds_lab_to_nwb.tools.tdt.tdt_reader import TDTReader
+
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.DEBUG)
+
+
+class RecManager():
+    def __init__(self, dataset):
+        self.dataset = dataset
+
+        if hasattr(self.dataset, 'htk_mark_path'):
+            logger.info('Using HTK')
+            self.rec_source = 'htk'
+            self.rec_reader = HTKReader(self.dataset.htk_path)
+        else:
+            logger.info('Using TDT')
+            self.rec_source = 'tdt'
+            self.rec_reader = TDTReader(self.dataset.tdt_path)
+
+    def read_info(self):
+        if self.rec_source == 'htk':
+            return None
+
+        return self.rec_reader.tdt_obj['info']
+
+    def read_neural_data(self, stream, dev_conf):
+        data, metadata = self.rec_reader.get_data(stream=stream, dev_conf=dev_conf)
+        return data, metadata
+
+    def read_marks(self):
+        # Read the mark track
+        if self.rec_source == 'htk':
+            mark_file = HTKFile(self.dataset.htk_mark_path)
+            mark_track, meta = mark_file.read_data()
+            rate = mark_file.sample_rate
+        else:
+            mark_track, meta = self.rec_reader.get_data(stream='mrk1')
+            rate = meta['sample_rate']
+        return mark_track, rate
+
+    def read_mark_events(self):
+        if self.rec_source == 'htk':
+            return None
+
+        # for tdt
+        try:
+            return self.rec_reader.get_events()
+        except AttributeError:
+            # there is no mark for baseline (no stimulus) block
+            return None

nsds_lab_to_nwb/components/neural_data/neural_data_originator.py

@@ -1,36 +1,27 @@
 import logging.config
 from pynwb.ecephys import ElectricalSeries
 
-from nsds_lab_to_nwb.tools.htk.htk_reader import HTKReader
-from nsds_lab_to_nwb.tools.tdt.tdt_reader import TDTReader
 from process_nwb.resample import resample
 
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.DEBUG)
 
 
 class NeuralDataOriginator():
-    def __init__(self, dataset, metadata, resample_flag=True):
-        self.dataset = dataset      # this should have all relavant paths
+    def __init__(self, rec_manager, metadata, resample_flag=True):
+        self.rec_manager = rec_manager
         self.metadata = metadata    # this should have all relevant metadata
         self.resample_flag = resample_flag
         self.hardware_rate = None
         self.resample_rate = None
 
-        if hasattr(self.dataset, 'htk_mark_path'):
-            logger.info('Using HTK')
-            self.neural_data_reader = HTKReader(self.dataset.htk_path)
-        else:
-            logger.info('Using TDT')
-            self.neural_data_reader = TDTReader(self.dataset.tdt_path)
-
     def make(self, nwb_content, electrode_table_regions):
         for device_name, dev_conf in self.metadata['device'].items():
             if isinstance(dev_conf, str): # skip other annotations
                 continue
 
             logger.info(f'Extracting {device_name} data...')
-            data, metadata = self.neural_data_reader.get_data(stream=device_name, dev_conf=dev_conf)
+            data, metadata = self.rec_manager.read_neural_data(stream=device_name, dev_conf=dev_conf)
             if data is None:
                 logger.info(f'No data available for {device_name}. Skipping...')
             else:

nsds_lab_to_nwb/components/stimulus/mark_manager.py

@@ -1,36 +1,44 @@
-from pynwb import TimeSeries
+import logging
 
-from nsds_lab_to_nwb.tools.htk.readers.htkfile import HTKFile
-from nsds_lab_to_nwb.tools.tdt.tdt_reader import TDTReader
+from nsds_lab_to_nwb.components.stimulus.utils import detect_events
+
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.DEBUG)
 
 
 class MarkManager():
-    def __init__(self, dataset):
-        self.dataset = dataset
-
-    def get_mark_track(self, starting_time, name='recorded_mark'):
-        # Read the mark track
-        if hasattr(self.dataset, 'htk_mark_path'):
-            mark_file = HTKFile(self.dataset.htk_mark_path)
-            mark_track, meta = mark_file.read_data()
-            rate = mark_file.sample_rate
-            mark_onsets = None
+    def __init__(self, rec_manager, stim_configs, use_tdt_mark_events=False):
+        self.rec_manager = rec_manager
+        self.stim_configs = stim_configs
+        self.use_tdt_mark_events = use_tdt_mark_events
+
+    def get_mark_track(self):
+        # read recorded mark tracks
+        mark_track, mark_rate = self.rec_manager.read_marks()
+
+        # detect marked event times
+        if self.rec_manager.rec_source == 'tdt' and self.use_tdt_mark_events:
+            mark_events = self.rec_manager.read_mark_events()
         else:
-            tdt_reader = TDTReader(self.dataset.tdt_path)
-            mark_track, meta = tdt_reader.get_data(stream='mrk1')
-            rate = meta['sample_rate']
-            try:
-                mark_onsets = tdt_reader.get_events()
-            except AttributeError:
-                # there is no mark for baseline (no stimulus) block
-                mark_onsets = None
-
-        # Create the mark timeseries
-        mark_time_series = TimeSeries(name=name,
-                                      data=mark_track,
-                                      unit='Volts',
-                                      starting_time=starting_time,
-                                      rate=rate,
-                                      description='Recorded mark that tracks stimulus onsets.')
-
-        return mark_time_series, mark_onsets
+            mark_events = None
+        mark_events = self.get_mark_events(mark_events, mark_track, mark_rate)
+
+        return mark_track, mark_rate, mark_events
+
+    def get_mark_events(self, mark_events_input, mark_data, mark_rate):
+        if mark_events_input is not None:
+            # loaded directly from TDT object
+            # (now suppressed by use_tdt_mark_events=False because wn2 requires re-detection)
+            logger.info('Using marker events directly loaded from TDT')
+            logger.debug(f'found {len(mark_events_input)} mark events')
+            return mark_events_input
+
+        logger.info('Detecting stimulus onsets by thresholding the mark track')
+        # note: the min_separation condition is probably not required,
+        # although it doesn't hurt to keep
+        stim_duration = self.stim_configs.get('duration', None)
+        mark_threshold = self.stim_configs['mark_threshold']
+        mark_events = detect_events(mark_data, mark_rate, mark_threshold,
+                                    min_separation=stim_duration)
+        logger.debug(f'found {len(mark_events)} mark events')
+        return mark_events