Clean up code and add info to change log/release notes.

Fixup changelogs/docs.
Fix linting.
Add code to catch active stimulus that should be marked but fall outside of the trials.
update presentations.pkl table.
Remove loading stimulus movie in sessions/experiments.
Remove load_stimulus_movie from tests.

CHANGELOG.md

@@ -1,6 +1,13 @@
 # Change Log
 All notable changes to this project will be documented in this file.
 
+## [2.16.0] = 2023-XX-XX
+- Python 3.10 and 3.11 support
+- Support for VBO update release.
+- Added new stimulus presentations columns that can be computed/added when 
+loading previous data.
+- Various bug fixes.
+
 ## [2.14.0] = 2022-12-12
 - Support for updated vbn release containing probe, lfp and behavior only data.
 - Updates to Ophys data in anticipation of a forthcoming updated data release

allensdk/brain_observatory/behavior/behavior_ophys_experiment.py

@@ -110,7 +110,6 @@ def from_lims(
         events_filter_scale_seconds: float = 2.0 / 31.0,
         events_filter_n_time_steps: int = 20,
         exclude_invalid_rois: bool = True,
-        load_stimulus_movie: bool = False
     ) -> "BehaviorOphysExperiment":
         """
         Parameters
@@ -127,9 +126,6 @@ def from_lims(
             See `BehaviorOphysExperiment.from_nwb`
         exclude_invalid_rois : bool
             Whether to exclude invalid rois
-        load_stimulus_movie : bool
-            Whether to load the stimulus movie (e.g natrual_movie_one) as
-            part of loading stimuli. Default True.
 
         Returns
         -------
@@ -188,7 +184,6 @@ def _get_motion_correction():
             date_of_acquisition=date_of_acquisition,
             eye_tracking_z_threshold=eye_tracking_z_threshold,
             eye_tracking_dilation_frames=eye_tracking_dilation_frames,
-            load_stimulus_movie=load_stimulus_movie
         )
         if is_multiplane_session:
             ophys_timestamps = OphysTimestampsMultiplane.from_sync_file(

allensdk/brain_observatory/behavior/behavior_session.py

@@ -358,7 +358,6 @@ def from_lims(
         date_of_acquisition: Optional[DateOfAcquisition] = None,
         eye_tracking_z_threshold: float = 3.0,
         eye_tracking_dilation_frames: int = 2,
-        load_stimulus_movie: bool = False,
     ) -> "BehaviorSession":
         """
 
@@ -384,11 +383,6 @@ def from_lims(
             See `BehaviorSession.from_nwb`, default 3.0
         eye_tracking_dilation_frames : int
             See `BehaviorSession.from_nwb`, default 2
-        load_stimulus_movie : bool
-            Whether to load the stimulus movie (e.g natrual_movie_one) as
-            part of loading stimuli. Default False. The warped+unwarped
-            movie loaded will be very large in memory/on disk so be
-            careful when requesting this option.
 
         Returns
         -------
@@ -456,7 +450,6 @@ def from_lims(
             project_code=ProjectCode.from_lims(
                 behavior_session_id=behavior_session_id.value, lims_db=lims_db
             ),
-            load_stimulus_movie=load_stimulus_movie,
         )
 
         if date_of_acquisition is None:
@@ -1153,34 +1146,6 @@ def stimulus_templates(self) -> Optional[pd.DataFrame]:
         else:
             return None
 
-    @property
-    def stimulus_natural_movie_template(self) -> Optional[pd.DataFrame]:
-        """Get stimulus templates movie for the behavior session.
-
-        Returns None if no stimulus movie is available.
-
-        Returns
-        -------
-        pd.DataFrame or None
-            A pandas DataFrame object containing the individual frames for the
-            movie shown during this experiment.
-
-            dataframe columns:
-                frame_number [index]: (int)
-                    Frame number in movie
-                unwarped: (array of int)
-                    image array of unwarped stimulus movie frame
-                warped: (array of int)
-                    image array of warped stimulus movie frame
-
-        """
-        if self._stimuli.templates.fingerprint_movie_template_key is not None:
-            return self._stimuli.templates.value[
-                self._stimuli.templates.fingerprint_movie_template_key
-            ].to_dataframe(index_name="frame_number", index_type="int")
-        else:
-            return None
-
     @property
     def stimulus_timestamps(self) -> np.ndarray:
         """Timestamps associated with the stimulus presetntation on
@@ -1415,7 +1380,6 @@ def _read_stimuli(
         trials: Trials,
         stimulus_presentation_columns: Optional[List[str]] = None,
         project_code: Optional[ProjectCode] = None,
-        load_stimulus_movie: bool = False,
     ) -> Stimuli:
         """
         Construct the Stimuli data object for this session
@@ -1434,7 +1398,6 @@ def _read_stimuli(
             presentation_columns=stimulus_presentation_columns,
             project_code=project_code,
             trials=trials,
-            load_stimulus_movie=load_stimulus_movie,
         )
 
     @classmethod
@@ -1514,7 +1477,6 @@ def _read_data_from_stimulus_file(
         include_stimuli: bool = True,
         stimulus_presentation_columns: Optional[List[str]] = None,
         project_code: Optional[ProjectCode] = None,
-        load_stimulus_movie: bool = False,
     ):
         """Helper method to read data from stimulus file"""
 
@@ -1551,7 +1513,6 @@ def _read_data_from_stimulus_file(
                 trials=trials,
                 stimulus_presentation_columns=stimulus_presentation_columns,
                 project_code=project_code,
-                load_stimulus_movie=load_stimulus_movie,
             )
         else:
             stimuli = None

allensdk/brain_observatory/behavior/stimulus_processing.py

@@ -696,6 +696,14 @@ def add_active_flag(
             & (~stim_pres_table.image_name.isna())
         )
         active[stim_mask] = True
+
+        # Clean up potential stimuli that fall outside in time of the trials
+        # but are part of the "active" stimulus block.
+        if "stimulus_block" in stim_pres_table.columns:
+            for stim_block in stim_pres_table["stimulus_block"].unique():
+                block_mask = stim_pres_table["stimulus_block"] == stim_block
+                if np.any(active[block_mask]):
+                    active[block_mask] = True
         stim_pres_table["active"] = active
         return stim_pres_table
 

allensdk/brain_observatory/nwb/nwb_api.py

@@ -1,5 +1,3 @@
-from pathlib import Path
-
 import pandas as pd
 import pynwb
 import SimpleITK as sitk
@@ -24,7 +22,8 @@ def nwbfile(self):
         return io.read()
 
     def __init__(self, path, **kwargs):
-        ''' Reads data for a single Brain Observatory session from an NWB 2.0 file
+        ''' Reads data for a single Brain Observatory session from an NWB 2.0
+        file
         '''
 
         self.path = path

allensdk/test/brain_observatory/behavior/conftest.py

@@ -127,6 +127,5 @@ def behavior_ophys_experiment_fixture():
 
     experiment_id = 953443028
     experiment = BehaviorOphysExperiment.from_lims(
-        experiment_id,
-        load_stimulus_movie=False)
+        experiment_id)
     return experiment

allensdk/test/brain_observatory/behavior/data_objects/test_stimuli.py

@@ -52,7 +52,7 @@ def data(self):
                 return pd.DataFrame(
                     {
                         "start_time": [300.0, 330.0, 360.0],
-                        "stop_time": [330.0, 360.0, 360.0],
+                        "stop_time": [330.0, 360.0, 390.0],
                         "catch": [False, True, False],
                         "change_frame": [-99, 99, -99],
                     }

allensdk/test/brain_observatory/behavior/test_behavior_ophys_experiment.py

@@ -33,7 +33,6 @@ def test_nwb_end_to_end(tmpdir_factory):
 
     d1 = BehaviorOphysExperiment.from_lims(
         oeid,
-        load_stimulus_movie=False
     )
     nwbfile = d1.to_nwb()
     with NWBHDF5IO(nwb_filepath, "w") as nwb_file_writer:
@@ -52,7 +51,6 @@ def test_visbeh_ophys_data_set():
     data_set = BehaviorOphysExperiment.from_lims(
         ophys_experiment_id,
         exclude_invalid_rois=False,
-        load_stimulus_movie=False
     )
 
     # TODO: need to improve testing here:
@@ -68,15 +66,13 @@ def test_visbeh_ophys_data_set():
     behavior_session_id = BehaviorSessionId.from_lims(
         db=lims_db,
         ophys_experiment_id=ophys_experiment_id,
-        load_stimulus_movie=False
     )
 
     # All sorts of assert relationships:
     assert (
         ForagingId.from_lims(
             behavior_session_id=behavior_session_id.value,
             lims_db=lims_db,
-            load_stimulus_movie=False
         ).value
         == data_set.metadata["behavior_session_uuid"]
     )
@@ -167,7 +163,6 @@ def test_legacy_dff_api():
     ophys_experiment_id = 792813858
     session = BehaviorOphysExperiment.from_lims(
         ophys_experiment_id=ophys_experiment_id,
-        load_stimulus_movie=False
     )
 
     _, dff_array = session.get_dff_traces()
@@ -187,7 +182,6 @@ def test_legacy_dff_api():
 def test_stimulus_presentations_omitted(ophys_experiment_id, number_omitted):
     session = BehaviorOphysExperiment.from_lims(
         ophys_experiment_id,
-        load_stimulus_movie=False
     )
     df = session.stimulus_presentations
     assert df["omitted"].sum() == number_omitted
@@ -198,7 +192,6 @@ def test_event_detection():
     ophys_experiment_id = 789359614
     session = BehaviorOphysExperiment.from_lims(
         ophys_experiment_id=ophys_experiment_id,
-        load_stimulus_movie=False
     )
     events = session.events
 
@@ -229,7 +222,6 @@ def test_BehaviorOphysExperiment_property_data():
     ophys_experiment_id = 960410026
     dataset = BehaviorOphysExperiment.from_lims(
         ophys_experiment_id,
-        load_stimulus_movie=False
     )
 
     assert dataset.ophys_session_id == 959458018
@@ -277,7 +269,6 @@ def test_behavior_ophys_experiment_list_data_attributes_and_methods(
         "segmentation_mask_image",
         "stimulus_presentations",
         "stimulus_templates",
-        'stimulus_natural_movie_template',
         "stimulus_timestamps",
         "task_parameters",
         "trials",

allensdk/test/brain_observatory/behavior/test_behavior_session.py

@@ -20,8 +20,7 @@ def test_nwb_end_to_end_session(
     tmpdir = pathlib.Path(tmpdir)
     nwb_path = tmpdir / f'session_{session_id}.nwb'
     session = BehaviorSession.from_lims(
-            behavior_session_id=session_id,
-            load_stimulus_movie=False)
+            behavior_session_id=session_id)
     nwb_file = session.to_nwb()
     with NWBHDF5IO(nwb_path, 'w') as nwb_file_writer:
         nwb_file_writer.write(nwb_file)
@@ -88,7 +87,6 @@ def dummy_init(self):
         'running_speed',
         'stimulus_presentations',
         'stimulus_templates',
-        'stimulus_natural_movie_template',
         'stimulus_timestamps',
         'task_parameters',
         'trials',
@@ -106,8 +104,7 @@ def test_behavior_session_equivalent_json_lims(session_data_fixture):
 
     behavior_session_id = session_data_fixture['behavior_session_id']
     lims_session = BehaviorSession.from_lims(
-        behavior_session_id,
-        load_stimulus_movie=False
+        behavior_session_id
     )
 
     assert sessions_are_equal(json_session, lims_session, reraise=True)
@@ -120,6 +117,5 @@ def test_eye_tracking_loaded_with_metadata_frame(self):
         sess_id = 1154034257
 
         sess = BehaviorSession.from_lims(behavior_session_id=sess_id,
-                                         lims_db=self.dbconn,
-                                         load_stimulus_movie=False)
+                                         lims_db=self.dbconn)
         assert not sess.eye_tracking.empty

allensdk/test/brain_observatory/behavior/test_incomplete_data_objects.py

@@ -54,8 +54,7 @@ def test_incomplete_eye_tracking_from_lims(
 
     incomplete_exp_id = 806456687
     incomplete_experiment = BehaviorOphysExperiment.from_lims(
-        incomplete_exp_id,
-        load_stimulus_movie=False)
+        incomplete_exp_id)
     complete = behavior_ophys_experiment_fixture.eye_tracking
     incomplete = incomplete_experiment.eye_tracking
 

doc_template/visual_behavior_neuropixels.rst

@@ -36,6 +36,13 @@ Visual Behavior - Neuropixels
 DATA FILE CHANGELOG
 -------------------
 
+**v0.4.0-fix**
+
+- Added stimulus presentations columns that are computed on load:
+  is_sham_change
+- Fixed sessions where mouse appears to be running backwards.
+- Fixed issues with loading probes
+
 **v0.4.0**
 
 New Data:

doc_template/visual_behavior_optical_physiology.rst

@@ -312,7 +312,7 @@ DATA FILE CHANGELOG
 
 Metadata Changes
 
-- Better consistency of integer typing.
+- Better consistency of integer typing throughout.
 - Additions to multiple tables
     - Added project_code and behavior_type (active/passive) value to all
       tables.
@@ -331,16 +331,17 @@ Metadata Changes
     - Added corner location of ROI cutout x,y for each ROI.
     - Added width, height of the ROI cutout for each ROI.
 
-Data Changes
+NWB Data Changes
 
-- The value for Age in the metadata, Session/Experiment objects. NWBs now
-  reflect the age of the animal at the time the session/experiment was taken.
+- The value for Age in the metadata, Session/Experiment objects now consistent.
+  NWBs now reflect the age of the animal at the time the session/experiment was
+  taken.
 - Enforced better and more consistent typing between the metadata tables and
   the session metadata.
-- All datetimes in metadata tables and NWBs are explicitly UTC timezone.
+- All datetimes in NWBs and metadata tables are now explicitly UTC timezone.
 - New columns in the stimulus_presentations table:
-    - is_image_novel, movie_frame_index, repeat, stimulus_block,
-      stimulus_block_name, stimulus_name,
+    - active, is_image_novel, is_sham_change, movie_frame_index, movie_repeat,
+      stimulus_block, stimulus_block_name, stimulus_name
 
 **v1.0.1**