diff --git a/CHANGELOG.md b/CHANGELOG.md index e5e110587..215e750f8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,13 @@ # Change Log All notable changes to this project will be documented in this file. +## [2.16.0] = 2023-XX-XX +- Python 3.10 and 3.11 support +- Support for VBO update release. +- Added new stimulus presentations columns that can be computed/added when +loading previous data. +- Various bug fixes. + ## [2.14.0] = 2022-12-12 - Support for updated vbn release containing probe, lfp and behavior only data. - Updates to Ophys data in anticipation of a forthcoming updated data release diff --git a/allensdk/brain_observatory/behavior/behavior_ophys_experiment.py b/allensdk/brain_observatory/behavior/behavior_ophys_experiment.py index 2acd6d796..54283948c 100644 --- a/allensdk/brain_observatory/behavior/behavior_ophys_experiment.py +++ b/allensdk/brain_observatory/behavior/behavior_ophys_experiment.py @@ -110,7 +110,6 @@ def from_lims( events_filter_scale_seconds: float = 2.0 / 31.0, events_filter_n_time_steps: int = 20, exclude_invalid_rois: bool = True, - load_stimulus_movie: bool = False ) -> "BehaviorOphysExperiment": """ Parameters @@ -127,9 +126,6 @@ def from_lims( See `BehaviorOphysExperiment.from_nwb` exclude_invalid_rois : bool Whether to exclude invalid rois - load_stimulus_movie : bool - Whether to load the stimulus movie (e.g natrual_movie_one) as - part of loading stimuli. Default True. Returns ------- @@ -188,7 +184,6 @@ def _get_motion_correction(): date_of_acquisition=date_of_acquisition, eye_tracking_z_threshold=eye_tracking_z_threshold, eye_tracking_dilation_frames=eye_tracking_dilation_frames, - load_stimulus_movie=load_stimulus_movie ) if is_multiplane_session: ophys_timestamps = OphysTimestampsMultiplane.from_sync_file( diff --git a/allensdk/brain_observatory/behavior/behavior_session.py b/allensdk/brain_observatory/behavior/behavior_session.py index 1a550d706..7e165e606 100644 --- a/allensdk/brain_observatory/behavior/behavior_session.py +++ b/allensdk/brain_observatory/behavior/behavior_session.py @@ -358,7 +358,6 @@ def from_lims( date_of_acquisition: Optional[DateOfAcquisition] = None, eye_tracking_z_threshold: float = 3.0, eye_tracking_dilation_frames: int = 2, - load_stimulus_movie: bool = False, ) -> "BehaviorSession": """ @@ -384,11 +383,6 @@ def from_lims( See `BehaviorSession.from_nwb`, default 3.0 eye_tracking_dilation_frames : int See `BehaviorSession.from_nwb`, default 2 - load_stimulus_movie : bool - Whether to load the stimulus movie (e.g natrual_movie_one) as - part of loading stimuli. Default False. The warped+unwarped - movie loaded will be very large in memory/on disk so be - careful when requesting this option. Returns ------- @@ -456,7 +450,6 @@ def from_lims( project_code=ProjectCode.from_lims( behavior_session_id=behavior_session_id.value, lims_db=lims_db ), - load_stimulus_movie=load_stimulus_movie, ) if date_of_acquisition is None: @@ -1153,34 +1146,6 @@ def stimulus_templates(self) -> Optional[pd.DataFrame]: else: return None - @property - def stimulus_natural_movie_template(self) -> Optional[pd.DataFrame]: - """Get stimulus templates movie for the behavior session. - - Returns None if no stimulus movie is available. - - Returns - ------- - pd.DataFrame or None - A pandas DataFrame object containing the individual frames for the - movie shown during this experiment. - - dataframe columns: - frame_number [index]: (int) - Frame number in movie - unwarped: (array of int) - image array of unwarped stimulus movie frame - warped: (array of int) - image array of warped stimulus movie frame - - """ - if self._stimuli.templates.fingerprint_movie_template_key is not None: - return self._stimuli.templates.value[ - self._stimuli.templates.fingerprint_movie_template_key - ].to_dataframe(index_name="frame_number", index_type="int") - else: - return None - @property def stimulus_timestamps(self) -> np.ndarray: """Timestamps associated with the stimulus presetntation on @@ -1415,7 +1380,6 @@ def _read_stimuli( trials: Trials, stimulus_presentation_columns: Optional[List[str]] = None, project_code: Optional[ProjectCode] = None, - load_stimulus_movie: bool = False, ) -> Stimuli: """ Construct the Stimuli data object for this session @@ -1434,7 +1398,6 @@ def _read_stimuli( presentation_columns=stimulus_presentation_columns, project_code=project_code, trials=trials, - load_stimulus_movie=load_stimulus_movie, ) @classmethod @@ -1514,7 +1477,6 @@ def _read_data_from_stimulus_file( include_stimuli: bool = True, stimulus_presentation_columns: Optional[List[str]] = None, project_code: Optional[ProjectCode] = None, - load_stimulus_movie: bool = False, ): """Helper method to read data from stimulus file""" @@ -1551,7 +1513,6 @@ def _read_data_from_stimulus_file( trials=trials, stimulus_presentation_columns=stimulus_presentation_columns, project_code=project_code, - load_stimulus_movie=load_stimulus_movie, ) else: stimuli = None diff --git a/allensdk/brain_observatory/behavior/stimulus_processing.py b/allensdk/brain_observatory/behavior/stimulus_processing.py index d28a6c386..f0c653f7d 100644 --- a/allensdk/brain_observatory/behavior/stimulus_processing.py +++ b/allensdk/brain_observatory/behavior/stimulus_processing.py @@ -696,6 +696,14 @@ def add_active_flag( & (~stim_pres_table.image_name.isna()) ) active[stim_mask] = True + + # Clean up potential stimuli that fall outside in time of the trials + # but are part of the "active" stimulus block. + if "stimulus_block" in stim_pres_table.columns: + for stim_block in stim_pres_table["stimulus_block"].unique(): + block_mask = stim_pres_table["stimulus_block"] == stim_block + if np.any(active[block_mask]): + active[block_mask] = True stim_pres_table["active"] = active return stim_pres_table diff --git a/allensdk/brain_observatory/nwb/nwb_api.py b/allensdk/brain_observatory/nwb/nwb_api.py index f2d66330f..f5d592710 100644 --- a/allensdk/brain_observatory/nwb/nwb_api.py +++ b/allensdk/brain_observatory/nwb/nwb_api.py @@ -1,5 +1,3 @@ -from pathlib import Path - import pandas as pd import pynwb import SimpleITK as sitk @@ -24,7 +22,8 @@ def nwbfile(self): return io.read() def __init__(self, path, **kwargs): - ''' Reads data for a single Brain Observatory session from an NWB 2.0 file + ''' Reads data for a single Brain Observatory session from an NWB 2.0 + file ''' self.path = path diff --git a/allensdk/test/brain_observatory/behavior/conftest.py b/allensdk/test/brain_observatory/behavior/conftest.py index 83723c6e0..5941b1e4f 100644 --- a/allensdk/test/brain_observatory/behavior/conftest.py +++ b/allensdk/test/brain_observatory/behavior/conftest.py @@ -127,6 +127,5 @@ def behavior_ophys_experiment_fixture(): experiment_id = 953443028 experiment = BehaviorOphysExperiment.from_lims( - experiment_id, - load_stimulus_movie=False) + experiment_id) return experiment diff --git a/allensdk/test/brain_observatory/behavior/data_objects/test_data/presentations.pkl b/allensdk/test/brain_observatory/behavior/data_objects/test_data/presentations.pkl index 22cba9f1e..67a807101 100644 Binary files a/allensdk/test/brain_observatory/behavior/data_objects/test_data/presentations.pkl and b/allensdk/test/brain_observatory/behavior/data_objects/test_data/presentations.pkl differ diff --git a/allensdk/test/brain_observatory/behavior/data_objects/test_stimuli.py b/allensdk/test/brain_observatory/behavior/data_objects/test_stimuli.py index c82ba4f51..0873f5326 100644 --- a/allensdk/test/brain_observatory/behavior/data_objects/test_stimuli.py +++ b/allensdk/test/brain_observatory/behavior/data_objects/test_stimuli.py @@ -52,7 +52,7 @@ def data(self): return pd.DataFrame( { "start_time": [300.0, 330.0, 360.0], - "stop_time": [330.0, 360.0, 360.0], + "stop_time": [330.0, 360.0, 390.0], "catch": [False, True, False], "change_frame": [-99, 99, -99], } diff --git a/allensdk/test/brain_observatory/behavior/test_behavior_ophys_experiment.py b/allensdk/test/brain_observatory/behavior/test_behavior_ophys_experiment.py index 79825b93a..779a6413a 100644 --- a/allensdk/test/brain_observatory/behavior/test_behavior_ophys_experiment.py +++ b/allensdk/test/brain_observatory/behavior/test_behavior_ophys_experiment.py @@ -33,7 +33,6 @@ def test_nwb_end_to_end(tmpdir_factory): d1 = BehaviorOphysExperiment.from_lims( oeid, - load_stimulus_movie=False ) nwbfile = d1.to_nwb() with NWBHDF5IO(nwb_filepath, "w") as nwb_file_writer: @@ -52,7 +51,6 @@ def test_visbeh_ophys_data_set(): data_set = BehaviorOphysExperiment.from_lims( ophys_experiment_id, exclude_invalid_rois=False, - load_stimulus_movie=False ) # TODO: need to improve testing here: @@ -68,7 +66,6 @@ def test_visbeh_ophys_data_set(): behavior_session_id = BehaviorSessionId.from_lims( db=lims_db, ophys_experiment_id=ophys_experiment_id, - load_stimulus_movie=False ) # All sorts of assert relationships: @@ -76,7 +73,6 @@ def test_visbeh_ophys_data_set(): ForagingId.from_lims( behavior_session_id=behavior_session_id.value, lims_db=lims_db, - load_stimulus_movie=False ).value == data_set.metadata["behavior_session_uuid"] ) @@ -167,7 +163,6 @@ def test_legacy_dff_api(): ophys_experiment_id = 792813858 session = BehaviorOphysExperiment.from_lims( ophys_experiment_id=ophys_experiment_id, - load_stimulus_movie=False ) _, dff_array = session.get_dff_traces() @@ -187,7 +182,6 @@ def test_legacy_dff_api(): def test_stimulus_presentations_omitted(ophys_experiment_id, number_omitted): session = BehaviorOphysExperiment.from_lims( ophys_experiment_id, - load_stimulus_movie=False ) df = session.stimulus_presentations assert df["omitted"].sum() == number_omitted @@ -198,7 +192,6 @@ def test_event_detection(): ophys_experiment_id = 789359614 session = BehaviorOphysExperiment.from_lims( ophys_experiment_id=ophys_experiment_id, - load_stimulus_movie=False ) events = session.events @@ -229,7 +222,6 @@ def test_BehaviorOphysExperiment_property_data(): ophys_experiment_id = 960410026 dataset = BehaviorOphysExperiment.from_lims( ophys_experiment_id, - load_stimulus_movie=False ) assert dataset.ophys_session_id == 959458018 @@ -277,7 +269,6 @@ def test_behavior_ophys_experiment_list_data_attributes_and_methods( "segmentation_mask_image", "stimulus_presentations", "stimulus_templates", - 'stimulus_natural_movie_template', "stimulus_timestamps", "task_parameters", "trials", diff --git a/allensdk/test/brain_observatory/behavior/test_behavior_session.py b/allensdk/test/brain_observatory/behavior/test_behavior_session.py index 97ccccbf4..66b50172c 100644 --- a/allensdk/test/brain_observatory/behavior/test_behavior_session.py +++ b/allensdk/test/brain_observatory/behavior/test_behavior_session.py @@ -20,8 +20,7 @@ def test_nwb_end_to_end_session( tmpdir = pathlib.Path(tmpdir) nwb_path = tmpdir / f'session_{session_id}.nwb' session = BehaviorSession.from_lims( - behavior_session_id=session_id, - load_stimulus_movie=False) + behavior_session_id=session_id) nwb_file = session.to_nwb() with NWBHDF5IO(nwb_path, 'w') as nwb_file_writer: nwb_file_writer.write(nwb_file) @@ -88,7 +87,6 @@ def dummy_init(self): 'running_speed', 'stimulus_presentations', 'stimulus_templates', - 'stimulus_natural_movie_template', 'stimulus_timestamps', 'task_parameters', 'trials', @@ -106,8 +104,7 @@ def test_behavior_session_equivalent_json_lims(session_data_fixture): behavior_session_id = session_data_fixture['behavior_session_id'] lims_session = BehaviorSession.from_lims( - behavior_session_id, - load_stimulus_movie=False + behavior_session_id ) assert sessions_are_equal(json_session, lims_session, reraise=True) @@ -120,6 +117,5 @@ def test_eye_tracking_loaded_with_metadata_frame(self): sess_id = 1154034257 sess = BehaviorSession.from_lims(behavior_session_id=sess_id, - lims_db=self.dbconn, - load_stimulus_movie=False) + lims_db=self.dbconn) assert not sess.eye_tracking.empty diff --git a/allensdk/test/brain_observatory/behavior/test_incomplete_data_objects.py b/allensdk/test/brain_observatory/behavior/test_incomplete_data_objects.py index df482be7b..c32a115a5 100644 --- a/allensdk/test/brain_observatory/behavior/test_incomplete_data_objects.py +++ b/allensdk/test/brain_observatory/behavior/test_incomplete_data_objects.py @@ -54,8 +54,7 @@ def test_incomplete_eye_tracking_from_lims( incomplete_exp_id = 806456687 incomplete_experiment = BehaviorOphysExperiment.from_lims( - incomplete_exp_id, - load_stimulus_movie=False) + incomplete_exp_id) complete = behavior_ophys_experiment_fixture.eye_tracking incomplete = incomplete_experiment.eye_tracking diff --git a/doc_template/visual_behavior_neuropixels.rst b/doc_template/visual_behavior_neuropixels.rst index 28400c51c..89aa57c86 100644 --- a/doc_template/visual_behavior_neuropixels.rst +++ b/doc_template/visual_behavior_neuropixels.rst @@ -36,6 +36,13 @@ Visual Behavior - Neuropixels DATA FILE CHANGELOG ------------------- +**v0.4.0-fix** + +- Added stimulus presentations columns that are computed on load: + is_sham_change +- Fixed sessions where mouse appears to be running backwards. +- Fixed issues with loading probes + **v0.4.0** New Data: diff --git a/doc_template/visual_behavior_optical_physiology.rst b/doc_template/visual_behavior_optical_physiology.rst index e08bba34e..3163a7b36 100644 --- a/doc_template/visual_behavior_optical_physiology.rst +++ b/doc_template/visual_behavior_optical_physiology.rst @@ -312,7 +312,7 @@ DATA FILE CHANGELOG Metadata Changes -- Better consistency of integer typing. +- Better consistency of integer typing throughout. - Additions to multiple tables - Added project_code and behavior_type (active/passive) value to all tables. @@ -331,16 +331,17 @@ Metadata Changes - Added corner location of ROI cutout x,y for each ROI. - Added width, height of the ROI cutout for each ROI. -Data Changes +NWB Data Changes -- The value for Age in the metadata, Session/Experiment objects. NWBs now - reflect the age of the animal at the time the session/experiment was taken. +- The value for Age in the metadata, Session/Experiment objects now consistent. + NWBs now reflect the age of the animal at the time the session/experiment was + taken. - Enforced better and more consistent typing between the metadata tables and the session metadata. -- All datetimes in metadata tables and NWBs are explicitly UTC timezone. +- All datetimes in NWBs and metadata tables are now explicitly UTC timezone. - New columns in the stimulus_presentations table: - - is_image_novel, movie_frame_index, repeat, stimulus_block, - stimulus_block_name, stimulus_name, + - active, is_image_novel, is_sham_change, movie_frame_index, movie_repeat, + stimulus_block, stimulus_block_name, stimulus_name **v1.0.1**