From a6ce50150ff5cd7b63daaa6ed55af54830b738ee Mon Sep 17 00:00:00 2001 From: weiglszonja Date: Mon, 23 Sep 2024 16:03:52 +0200 Subject: [PATCH 1/4] add custom interface for processed bpod data --- .../interfaces/__init__.py | 1 + ...embargo_2024_processedbehaviorinterface.py | 81 +++++++++++++++++++ 2 files changed, 82 insertions(+) create mode 100644 src/constantinople_lab_to_nwb/schierek_embargo_2024/interfaces/schierek_embargo_2024_processedbehaviorinterface.py diff --git a/src/constantinople_lab_to_nwb/schierek_embargo_2024/interfaces/__init__.py b/src/constantinople_lab_to_nwb/schierek_embargo_2024/interfaces/__init__.py index fb1223a..d703aa4 100644 --- a/src/constantinople_lab_to_nwb/schierek_embargo_2024/interfaces/__init__.py +++ b/src/constantinople_lab_to_nwb/schierek_embargo_2024/interfaces/__init__.py @@ -1 +1,2 @@ from .schierek_embargo_2024_sortinginterface import SchierekEmbargo2024SortingInterface +from .schierek_embargo_2024_processedbehaviorinterface import SchierekEmbargo2024ProcessedBehaviorInterface diff --git a/src/constantinople_lab_to_nwb/schierek_embargo_2024/interfaces/schierek_embargo_2024_processedbehaviorinterface.py b/src/constantinople_lab_to_nwb/schierek_embargo_2024/interfaces/schierek_embargo_2024_processedbehaviorinterface.py new file mode 100644 index 0000000..896fc24 --- /dev/null +++ b/src/constantinople_lab_to_nwb/schierek_embargo_2024/interfaces/schierek_embargo_2024_processedbehaviorinterface.py @@ -0,0 +1,81 @@ +"""Primary class for converting experiment-specific behavior.""" + +from pathlib import Path +from typing import Optional, Union + +import numpy as np +from warnings import warn +from ndx_structured_behavior.utils import loadmat +from neuroconv import BaseDataInterface +from pynwb.file import NWBFile + + +class SchierekEmbargo2024ProcessedBehaviorInterface(BaseDataInterface): + """Behavior interface for schierek_embargo_2024 conversion""" + + def __init__( + self, + file_path: Union[str, Path], + default_struct_name: str = "S", + verbose: bool = True, + ): + """ + Interface for adding data from the processed behavior file to an existing NWB file. + + Parameters + ---------- + file_path: Union[str, Path] + Path to the .mat file containing the processed behavior data. + default_struct_name: str, optional + The struct name to load from the .mat file, default is "A". + """ + + self.default_struct_name = default_struct_name + super().__init__(file_path=file_path, verbose=verbose) + + def _read_file(self, file_path: Union[str, Path]) -> dict: + behavior_data = loadmat(file_path) + if self.default_struct_name not in behavior_data: + raise ValueError(f"The struct name '{self.default_struct_name}' not found in {file_path}.") + + return behavior_data[self.default_struct_name] + + def add_to_nwbfile( + self, + nwbfile: NWBFile, + metadata: dict, + column_name_mapping: Optional[dict] = None, + column_descriptions: Optional[dict] = None, + ) -> None: + data = self._read_file(file_path=self.source_data["file_path"]) + + if "RewardedSide" in data: + side_mapping = {"L": "Left", "R": "Right"} + data["RewardedSide"] = [side_mapping[side] for side in data["RewardedSide"]] + + columns_with_boolean = ["hits", "vios", "optout"] + for column in columns_with_boolean: + if column in data: + data[column] = list(np.array(data[column]).astype(bool)) + + columns_to_add = column_name_mapping.keys() if column_name_mapping is not None else data.keys() + + trials = nwbfile.trials + if trials is None: + raise ValueError("Trials table not found in NWB file.") + + for column_name in columns_to_add: + if column_name not in data: + warn(f"Column '{column_name}' not found in processed behavior data.", UserWarning) + continue + name = column_name_mapping.get(column_name, column_name) if column_name_mapping is not None else column_name + description = ( + column_descriptions.get(column_name, "no description") + if column_descriptions is not None + else "no description" + ) + trials.add_column( + name=name, + description=description, + data=data[column_name], + ) From 35da36f61e6e57b27abe2e4a33eb749656f9f952 Mon Sep 17 00:00:00 2001 From: weiglszonja Date: Mon, 23 Sep 2024 16:04:25 +0200 Subject: [PATCH 2/4] add behavior interfaces to converter --- .../schierek_embargo_2024_nwbconverter.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/constantinople_lab_to_nwb/schierek_embargo_2024/schierek_embargo_2024_nwbconverter.py b/src/constantinople_lab_to_nwb/schierek_embargo_2024/schierek_embargo_2024_nwbconverter.py index b0fdb35..cf1b8b9 100644 --- a/src/constantinople_lab_to_nwb/schierek_embargo_2024/schierek_embargo_2024_nwbconverter.py +++ b/src/constantinople_lab_to_nwb/schierek_embargo_2024/schierek_embargo_2024_nwbconverter.py @@ -12,7 +12,13 @@ from neuroconv.utils import FilePathType from probeinterface import read_probeinterface, Probe -from constantinople_lab_to_nwb.schierek_embargo_2024.interfaces import SchierekEmbargo2024SortingInterface +# TODO: move to general_interfaces +from constantinople_lab_to_nwb.mah_2024.interfaces import Mah2024BpodInterface + +from constantinople_lab_to_nwb.schierek_embargo_2024.interfaces import ( + SchierekEmbargo2024SortingInterface, + SchierekEmbargo2024ProcessedBehaviorInterface, +) class SchierekEmbargo2024NWBConverter(NWBConverter): @@ -23,6 +29,8 @@ class SchierekEmbargo2024NWBConverter(NWBConverter): RecordingLFP=OpenEphysRecordingInterface, PhySorting=PhySortingInterface, ProcessedSorting=SchierekEmbargo2024SortingInterface, + RawBehavior=Mah2024BpodInterface, + ProcessedBehavior=SchierekEmbargo2024ProcessedBehaviorInterface, ) def _set_probe_properties_for_recording_interface( From 09a28b67e9e7bb910e6af38c2ca41df574bee321 Mon Sep 17 00:00:00 2001 From: weiglszonja Date: Mon, 23 Sep 2024 16:04:39 +0200 Subject: [PATCH 3/4] add behavior metadata --- ...hierek_embargo_2024_behavior_metadata.yaml | 250 ++++++++++++++++++ 1 file changed, 250 insertions(+) create mode 100644 src/constantinople_lab_to_nwb/schierek_embargo_2024/metadata/schierek_embargo_2024_behavior_metadata.yaml diff --git a/src/constantinople_lab_to_nwb/schierek_embargo_2024/metadata/schierek_embargo_2024_behavior_metadata.yaml b/src/constantinople_lab_to_nwb/schierek_embargo_2024/metadata/schierek_embargo_2024_behavior_metadata.yaml new file mode 100644 index 0000000..aa379e1 --- /dev/null +++ b/src/constantinople_lab_to_nwb/schierek_embargo_2024/metadata/schierek_embargo_2024_behavior_metadata.yaml @@ -0,0 +1,250 @@ +Behavior: + TrialsTable: + description: | + LED illumination from the center port indicated that the animal could initiate a trial by poking its nose in that + port - upon trial initiation the center LED turned off. While in the center port, rats needed to maintain center + fixation for a duration drawn uniformly from [0.8, 1.2] seconds. During the fixation period, a tone played from + both speakers, the frequency of which indicated the volume of the offered water reward for that trial + [1, 2, 4, 8, 16kHz, indicating 5, 10, 20, 40, 80μL rewards]. Following the fixation period, one of the two side + LEDs was illuminated, indicating that the reward might be delivered at that port; the side was randomly chosen on + each trial.This event (side LED ON) also initiated a variable and unpredictable delay period, which was randomly + drawn from an exponential distribution with mean=2.5s. The reward port LED remained illuminated for the duration + of the delay period, and rats were not required to maintain fixation during this period, although they tended to + fixate in the reward port. When reward was available, the reward port LED turned off, and rats could collect the + offered reward by nose poking in that port. The rat could also choose to terminate the trial (opt-out) at any time + by nose poking in the opposite, un-illuminated side port, after which a new trial would immediately begin. On a + proportion of trials (15–25%), the delay period would only end if the rat opted out (catch trials). If rats did + not opt-out within 100s on catch trials, the trial would terminate. The trials were self-paced: after receiving + their reward or opting out, rats were free to initiate another trial immediately. However, if rats terminated + center fixation prematurely, they were penalized with a white noise sound and a time out penalty (typically 2s, + although adjusted to individual animals). Following premature fixation breaks, the rats received the same offered + reward, in order to disincentivize premature terminations for small volume offers. We introduced semi-observable, + hidden states in the task by including uncued blocks of trials with varying reward statistics: high and low blocks + , which offered the highest three or lowest three rewards, respectively, and were interspersed with mixed blocks, + which offered all volumes. There was a hierarchical structure to the blocks, such that high and low blocks + alternated after mixed blocks (e.g., mixed-high-mixed-low, or mixed-low-mixed-high). The first block of each + session was a mixed block. Blocks transitioned after 40 successfully completed trials. Because rats prematurely + broke fixation on a subset of trials, in practice, block durations were variable. + StateTypesTable: + description: Contains the name of the states in the task. + WaitForPoke: + name: wait_for_poke + NoseInCenter: + name: nose_in_center + PunishViolation: + name: punish_violation + GoCue: + name: go_cue + WaitForSidePoke: + name: wait_for_side_poke + PortOut: + name: port_out + AnnounceReward: + name: announce_reward + Reward: + name: reward + OptOut: + name: opt_out + StopSound: + name: stop_sound + StatesTable: + description: Contains the start and end times of each state in the task. + EventTypesTable: + description: Contains the name of the events in the task. + Tup: + name: state_timer + GlobalTimer1_Start: + name: state_timer + GlobalTimer1_End: + name: state_timer + Port1In: + name: left_port_poke + Port1Out: + name: left_port_poke + Port2In: + name: center_port_poke + Port2Out: + name: center_port_poke + Port3In: + name : right_port_poke + Port3Out: + name : right_port_poke + EventsTable: + description: Contains the onset times of events in the task. + ActionTypesTable: + description: Contains the name of the task output actions. + SoundOutput: + name: sound_output + ActionsTable: + description: Contains the onset times of the task output actions (e.g. LED turned on/off). + TaskArgumentsTable: + RewardAmount: + name: reward_volume_ul + description: The volume of reward in microliters. + expression_type: integer + output_type: numeric + NoseInCenter: + name: nose_in_center + description: The time in seconds when the animal is required to maintain center port to initiate the trial (uniformly drawn from 0.8 - 1.2 seconds). + expression_type: double + output_type: numeric + NICincrement: + name: time_increment_for_nose_in_center + description: The time increment for nose in center in seconds. + expression_type: double + output_type: numeric + TargetNIC: + name: target_duration_for_nose_in_center + description: The goal for how long the animal must poke center in seconds. + expression_type: double + output_type: numeric + TrainingStage: + name: training_stage + description: The stage of the training. + expression_type: integer + output_type: numeric + DelayToReward: + name: reward_delay + description: The delay in seconds to receive reward, drawn from exponential distribution with mean = 2.5 seconds. + expression_type: double + output_type: numeric + TargetDelayToReward: + name: target_reward_delay + description: The target delay in seconds to receive reward. + expression_type: double + output_type: numeric + DTRincrement: + name: time_increment_for_reward_delay + description: The time increment during monotonic increase of reward delay. + expression_type: double + output_type: numeric + ViolationTO: + name: violation_time_out + description: The time-out if nose is center is not satisfied in seconds. + expression_type: double + output_type: numeric + Block: + name: block_type + description: The block type (High, Low or Test). High and Low blocks are high reward (20, 40, or 80μL) or low reward (5, 10, or 20μL) blocks. Test blocks are mixed blocks. + expression_type: string + output_type: string + BlockLengthTest: + name: num_trials_in_test_blocks + description: The number of trials in each mixed blocks. + expression_type: integer + output_type: numeric + BlockLengthAd: + name: num_trials_in_adaptation_blocks + description: The number of trials in each high reward (20, 40, or 80μL) or low reward (5, 10, or 20μL) blocks. + expression_type: integer + output_type: numeric + PunishSound: + name: punish_sound_enabled + description: Whether to play a white noise pulse on error. + expression_type: boolean + output_type: boolean + ProbCatch: + name: catch_percentage + description: The percentage of catch trials. + expression_type: double + output_type: numeric + IsCatch: + name: is_catch + description: Whether the trial is a catch trial. + expression_type: boolean + output_type: boolean + CTrial: + name: current_trial + description: The current trial number. + expression_type: integer + output_type: numeric + VolumeDelivered: + name: cumulative_reward_volume_ul + description: The cumulative volume received during session in microliters. + expression_type: double + output_type: numeric + WarmUp: + name: is_warm_up + description: Whether the trial is warm-up. + expression_type: boolean + output_type: boolean + OverrideNIC: + name: override_nose_in_center + description: Whether the required time for maintaining center port is overridden. + expression_type: boolean + output_type: boolean + TrialsInStage: + name: trials_in_stage + description: The cumulative number of trials in the stages. + expression_type: integer + output_type: numeric + MinimumVol: + name: min_reward_volume_ul + description: The minimum volume of reward in microliters. (The minimum volume is 4 ul for females and 6 ul for males.) + expression_type: double + output_type: numeric + AutoProbCatch: + name: auto_change_catch_probability + description: Whether to change the probability automatically after a certain number of trials. + expression_type: boolean + output_type: boolean + PrevWasViol: + name: previous_was_violation + description: Whether the previous trial was a violation. + expression_type: boolean + output_type: boolean + changed: + name: changed + description: Whether a block transition occurred for the trial. + expression_type: boolean + output_type: boolean + CPCue: + name: center_port_cue + description: Task parameter. + expression_type: boolean + output_type: boolean + CycleBlocks: + name: cycle_blocks + description: Task parameter. + expression_type: boolean + output_type: boolean + HitFrac: + name: hit_percentage + description: The percentage of hit trials. + expression_type: double + output_type: numeric + hits: + name: hits + description: The number of trials where reward was delivered. + expression_type: integer + output_type: numeric + TrialsStage2: + name: num_trials_in_stage_2 + description: Determines how many trials occur in stage 2 before transition. + expression_type: integer + output_type: numeric + TrialsStage3: + name: num_trials_in_stage_3 + description: Determines how many trials occur in stage 3 before transition. + expression_type: integer + output_type: numeric + TrialsStage4: + name: num_trials_in_stage_4 + description: Determines how many trials occur in stage 4 before transition. + expression_type: integer + output_type: numeric + TrialsStage5: + name: num_trials_in_stage_5 + description: Determines how many trials occur in stage 5 before transition. + expression_type: integer + output_type: numeric + TrialsStage6: + name: num_trials_in_stage_6 + description: Determines how many trials occur in stage 6 before transition. + expression_type: integer + output_type: numeric + TrialsStage8: + name: num_trials_in_stage_8 + description: Determines how many trials occur in stage 8 before transition. + expression_type: integer + output_type: numeric From dee66a0fb981a11381b05a1d1c9ad5f0e1d46b03 Mon Sep 17 00:00:00 2001 From: weiglszonja Date: Mon, 23 Sep 2024 16:06:16 +0200 Subject: [PATCH 4/4] update convert session script --- .../schierek_embargo_2024_convert_session.py | 84 ++++++++++++++++++- 1 file changed, 80 insertions(+), 4 deletions(-) diff --git a/src/constantinople_lab_to_nwb/schierek_embargo_2024/schierek_embargo_2024_convert_session.py b/src/constantinople_lab_to_nwb/schierek_embargo_2024/schierek_embargo_2024_convert_session.py index 57792fb..d05ed1e 100644 --- a/src/constantinople_lab_to_nwb/schierek_embargo_2024/schierek_embargo_2024_convert_session.py +++ b/src/constantinople_lab_to_nwb/schierek_embargo_2024/schierek_embargo_2024_convert_session.py @@ -2,7 +2,7 @@ import os from pathlib import Path -from typing import Union +from typing import Union, Optional from dateutil import tz from neuroconv.datainterfaces import OpenEphysRecordingInterface @@ -16,7 +16,10 @@ def session_to_nwb( openephys_recording_folder_path: Union[str, Path], spike_sorting_folder_path: Union[str, Path], processed_spike_sorting_file_path: Union[str, Path], + raw_behavior_file_path: Union[str, Path], nwbfile_path: Union[str, Path], + column_name_mapping: Optional[dict] = None, + column_descriptions: Optional[dict] = None, stub_test: bool = False, overwrite: bool = False, ): @@ -76,14 +79,35 @@ def session_to_nwb( dict( ProcessedSorting=dict( file_path=processed_spike_sorting_file_path, sampling_frequency=sampling_frequency - ) + ), + ProcessedBehavior=dict( + file_path=processed_spike_sorting_file_path, + ), ) ) conversion_options.update(dict(ProcessedSorting=dict(write_as="processing", stub_test=False))) + conversion_options.update( + dict( + ProcessedBehavior=dict(column_name_mapping=column_name_mapping, column_descriptions=column_descriptions) + ) + ) # Add Behavior - # source_data.update(dict(Behavior=dict())) - # conversion_options.update(dict(Behavior=dict())) + source_data.update(dict(RawBehavior=dict(file_path=raw_behavior_file_path))) + # Exclude some task arguments from the trials table that are the same for all trials + task_arguments_to_exclude = [ + "BlockLengthTest", + "BlockLengthAd", + "TrialsStage2", + "TrialsStage3", + "TrialsStage4", + "TrialsStage5", + "TrialsStage6", + "TrialsStage8", + "CTrial", + "HiITI", + ] + conversion_options.update(dict(RawBehavior=dict(task_arguments_to_exclude=task_arguments_to_exclude))) recording_folder_name = recording_folder_path.stem subject_id, session_id = recording_folder_name.split("_", maxsplit=1) @@ -108,6 +132,7 @@ def session_to_nwb( metadata["NWBFile"].update( session_start_time=session_start_time.replace(tzinfo=tzinfo), session_id=session_id, + # TODO: add protocol name for behavior task ) # Update default metadata with the editable in the corresponding yaml file @@ -115,6 +140,11 @@ def session_to_nwb( editable_metadata = load_dict_from_file(editable_metadata_path) metadata = dict_deep_update(metadata, editable_metadata) + # Update behavior metadata + behavior_metadata_path = Path(__file__).parent / "metadata" / "schierek_embargo_2024_behavior_metadata.yaml" + behavior_metadata = load_dict_from_file(behavior_metadata_path) + metadata = dict_deep_update(metadata, behavior_metadata) + metadata["Subject"].update(subject_id=subject_id) # Run conversion @@ -134,6 +164,49 @@ def session_to_nwb( openephys_recording_folder_path / "Record Node 117/experiment1/recording1/continuous/Neuropix-PXI-119.ProbeA-AP" ) processed_sorting_file_path = Path("/Volumes/T9/Constantinople/Ephys Data/J076_2023-12-12.mat") + bpod_file_path = Path("/Volumes/T9/Constantinople/raw_Bpod/J076/DataFiles/J076_RWTautowait2_20231212_145250.mat") + + # The column name mapping is used to rename the columns in the processed data to more descriptive column names. (optional) + column_name_mapping = dict( + hits="is_rewarded", + vios="is_violation", + optout="is_opt_out", + wait_time="wait_time", + wait_thresh="wait_time_threshold", + wait_for_cpoke="wait_for_center_poke", + zwait_for_cpoke="z_scored_wait_for_center_poke", + RewardedSide="rewarded_port", + Cled="center_poke_times", + Lled="left_poke_times", + Rled="right_poke_times", + l_opt="left_opt_out_times", + r_opt="right_opt_out_times", + ReactionTime="reaction_time", + slrt="short_latency_reaction_time", + iti="inter_trial_interval", + ) + # The column descriptions are used to add descriptions to the columns in the processed data. (optional) + column_descriptions = dict( + hits="Whether the subject received reward for each trial.", + vios="Whether the subject violated the trial by not maintaining center poke for the time required by 'nose_in_center'.", + optout="Whether the subject opted out for each trial.", + wait_time="The wait time for the subject for for each trial in seconds, after removing outliers." + " For hit trials (when reward was delivered) the wait time is equal to the reward delay." + " For opt-out trials, the wait time is equal to the time waited from trial start to opting out.", + wait_for_cpoke="The time between side port poke and center poke in seconds, includes the time when the subject is consuming the reward.", + zwait_for_cpoke="The z-scored wait_for_cpoke using all trials.", + RewardedSide="The rewarded port (Left or Right) for each trial.", + Cled="The time of center port LED on/off for each trial (2 x ntrials).", + Lled="The time of left port LED on/off for each trial (2 x ntrials).", + Rled="The time of right port LED on/off for each trial (2 x ntrials).", + l_opt="The time of left port entered/exited for each trial (2 x ntrials).", + r_opt="The time of right port entered/exited for each trial (2 x ntrials).", + ReactionTime="The reaction time in seconds.", + slrt="The short-latency reaction time in seconds.", + iti="The time to initiate trial in seconds (the time between the end of the consummatory period and the time to initiate the next trial).", + wait_thresh="The threshold in seconds to remove wait-times (mean + 1*std of all cumulative wait-times).", + ) + nwbfile_path = Path("/Volumes/T9/Constantinople/nwbfiles/J076_2023-12-12_14-52-04.nwb") if not nwbfile_path.parent.exists(): os.makedirs(nwbfile_path.parent, exist_ok=True) @@ -145,6 +218,9 @@ def session_to_nwb( openephys_recording_folder_path=openephys_recording_folder_path, spike_sorting_folder_path=phy_sorting_folder_path, processed_spike_sorting_file_path=processed_sorting_file_path, + raw_behavior_file_path=bpod_file_path, + column_name_mapping=column_name_mapping, + column_descriptions=column_descriptions, nwbfile_path=nwbfile_path, stub_test=stub_test, overwrite=overwrite,