From a4cdfc7dea05b6107fb146b1e5bcb337eb4e00d4 Mon Sep 17 00:00:00 2001 From: Chris Broz Date: Tue, 4 Jan 2022 17:47:14 -0600 Subject: [PATCH 01/19] see ReadMe notes on progress --- .gitignore | 1 + CHANGELOG.md | 8 +- CONTRIBUTING.md | 3 + README.md | 105 ++++++++-- element_trial/__init__.py | 13 ++ element_trial/event.py | 1 + element_trial/export/__init__.py | 14 ++ element_trial/export/nwb.py | 85 ++++++++ element_trial/readers/bpod.py | 76 +++++++ element_trial/readers/bpod_fields_notes.py | 25 +++ element_trial/trial.py | 198 ++++++++++++++++++ .../version.py | 0 element_trial_behavior/__init__.py | 68 ------ element_trial_behavior/behavior_trials.py | 169 --------------- element_trial_behavior/readers/bpod.py | 49 ----- setup.py | 30 +++ 16 files changed, 537 insertions(+), 308 deletions(-) create mode 100644 CONTRIBUTING.md create mode 100644 element_trial/__init__.py create mode 100644 element_trial/event.py create mode 100644 element_trial/export/__init__.py create mode 100644 element_trial/export/nwb.py create mode 100644 element_trial/readers/bpod.py create mode 100644 element_trial/readers/bpod_fields_notes.py create mode 100644 element_trial/trial.py rename {element_trial_behavior => element_trial}/version.py (100%) delete mode 100644 element_trial_behavior/__init__.py delete mode 100644 element_trial_behavior/behavior_trials.py delete mode 100644 element_trial_behavior/readers/bpod.py create mode 100644 setup.py diff --git a/.gitignore b/.gitignore index 27a6378..2742b10 100644 --- a/.gitignore +++ b/.gitignore @@ -123,3 +123,4 @@ docker-compose.y*ml # notes temp* +*/temp* \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 68d8569..ed66e14 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,10 +2,10 @@ Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) convention. -## [0.1.0c0] - 2021-11-03 +## [0.1.0b0] - [unreleased] ### Added -+ First draft begins ++ First beta release -## [0.1.0b0] - 2021-00-00 +## [0.1.0c0] - 2021-11-15 ### Added -+ First beta release ++ First draft begins diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..5836c18 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,3 @@ +# Contribution Guidelines + +This project follows the [DataJoint Contribution Guidelines](https://docs.datajoint.io/python/community/02-Contribute.html). Please reference the link for more full details. diff --git a/README.md b/README.md index 4995c4d..527939c 100644 --- a/README.md +++ b/README.md @@ -1,23 +1,71 @@ -# canonical-behavior -This repository is a work in progress. It serves as a draft of a DataJoints element for trial-based behavior for our U24 itiative. +# element-trial +This repository is a work in progress not yet ready for public release. +It serves as a draft of a DataJoint element for trialized experiments behavior +for our U24 itiative. + +## Schemas +`trial` should be activated if all events are tied to specific trials. +`event` should be activated if events occur independently of trials. + +The current draft of `trial` contains all proposed tables. When we're sure of +the contents, including make functions, one would copy the contents of the +`trial` schema over to events, and then remove the trialized version of the +event table. + +## Filetypes + +### Bpod +MATLAB is the only officially supported environment. SciPy has a lot of support +for loading matlab files. In the current draft, SciPy is used to import Bpod +.mat files as embedded dictionaries. Micheal Wulf shared the Bpod files that +are currently in `workflow-trial/user_data/Bpod_files` as a diverse set of +examples. + +Under `element_trial/readers/`, there are two files: `Bpod.py` and +`Bpod_fields_notes.py`. The former is a draft of an eventual reader that could +be folded into `element_data_loader`. It handles general ingestion for fields +that are consisitent across examples. The latter highlights just how different +Bpod files can be. A fully realized loader may need to pull directly from +Bpod 'raw' fields to reconstruct the experiement, as many other fields are +organized differently across trials. + +Development for this reader was partially conducted within +`workflow_trial/notebooks/1_.ipynb`. This contains code snipits for exploring +the loaded Bpod data. + +Ecosystem: The SanWorks team +([git repositories here](https://github.com/sanworks?tab=repositories)) offers +no offical python support, suggesting instead that users +[export to JSON](https://sanworks.io/forum/showthread.php?tid=626&pid=1169). +The PyBpod project ([github](https://github.com/pyBpod/pyBpod), +[docs](https://pyBpod.readthedocs.io/en/v1.8.1/)) offers a python-based GUI +alternative for running Bpod hardware. So far as I could tell, they do not +offer any usefull ingestion functions we would want to incorporate. + + +### Alternate generalized format + +We previously discussed also supporing a univariate csv format that would +contain 4 values: timestamp on, timestamp off, data type and value. This has +not yet been implemented. -## Notes: -I looked at the structure for `element-array-ephys` for general principle on how to call and load files. I mirrored the main DataJoint implementation as split from 'readers'. I incorporated feedback from project-specific `behavior.py` elsewhere in table development. ## To do: -- [ ] Support functions - - [ ] Other elements/workflows pull `find_full_path` and `find_root_directory` either from their own `__init__.py` files or from `element-data-loader.utils`. Which is best practice? - - [ ] `workflow-array-ephys` relies on the linking module for functions to get root and session directories, but the MAP project defines these internally. Which is best practice? +- [X] Support functions + - [X] Pull `find_full_path` and `find_root_directory` from `element-data-loader.utils`. + - [X] To find root/session dirs, refer to config file if exists. If not, linking module - [ ] Table definitions - - [ ] Discuss table structure - - [ ] Decide supported filetypes - - [ ] BPOD - - [ ] Kepec standard, TBD - - [ ] Generalizable CSV with user-determined column name to DJ variable name correspondence? -- [ ] Contact the [BPod team](https://github.com/sanworks/) - - [ ] Already an implementation of loading to Python? - - [ ] Create joint sustainability roadmap -- [ ] Contact Kepec team - joint sustainability roadmap + - [X] Discuss table structure + - [X] Decide supported filetypes: + - Bpod + - Generalizable 'Univariate' CSV: timestamp on, off, data type, value + - [ ] Develop ingestion functions +- [ ] Test tables with example data + - Aeon, bonsai system API + - Kepecs group provided Bpod files +- [X] Contact the [Bpod team](https://github.com/sanworks/) + - [X] Already an implementation of loading to Python - No + - [X] Create joint sustainability roadmap - Unlikely - [ ] Analysis package - [ ] Load processed data - [ ] Create table definitions @@ -29,7 +77,28 @@ I looked at the structure for `element-array-ephys` for general principle on how - [ ] Tutorials in text format (i.e. Jupyter notebook) - [ ] Tutorial in video format - [ ] Docker for tests - - [ ] Example dataset(s) for public release, in DJ Archive + - [X] Example dataset(s) for public release + - [ ] Example dataset loaded to DJ Archive - [ ] NWB export - [ ] README -- [ ] RRID \ No newline at end of file +- [ ] RRID + +## Installation + +``` +pip install element-trial +``` + +if you have an older version of ***element-trial*** installed, using `pip`, upgrade with + +``` +pip install --upgrade element-trial +``` + +Install `element-data-loader` + +``` +element-data-loader @ git+https://github.com/datajoint/element-data-loader +``` + + diff --git a/element_trial/__init__.py b/element_trial/__init__.py new file mode 100644 index 0000000..6ff17f9 --- /dev/null +++ b/element_trial/__init__.py @@ -0,0 +1,13 @@ +__author__ = "DataJoint" +__date__ = "December, 2021" +__version__ = "0.1.0c0" + +__all__ = ['__author__', '__version__', '__date__'] + +import datajoint as dj +dj.config['enable_python_native_blobs'] = True + +""" +Two schemas, trial and event. Trial for fully trialized, segmented. +Event for events independent of trials, like an act of naturaistic behavior. +""" diff --git a/element_trial/event.py b/element_trial/event.py new file mode 100644 index 0000000..0a7f5da --- /dev/null +++ b/element_trial/event.py @@ -0,0 +1 @@ +# Copy from trial, remove 'EventTrialized' diff --git a/element_trial/export/__init__.py b/element_trial/export/__init__.py new file mode 100644 index 0000000..0fa2104 --- /dev/null +++ b/element_trial/export/__init__.py @@ -0,0 +1,14 @@ + + +__author__ = "DataJoint" +__date__ = "November, 2021" +__version__ = "0.1.0c0" + +__all__ = ['__author__', '__version__', '__date__'] + + +import datajoint as dj +import pathlib + +## https://github.com/nwb-extensions/ndx-events-record + diff --git a/element_trial/export/nwb.py b/element_trial/export/nwb.py new file mode 100644 index 0000000..0a13cb6 --- /dev/null +++ b/element_trial/export/nwb.py @@ -0,0 +1,85 @@ +import numpy as np +from datetime import datetime +from dateutil.tz import tzlocal +from pynwb import NWBFile + +from element_session import session + +def trial_to_nwb(trial_key): + scan_query = trial.TrialEvent & trial_key + # scan_query = scan_query.join(XXX, left=True) + lab_info = lab.query.fetch1() + + return NWBFile( + # https://nwb-schema.readthedocs.io/en/latest/format.html#optical-physiology + + ) + +""" From CHEN 2017 + + + # =============================================================================== + # =============================== BEHAVIOR TRIALS =============================== + # =============================================================================== + + # =============== TrialSet ==================== + # NWB 'trial' (of type dynamic table) by default comes with three mandatory attributes: 'start_time' and 'stop_time' + # Other trial-related information needs to be added in to the trial-table as additional columns (with column name + # and column description) + + dj_trial = experiment.SessionTrial * experiment.BehaviorTrial + skip_adding_columns = experiment.Session.primary_key + ['trial_uid', 'trial'] + + if experiment.SessionTrial & session_key: + # Get trial descriptors from TrialSet.Trial and TrialStimInfo + trial_columns = [{'name': tag, + 'description': re.sub('\s+:|\s+', ' ', re.search( + f'(?<={tag})(.*)', str(dj_trial.heading)).group()).strip()} + for tag in dj_trial.heading.names + if tag not in skip_adding_columns + ['start_time', 'stop_time']] + + # Add new table columns to nwb trial-table for trial-label + for c in trial_columns: + nwbfile.add_trial_column(**c) + + # Add entry to the trial-table + for trial in (dj_trial & session_key).fetch(as_dict=True): + trial['start_time'] = float(trial['start_time']) + trial['stop_time'] = float(trial['stop_time']) if trial['stop_time'] else 5.0 + [trial.pop(k) for k in skip_adding_columns] + trial['early_lick'] = True if trial['early_lick'] == 'early' else False + nwbfile.add_trial(**trial) + + # =============================================================================== + # =============================== BEHAVIOR TRIAL EVENTS ========================== + # =============================================================================== + + behavior = nwbfile.create_processing_module( + 'behavior', 'Time of behavioral events in this session') + behav_event = pynwb.behavior.BehavioralEvents(name='BehavioralEvents') + behavior.add_data_interface(behav_event) + + for trial_event_type in \ + (experiment.TrialEventType & \ + experiment.TrialEvent & session_key).fetch('trial_event_type'): + event_times, trial_starts = \ + (experiment.TrialEvent * experiment.SessionTrial + & session_key & {'trial_event_type': trial_event_type}).fetch( + 'trial_event_time', 'start_time') + + if trial_event_type == 'sample': + description = 'Time stamps of the beginning of the sampling period on each trial.' + elif trial_event_type == 'delay': + description = 'Time stamps of the beginning of the delay period on each trial.' + elif trial_event_type == 'go': + description = 'Time stamps of the go cue signal on each trial.' + + if len(event_times) > 0: + event_times = np.hstack(event_times.astype(float) + trial_starts.astype(float)) + behav_event.create_timeseries( + name=trial_event_type, unit='a.u.', conversion=1.0, + data=np.full_like(event_times, 1), + timestamps=event_times, + description=description) + +""" \ No newline at end of file diff --git a/element_trial/readers/bpod.py b/element_trial/readers/bpod.py new file mode 100644 index 0000000..ab50d11 --- /dev/null +++ b/element_trial/readers/bpod.py @@ -0,0 +1,76 @@ +''' bpod_loader.py +For eventual inclusion in element-data-loader +''' + +import pathlib +import scipy.io as spio + + +class BPod: + """ Parse a bpod file into the following objects + fields: top-level bpod fields + number_trials: total number of trials + trial_start_times: list of floats, seconds relative to session start + trial_types: array of tinyint designating condition number + """ + def __init__(self, full_dir): + try: # check for file in path + self.filepath = next(pathlib.Path(full_dir).glob('*.mat')) + except StopIteration: + raise FileNotFoundError(f'No .mat file found at: {full_dir}') + self.data = load_bpod_matfile(self.filepath) # see helper below + + @property + def fields(self): + if self._fields is None: + self._fields = list(self.data.keys()) + return self._fields + + @property + def number_trials(self): + if self._number_trials is None: + self._number_trials = self.data['nTrials'] + return self._number_trials + + @property + def trial_start_times(self): + if self._trial_start_times is None: + self._trial_start_times = list(self.data['TrialStartTimestamp']) + return self._trial_start_times + + @property + def trial_types(self): + if self._trial_types is None and 'TrialTypes' in self.fields: + self._trial_types = self.data['TrialTypes'] + return self._trial_types + + ''' + @property + def [each relevant bpod property](self)]: + if self.[relevant property] is None: + self.[relevant property] = self.file.[specific structure] + ''' + +# --------------------- HELPER LOADER FUNCTIONS ----------------- + + +def load_bpod_matfile(mat_filepath): + """ + Loading routine for behavioral file, bpod .mat + """ + # loadmat optionally takes mdict, existing dictionary which it loads into + # simplify_cells returns a simplified dict structure, instead of reversible + # mat-like files + # squeeze_me compresses matrix dimensions + mat_file = spio.loadmat(mat_filepath.as_posix(),squeeze_me=True, + struct_as_record=False,simplify_cells=True) + # bpod files load as dict with the following keys + # __header__ : mat version, flatform, creation date + # __version__ : file version + # __globals__ + # SessionData : mat_struct + if 'SessionData' in mat_file.keys(): + return mat_file['SessionData'] + else: + raise FileNotFoundError('.mat file missing SessionData' + + f'field at: {mat_filepath}') diff --git a/element_trial/readers/bpod_fields_notes.py b/element_trial/readers/bpod_fields_notes.py new file mode 100644 index 0000000..e800868 --- /dev/null +++ b/element_trial/readers/bpod_fields_notes.py @@ -0,0 +1,25 @@ +'''Temporary file for keeping track of bpod structures + +The RAW prefix in bpod structures seems to always have data elsewhere + Raw is where it is initially stored, and then ported elsewhere during export + +Inconsistent across examples: + BalbC top level: LeftValveTime, RightValveTime, LeftAmount, RightAmount,changebridge, bridgepos, subject, delays + 1119a top level: Notes, MarkerCodes + +Custom is determined by user: + 1119a Custom: contrastLevels, dbSplTrialType3, dbSplTrialType6,startSessionDatetime, startSessionTimeStamp,TQ03_Dual2AFC_Jun18_2021_Session1.mat, BlockNumber,BlockTrial, ChoiceLeft, ChoiceCorrect, Feedback, FeedbackTimeFixBroke, EarlyWithdrawal, FixDur, MT, CatchTrial, OdorFracA,OdorID, OdorPair, ST, ResolutionTime, Rewarded,RewardMagnitude, number_trials, LaserTrial,LaserTrialTrainStart, AuditoryTrial, ClickTask,OlfactometerStartup, PsychtoolboxStartup, AuditoryOmega,LeftClickRate, RightClickRate, LeftClickTrain,RightClickTrain, LeftRewarded, DV, Rig, Subject,PulsePalParamStimulus, PulsePalParamFeedback, StimDelay,FeedbackDelay, MinSampleAud, + TP24 Custom: BlockNumber, BlockTrial, ChoiceLeft, ChoiceCorrect, Feedback,FeedbackTime, FixBroke, EarlyWithdrawal, FixDur, MT,CatchTrial, OdorFracA, OdorID, OdorPair, ST, Rewarded,RewardMagnitude, number_trials, LaserTrial, LaserTrialTrainStart,AuditoryTrial, ClickTask, OlfactometerStartup,PsychtoolboxStartup, AuditoryOmega, LeftClickRate,RightClickRate, LeftClickTrain, RightClickTrain, LeftRewarded,DV, Rig, Subject, PulsePalParamStimulus, PulsePalParamFeedback,StimDelay, FeedbackDelay, MinSampleAud + +Settings is inconsistently used: + BalcC: Nothing + TQ03 : same as TrialSettings + +TrialSettings is also file-specific: + BalbC TrialSettings: Subject, RewardAmount, RewardDelay, ChangeOver, block, depleft, depright, MaxTrials, randomDelay, Lange, Bridge, Drugs + 1119A TrialSettings: MaxTrialNum, StartPhaseLength, TrialType1, TrialType2,TrialType3, TrialType4, TrialType5, TrialType6, TrialType7, RewardAmount, ContrastLevelMin, ContrastLevelMax, UseBonsai, UsePulsePal, PulsePalComPort, PulsePalParameterFile, UseOptoGeneticPulseLength, NumberOfPulses, PulseFrequency,Bpod_BNC_Ch_Opto, PulsePal_Trig_Ch_Opto, PulsePal_Out_Ch_Opto, Opto_Stim_In_TrialType2, Opto_Stim_In_TrialType3, Opto_Stim_In_TrialType5, Opto_Stim_In_TrialType6, UseSoundStimulation, dB_SPL_min, dB_SPL_max, dB_SPL_step, Photometry, DbleFibers, Isobestic405, RedChannel,PhotometryVersion, Modulation, NidaqDuration, NidaqSamplingRate, DecimateFactor, LED1_Name, LED1_Amp, LED1_Freq, LED2_Name, LED2_Amp, LED2_Freq, LED1b_Name, LED1b_Amp, LED1b_Freq + TQ03_ TrialSettings: Aud_Levels, Aud_NoEvidence, Aud_Ramp, Aud_SamplingRate, Aud_ToneDuration, Aud_ToneOverlap, Aud_UseMiddleOctave, Aud_Volume, Aud_maxFreq, Aud_minFreq, Aud_nFreq, AuditoryAlpha, AuditoryStimulusTime, AuditoryStimulusType, BaselineBegin, BaselineEnd, BlockTable, CatchError, CenterWaitMax, ChoiceDeadLine, DbleFibers, DecimateFactor, DrinkingGrace, DrinkingTime, FeedbackDelay, FeedbackDelayDecr, FeedbackDelayGrace, FeedbackDelayIncr, FeedbackDelayMax, FeedbackDelayMin, FeedbackDelaySelection, FeedbackDelayTau, ITI, IncorrectChoiceFeedbackType, Isobestic405, LED1_Amp, LED1_Freq, LED1_Name, LED1b_Amp, LED1b_Freq, LED1b_Name, LED2_Amp, LED2_Freq, LED2_Name, LaserAmp, LaserFeedback, LaserITI, LaserMov, LaserPreStim, LaserPulseDuration_ms, LaserRampDuration_ms, LaserRew, LaserSoftCode, LaserStim, LaserStimFreq, LaserTimeInvestment, LaserTrainDuration_ms, LaserTrainRandStart, LaserTrainStartMax_s, LaserTrainStartMin_s, LaserTrials, LeftBiasAud, MaxSessionTime, MinSampleAud, MinSampleAudAutoincrement, MinSampleAudDecr, MinSampleAudIncr, MinSampleAudMax, MinSampleAudMin, Modulation, NidaqDuration, NidaqMax, NidaqMin, NidaqSamplingRate, OdorA_bank, OdorB_bank, OdorStimulusTimeMin, OdorTable, Percent50Fifty, PercentAuditory, PercentCatch, PhotoPlotReward, PhotoPlotSidePokeIn, PhotoPlotSidePokeLeave, Photometry, PhotometryVersion, PortLEDs, Ports_LMR, PreITI, RedChannel, RewardAmount, ShowFeedback, ShowFix, ShowPsycAud, ShowPsycOlf, ShowST, ShowTrialRate, ShowVevaiometric, SkippedFeedbackFeedbackType, StartEasyTrials, StimDelay, StimDelayAutoincrement, StimDelayDecr, StimDelayIncr, StimDelayMax, StimDelayMin, SumRates, TimeMax, TimeMin, TimeOutBrokeFixation, TimeOutEarlyWithdrawal, TimeOutIncorrectChoice, TimeOutSkippedFeedback, TrialSelection, VevaiometricMinWT, VevaiometricNBin, VevaiometricShowPoints, VideoTrials, Wire1VideoTrigger, nidaqDev + TP24_ TrialSettings: Aud_Levels, Aud_NoEvidence, Aud_Ramp, Aud_SamplingRate, Aud_ToneDuration, Aud_ToneOverlap, Aud_UseMiddleOctave, Aud_Volume, Aud_maxFreq, Aud_minFreq, Aud_nFreq, AuditoryAlpha, AuditoryStimulusTime, AuditoryStimulusType, BlockTable, CatchError, ChoiceDeadLine, FeedbackDelay, FeedbackDelayDecr, FeedbackDelayGrace, FeedbackDelayIncr, FeedbackDelayMax, FeedbackDelayMin, FeedbackDelaySelection, FeedbackDelayTau, ITI, IncorrectChoiceFeedbackType, LaserFeedback, LaserITI, LaserMov, LaserPreStim, LaserPulseDuration_ms, LaserRew, LaserStim, LaserStimFreq, LaserTimeInvestment, LaserTrainDuration_ms, LaserTrainRandStart, LaserTrainStartMax_s, LaserTrainStartMin_s, LaserTrials, LeftBiasAud, MaxSessionTime, MinSampleAud, MinSampleAudAutoincrement, MinSampleAudDecr, MinSampleAudIncr, MinSampleAudMax, MinSampleAudMin, OdorA_bank, OdorB_bank, OdorStimulusTimeMin, OdorTable, Percent50Fifty, PercentAuditory, PercentCatch, PortLEDs, Ports_LMR, RewardAmount, ShowFeedback, ShowFix, ShowPsycAud, ShowPsycOlf, ShowST, ShowTrialRate, ShowVevaiometric, SkippedFeedbackFeedbackType, StartEasyTrials, StimDelay, StimDelayAutoincrement, StimDelayDecr, StimDelayIncr, StimDelayMax, StimDelayMin, SumRates, TimeOutBrokeFixation, TimeOutEarlyWithdrawal, TimeOutIncorrectChoice, TimeOutSkippedFeedback, TrialSelection, VevaiometricMinWT, VevaiometricNBin, VevaiometricShowPoints, Wire1VideoTrigger + +TrialTypes: array of condition? Not consistent across all trials +''' diff --git a/element_trial/trial.py b/element_trial/trial.py new file mode 100644 index 0000000..59d575d --- /dev/null +++ b/element_trial/trial.py @@ -0,0 +1,198 @@ +import datajoint as dj +import inspect +import importlib +import pathlib +from element_data_loader.utils import find_full_path +from .readers import bpod + +schema = dj.schema() + +_linking_module = None + + +def activate(schema_name, *, create_schema=True, create_tables=True, + linking_module=None): + """ + activate(schema_name, *, create_schema=True, create_tables=True, + linking_module=None) + :param schema_name: schema name on the database server to activate + the `behavior` element + :param create_schema: when True (default), create schema in the + database if it does not yet exist. + :param create_tables: when True (default), create tables in the + database if they do not yet exist. + :param linking_module: a module (or name) containing the required + dependencies to activate the `session` element: + Upstream tables: + + Session: parent table to ProbeInsertion, typically + identifying a recording session. + Functions: + + get_trial_root_dir() -> list + Retrieve the root data director(y/ies) with behavioral + recordings (e.g., bpod files) for all subject/sessions. + :return: a string for full path to the root data directory + + get_trial_sess_dir(session_key: dict) -> str + Retrieve the session directory containing the recording(s) + for a given Session + :param session_key: a dictionary of one Session `key` + :return: a string for full path to the session directory + """ + if isinstance(linking_module, str): + linking_module = importlib.import_module(linking_module) + assert inspect.ismodule(linking_module), "The argument 'dependency' must"\ + + " be a module or module name" + + schema.activate(schema_name, create_schema=create_schema, + create_tables=create_tables, + add_objects=linking_module.__dict__) + +# -------------- Functions required by the element-trial --------------- + + +def get_trial_root_dir() -> list: + """ + All data paths, directories in DataJoint Elements are recommended to be + stored as relative paths, with respect to some user-configured "root" + directory, which varies from machine to machine + + get_trial_root_dir() -> list + This user-provided function retrieves the list of possible root data + directories containing the behavioral data for all subjects/sessions + :return: a string for full path to the behavioral root data directory, + or list of strings for possible root data directories + """ + return _linking_module.get_trial_root_dir() + + +def get_trial_sess_dir(session_key: dict) -> str: + """ + get_trial_sess_dir(session_key: dict) -> str + Retrieve the session directory, with all recordings for a given Session + :param session_key: a dictionary of one Session `key` + :return: a string for full path to the session directory + """ + return _linking_module.get_trial_sess_dir(session_key) + +# ----------------------------- Table declarations ---------------------- + + +@schema +class TrialType(dj.Lookup): + definition = """ + trial_type: varchar(16) + --- + trial_type_description: varchar(256) + """ + + +@schema +class Trial(dj.Imported): + definition = """ + -> Session + trial : smallint # trial number (1-based indexing) + --- + start_time : float # (second) relative to recording start + stop_time : float # (second) relative to recording start + """ + + +@schema +class EventType(dj.Lookup): + definition = """ + event_type: varchar(16) + --- + event_type_description='': varchar(256) + """ + + +@schema +class EventTrialized(dj.Imported): + definition = """ # TRIAL SCHEMA, rename to Event + -> Trial + -> EventType + event_start_time : float # (second) relative to recording start + --- + event_end_time=null: float # (second) relative to recording start + """ + + +@schema +class Event(dj.Imported): + definition = """ # EVENT SCHEMA + -> Session + -> EventType + event_start_time : float # (second) relative to recording start + --- + event_end_time=null: float # (second) relative to recording start + """ + + +@schema +class TrialEvent(dj.Imported): + definition = """ + -> Trial + -> Event + """ + + +@schema +class BehaviorTrial(dj.Imported): + definition = """ + -> Trial + --- + -> TrialType + """ + + class TrialVariable(dj.Part): + definition = """ + -> master + variable_name: varchar(16) + --- + variable_value: varchar(2000) + """ + + +@schema +class BehaviorRecording(dj.Imported): + definition = """ + -> session.SessionDirectory + recording_id: varchar(16) + --- + recording_notes: varchar(256) + """ + + class BehFile(dj.Part): + definition = """ + -> master + filetype: varchar(16) + """ + + def make(self, key): + trial_root_dir = pathlib.Path(get_trial_root_dir(key)) + trial_sess_dir = pathlib.Path(get_trial_sess_dir(key)) + trial_dir_full = find_full_path(trial_root_dir, trial_sess_dir) + + for trial_pattern, trial_file_type in zip(['*mat', '*_u.csv'], + ['bpod', 'UnivariateCSV']): + trial_filepaths = [fp for fp in + trial_dir_full.rglob(trial_pattern)] + if trial_filepaths: + filetype = trial_file_type + break + else: + raise FileNotFoundError('Neither bpod mat file nor _u.csv file' + f' found in {trial_sess_dir}') + + if filetype == 'bpod': + for filepath in trial_filepaths: + bpod_data = bpod.BPod(filepath) + ''' + key['property'] = bpod_data.property + ''' + return bpod_data + elif filetype == 'UnivariateCSV': + for filepath in trial_filepaths: + pass + else: + raise NotImplementedError(f'{filetype} files of type are ' + + 'not yet supported') diff --git a/element_trial_behavior/version.py b/element_trial/version.py similarity index 100% rename from element_trial_behavior/version.py rename to element_trial/version.py diff --git a/element_trial_behavior/__init__.py b/element_trial_behavior/__init__.py deleted file mode 100644 index 5f3c8ce..0000000 --- a/element_trial_behavior/__init__.py +++ /dev/null @@ -1,68 +0,0 @@ - -## Much of contents ported over from element_array_ephys/__init__.py -## functions duplicated in element init.py and dataloader utils -## Should we reduce duplication, always point to utils? - -__author__ = "DataJoint" -__date__ = "November, 2021" -__version__ = "0.1.0c0" - -__all__ = ['__author__', '__version__', '__date__'] - - -import datajoint as dj -import pathlib - - -dj.config['enable_python_native_blobs'] = True - - -def find_full_path(root_directories, relative_path): - """ - Given a relative path, search and return the full-path - from provided potential root directories (in the given order) - :param root_directories: potential root directories - :param relative_path: the relative path to find the valid root directory - :return: root_directory (pathlib.Path object) - """ - relative_path = pathlib.Path(relative_path) - - if relative_path.exists(): - return relative_path - - # turn to list if only a single root directory is provided - if isinstance(root_directories, (str, pathlib.Path)): - root_directories = [root_directories] - - for root_dir in root_directories: - if (pathlib.Path(root_dir) / relative_path).exists(): - return pathlib.Path(root_dir) / relative_path - - raise FileNotFoundError('No valid full-path found (from {})' - ' for {}'.format(root_directories, relative_path)) - - -def find_root_directory(root_directories, full_path): - """ - Given multiple potential root directories and a full-path, - search and return one directory that is the parent of the given path - :param root_directories: potential root directories - :param full_path: the relative path to search the root directory - :return: full-path (pathlib.Path object) - """ - full_path = pathlib.Path(full_path) - - if not full_path.exists(): - raise FileNotFoundError(f'{full_path} does not exist!') - - # turn to list if only a single root directory is provided - if isinstance(root_directories, (str, pathlib.Path)): - root_directories = [root_directories] - - try: - return next(pathlib.Path(root_dir) for root_dir in root_directories - if pathlib.Path(root_dir) in set(full_path.parents)) - - except StopIteration: - raise FileNotFoundError('No valid root directory found (from {})' - ' for {}'.format(root_directories, full_path)) \ No newline at end of file diff --git a/element_trial_behavior/behavior_trials.py b/element_trial_behavior/behavior_trials.py deleted file mode 100644 index 783f8ea..0000000 --- a/element_trial_behavior/behavior_trials.py +++ /dev/null @@ -1,169 +0,0 @@ -import datajoint as dj -import inspect -import importlib - -schema = dj.schema() - -def activate(schema_name, *, create_schema=True, create_tables=True, linking_module=None): - """ - activate(schema_name, *, create_schema=True, create_tables=True, linking_module=None) - :param schema_name: schema name on the database server to activate the `behavior` element - :param create_schema: when True (default), create schema in the database if it does not yet exist. - :param create_tables: when True (default), create tables in the database if they do not yet exist. - :param linking_module: a module name or a module containing the required - dependencies to activate the `session` element: - Upstream tables: - + Session: parent table to ProbeInsertion, typically identifying a recording session. - Functions: - + get_beh_root_dir() -> list - Retrieve the root data directory/directories containing behavioral recordings - (e.g., bpod files) for all subject/sessions. - :return: a string for full path to the root data directory - + get_beh_sess_dir(session_key: dict) -> str - Retrieve the session directory containing the recording(s) for a given Session - :param session_key: a dictionary of one Session `key` - :return: a string for full path to the session directory - """ - if isinstance(linking_module, str): - linking_module = importlib.import_module(linking_module) - assert inspect.ismodule(linking_module), "The argument 'dependency' must be a module's name or a module" - - schema.activate(schema_name, create_schema=create_schema, - create_tables=create_tables, add_objects=linking_module.__dict__) - -# -------------- Functions required by the elements-ephys --------------- - -# CB: Ephys pulls from linking module, MAP defines internally - default to ephys style? - -def get_beh_root_dir() -> list: - """ - All data paths, directories in DataJoint Elements are recommended to be stored as - relative paths, with respect to some user-configured "root" directory, - which varies from machine to machine (e.g. different mounted drive locations) - - get_beh_root_dir() -> list - This user-provided function retrieves the list of possible root data directories - containing the behavioral data for all subjects/sessions (e.g., bpod files) - :return: a string for full path to the behavioral root data directory, - or list of strings for possible root data directories - """ - return _linking_module.get_beh_root_dir() - - -def get_beh_sess_dir(session_key: dict) -> str: - """ - get_beh_sess_dir(session_key: dict) -> str - Retrieve the session directory containing the recordings for a given Session - :param session_key: a dictionary of one Session `key` - :return: a string for full path to the session directory - """ - return _linking_module.get_beh_sess_dir(session_key) - -# ----------------------------- Table declarations ---------------------- - -@schema -class TrialType(dj.Lookup): - definition = """ - trial_type: varchar(16) - --- - trial_type_description: varchar(256) - """ - -@schema -class Trial(dj.Imported): - definition = """ - -> Session - trial : smallint # trial number (1-based indexing) - --- - start_time : float # (second) relative to the start of the recording - stop_time : float # (second) relative to the start of the recording - """ - -@schema -class EventType(dj.Lookup): - definition = """ - event_type: varchar(16) - --- - event_type_description='': varchar(256) - """ - -@schema -class Event(dj.Imported): - definition = """ - -> Event - -> EventType - event_start_time: decimal(8, 4) # (s) from recording start - --- - event_end_time=null: decimal(8, 4) # (s) from recording start - """ - -@schema -class TrialEvent(dj.Imported): - definition = """ - -> Trial - -> Event - """ - -@schema -class BehaviorTrial(dj.Imported): - definition = """ - -> Trial - --- - -> TrialType - """ - - class TrialVariable(dj.Part): - definition = """ - -> master - variable_name: varchar(16) - --- - variable_value: varchar(2000) - """ - -@schema -class BehaviorRecording(dj.Imported) - definition = """ - -> session.SessionDirectory - recording_id: varchar(16) - --- - recording_notes: varchar(256) - """ - - class BehFile(dj.Part): - definition = """ - -> master - filetype: varchar(16) - """ - - def make(self, key): - beh_root_dir = pathlib.Path(get_beh_root_dir(key)) - beh_sess_dir = pathlib.Path(get_beh_sess_dir(key)) - beh_dir_full = find_full_path(beh_root_dir,beh_sess_dir) - - for beh_pattern, beh_file_type in zip(['*mat','*XXX'], - ['bpod','Kepecs Standard']) - # CB: critique - assumes .mat is bpod - # I wanted to save trying to open the file, so the bpod init - # checks and errors out if not. Are there better ways? - beh_filepaths = [fp for fp in beh_dir_full.rglob(beh_pattern)] - if beh_filepaths: - filetype = beh_file_type - break - else: - raise FileNotFoundError( - f'Neither bpod mat file nor csv file found in {beh_sess_dir}') - - if filetype == 'bpod': - for filepath in beh_filepaths: - bpod_data = bpod.BPod(filepath) - ''' - key['property'] = bpod_data.property - ''' - elif filetype == 'Kepecs Standard': - for filepath in beh_filepaths: - pass - # Waiting on more info from Kepecs Lab - # elif filetype == "other standard??" what others? - else: - raise NotImplementedError(f'Behavioral file of type {filetype}' - ' is not yet implemented') diff --git a/element_trial_behavior/readers/bpod.py b/element_trial_behavior/readers/bpod.py deleted file mode 100644 index 390133f..0000000 --- a/element_trial_behavior/readers/bpod.py +++ /dev/null @@ -1,49 +0,0 @@ -import os -import pathlib - -import datajoint as dj -from . import find_full_path, find_root_directory - -import scipy.io as spio -import numpy as np - -class BPod: - - def __init__(self, full_dir): - # check for file - try: - self.filepath=next(pathlib.Path(full_dir).glob('*.mat')) - except StopIteration: - raise FileNotFoundError(f'No bpod .mat file found at: {full_dir}') - # check that .mat file found is bpod, by checking that it has a SessionData section - try: self.file=load_bpod_matfile(self.file) - except StopIteration: - raise FileNotFoundError(f'.mat file missing SessionData field at: {full_dir}') - - @property - def trialnumber(self): - if self._trialnumber is None: - self._trialnumber = self.file.nTrials - return self._trialnumber - - ''' - @property - def [each relevant bpod property](self)]: - if self.[relevant property] is None: - self.[relevant property] = self.file.[specific structure] - ''' - -# --------------------- HELPER LOADER FUNCTIONS ----------------- - -# see full example here: -# https://github.com/mesoscale-activity-map/map-ephys/blob/master/pipeline/ingest/behavior.py - -def load_bpod_matfile(dir, matlab_filepath): - """ - Loading routine for behavioral file, bpod .mat - """ - #Loading the file - SessionData = spio.loadmat(matlab_filepath.as_posix(), - squeeze_me=True, - struct_as_record=False)['SessionData'] - return SessionData \ No newline at end of file diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..40cda4c --- /dev/null +++ b/setup.py @@ -0,0 +1,30 @@ +from setuptools import setup, find_packages +from os import path + +pkg_name = next(p for p in find_packages() if '.' not in p) +here = path.abspath(path.dirname(__file__)) + +with open(path.join(here, 'README.md'), 'r') as f: + long_description = f.read() + +with open(path.join(here, 'requirements.txt')) as f: + requirements = f.read().splitlines() + +with open(path.join(here, pkg_name, 'version.py')) as f: + exec(f.read()) + +setup( + name=pkg_name.replace('_', '-'), + version=__version__, + description="DataJoint Elements for Trialized Experiments", + long_description=long_description, + long_description_content_type='text/markdown', + author='DataJoint', + author_email='info@datajoint.com', + license='MIT', + url=f'https://github.com/datajoint/{pkg_name.replace("_", "-")}', + keywords='neuroscience behavior bpod trials datajoint', + packages=find_packages(exclude=['contrib', 'docs', 'tests*']), + scripts=[], + install_requires=requirements, +) From cf7947e724f671e723bf3440d91b84fc07b56781 Mon Sep 17 00:00:00 2001 From: Chris Broz Date: Wed, 5 Jan 2022 11:08:42 -0600 Subject: [PATCH 02/19] more bpod notes --- element_trial/readers/bpod_fields_notes.py | 78 ++++++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/element_trial/readers/bpod_fields_notes.py b/element_trial/readers/bpod_fields_notes.py index e800868..caa667c 100644 --- a/element_trial/readers/bpod_fields_notes.py +++ b/element_trial/readers/bpod_fields_notes.py @@ -22,4 +22,82 @@ TP24_ TrialSettings: Aud_Levels, Aud_NoEvidence, Aud_Ramp, Aud_SamplingRate, Aud_ToneDuration, Aud_ToneOverlap, Aud_UseMiddleOctave, Aud_Volume, Aud_maxFreq, Aud_minFreq, Aud_nFreq, AuditoryAlpha, AuditoryStimulusTime, AuditoryStimulusType, BlockTable, CatchError, ChoiceDeadLine, FeedbackDelay, FeedbackDelayDecr, FeedbackDelayGrace, FeedbackDelayIncr, FeedbackDelayMax, FeedbackDelayMin, FeedbackDelaySelection, FeedbackDelayTau, ITI, IncorrectChoiceFeedbackType, LaserFeedback, LaserITI, LaserMov, LaserPreStim, LaserPulseDuration_ms, LaserRew, LaserStim, LaserStimFreq, LaserTimeInvestment, LaserTrainDuration_ms, LaserTrainRandStart, LaserTrainStartMax_s, LaserTrainStartMin_s, LaserTrials, LeftBiasAud, MaxSessionTime, MinSampleAud, MinSampleAudAutoincrement, MinSampleAudDecr, MinSampleAudIncr, MinSampleAudMax, MinSampleAudMin, OdorA_bank, OdorB_bank, OdorStimulusTimeMin, OdorTable, Percent50Fifty, PercentAuditory, PercentCatch, PortLEDs, Ports_LMR, RewardAmount, ShowFeedback, ShowFix, ShowPsycAud, ShowPsycOlf, ShowST, ShowTrialRate, ShowVevaiometric, SkippedFeedbackFeedbackType, StartEasyTrials, StimDelay, StimDelayAutoincrement, StimDelayDecr, StimDelayIncr, StimDelayMax, StimDelayMin, SumRates, TimeOutBrokeFixation, TimeOutEarlyWithdrawal, TimeOutIncorrectChoice, TimeOutSkippedFeedback, TrialSelection, VevaiometricMinWT, VevaiometricNBin, VevaiometricShowPoints, Wire1VideoTrigger TrialTypes: array of condition? Not consistent across all trials + +['SessionData'].RawEvents.Trial[0].States +BalbC_Ph_W1_LH_Randdelay_changeover_Aug08_2021_Session1.mat +RawEvents +RawData + +Bonn bpod SessionData structure + TrialTypes - 1,2,3,1,2,3 + TrialTypeNames - Visibile,Visible,Fading + Info + StateMachineVersion + SessionDate + SessionStartTime_UTC + SessionStartTime_MATLAB + nTrials (# trials in session, here 54) + RawEvents (timestamps for each trial's state transitions/recorded events) + Trial{1,n}.States #Which of these are important? + WaitForPosTriggerSoftCode + CueDelay + WaitForResponse + Port2RewardDelay + Port2Reward + CloseValves + Drinking + Port1RewardDelay + Port3RewardDelay + Port4RewardDelay + Port5RewardDelay + Port6RewardDelay + Port7RewardDelay + Port8RewardDelay + Port1Reward + Port3Reward + Port4Reward + Port5Reward + Port6Reward + Port7Reward + Port8Reward + Punish + Punishexit + EarlyWithdrawal + Trial{1,n}.Events + Port4In + Port4Out + SoftCode10 + Tup + Port2In + Port2Out + RawData (copy of raw data from state machine) + TrialStartTimestamp (time when trial started on Bpod's clock) + Note: Timestamps in RawEvents are relative to each trial's start + TrialEndTimestamp + SettingsFile (the settings file you selected in the launch manager) + Notes + MarkerCodes + CurrentSubjectName + TrialSettings + GUI + GUIMeta + GUIPanels + polling + debug + debugvis + Data + arm_number + arm_baited_orig + arm_baited + SF + rotation + position + StimAlpha + StimPos + TriggerLocPix + TriggerLocOptitrackHitbox + TriggerLocOptitrackCenter + TriggerLocOptitrackCircleHitRadius + tform +''' ''' From 8981c2d7a1b702a7bfc09366ac28ad28f230b4f2 Mon Sep 17 00:00:00 2001 From: Chris Broz Date: Wed, 5 Jan 2022 12:23:35 -0600 Subject: [PATCH 03/19] minor nwb comment edits --- element_trial/export/__init__.py | 7 - element_trial/export/nwb.py | 158 ++++++++++----------- element_trial/readers/bpod_fields_notes.py | 1 - 3 files changed, 78 insertions(+), 88 deletions(-) diff --git a/element_trial/export/__init__.py b/element_trial/export/__init__.py index 0fa2104..8d41ba5 100644 --- a/element_trial/export/__init__.py +++ b/element_trial/export/__init__.py @@ -5,10 +5,3 @@ __version__ = "0.1.0c0" __all__ = ['__author__', '__version__', '__date__'] - - -import datajoint as dj -import pathlib - -## https://github.com/nwb-extensions/ndx-events-record - diff --git a/element_trial/export/nwb.py b/element_trial/export/nwb.py index 0a13cb6..7e89675 100644 --- a/element_trial/export/nwb.py +++ b/element_trial/export/nwb.py @@ -1,85 +1,83 @@ -import numpy as np -from datetime import datetime -from dateutil.tz import tzlocal from pynwb import NWBFile from element_session import session +from element_trial import trial + def trial_to_nwb(trial_key): - scan_query = trial.TrialEvent & trial_key - # scan_query = scan_query.join(XXX, left=True) - lab_info = lab.query.fetch1() - - return NWBFile( - # https://nwb-schema.readthedocs.io/en/latest/format.html#optical-physiology - - ) - -""" From CHEN 2017 - - - # =============================================================================== - # =============================== BEHAVIOR TRIALS =============================== - # =============================================================================== - - # =============== TrialSet ==================== - # NWB 'trial' (of type dynamic table) by default comes with three mandatory attributes: 'start_time' and 'stop_time' - # Other trial-related information needs to be added in to the trial-table as additional columns (with column name - # and column description) - - dj_trial = experiment.SessionTrial * experiment.BehaviorTrial - skip_adding_columns = experiment.Session.primary_key + ['trial_uid', 'trial'] - - if experiment.SessionTrial & session_key: - # Get trial descriptors from TrialSet.Trial and TrialStimInfo - trial_columns = [{'name': tag, - 'description': re.sub('\s+:|\s+', ' ', re.search( - f'(?<={tag})(.*)', str(dj_trial.heading)).group()).strip()} - for tag in dj_trial.heading.names - if tag not in skip_adding_columns + ['start_time', 'stop_time']] - - # Add new table columns to nwb trial-table for trial-label - for c in trial_columns: - nwbfile.add_trial_column(**c) - - # Add entry to the trial-table - for trial in (dj_trial & session_key).fetch(as_dict=True): - trial['start_time'] = float(trial['start_time']) - trial['stop_time'] = float(trial['stop_time']) if trial['stop_time'] else 5.0 - [trial.pop(k) for k in skip_adding_columns] - trial['early_lick'] = True if trial['early_lick'] == 'early' else False - nwbfile.add_trial(**trial) - - # =============================================================================== - # =============================== BEHAVIOR TRIAL EVENTS ========================== - # =============================================================================== - - behavior = nwbfile.create_processing_module( - 'behavior', 'Time of behavioral events in this session') - behav_event = pynwb.behavior.BehavioralEvents(name='BehavioralEvents') - behavior.add_data_interface(behav_event) - - for trial_event_type in \ - (experiment.TrialEventType & \ - experiment.TrialEvent & session_key).fetch('trial_event_type'): - event_times, trial_starts = \ - (experiment.TrialEvent * experiment.SessionTrial - & session_key & {'trial_event_type': trial_event_type}).fetch( - 'trial_event_time', 'start_time') - - if trial_event_type == 'sample': - description = 'Time stamps of the beginning of the sampling period on each trial.' - elif trial_event_type == 'delay': - description = 'Time stamps of the beginning of the delay period on each trial.' - elif trial_event_type == 'go': - description = 'Time stamps of the go cue signal on each trial.' - - if len(event_times) > 0: - event_times = np.hstack(event_times.astype(float) + trial_starts.astype(float)) - behav_event.create_timeseries( - name=trial_event_type, unit='a.u.', conversion=1.0, - data=np.full_like(event_times, 1), - timestamps=event_times, - description=description) - -""" \ No newline at end of file + scan_query = session.Session & trial.TrialEvent & trial_key + # https://github.com/nwb-extensions/ndx-events-record + return NWBFile(scan_query) + + +""" From CHEN 2017 https://github.com/vathes/DJ-NWB-Chen-2017 + + +# =========================================================================== +# ============================= BEHAVIOR TRIALS ============================= +# =========================================================================== + +# =============== TrialSet ==================== +# NWB 'trial' (of type dynamic table) by default comes with three mandatory +# attributes: 'start_time' and 'stop_time'. Other trial-related information +# needs to be added in to the trial-table as additional columns (with column +# name and column description) + +dj_trial = experiment.SessionTrial * experiment.BehaviorTrial +skip_adding_columns = experiment.Session.primary_key + ['trial_uid', 'trial'] + +if experiment.SessionTrial & session_key: + # Get trial descriptors from TrialSet.Trial and TrialStimInfo + trial_columns = [{'name': tag, + 'description': re.sub('\s+:|\s+', ' ', re.search( + f'(?<={tag})(.*)', str(dj_trial.heading)).group()).strip()} + for tag in dj_trial.heading.names + if tag not in skip_adding_columns + ['start_time', 'stop_time']] + + # Add new table columns to nwb trial-table for trial-label + for c in trial_columns: + nwbfile.add_trial_column(**c) + + # Add entry to the trial-table + for trial in (dj_trial & session_key).fetch(as_dict=True): + trial['start_time'] = float(trial['start_time']) + trial['stop_time'] = (float(trial['stop_time']) if + trial['stop_time'] else 5.0) + [trial.pop(k) for k in skip_adding_columns] + trial['early_lick'] = True if trial['early_lick'] == 'early' else False + nwbfile.add_trial(**trial) + +# =========================================================================== +# ============================= BEHAVIOR TRIAL EVENTS ======================= +# =========================================================================== + +behavior = nwbfile.create_processing_module( + 'behavior', 'Time of behavioral events in this session') +behav_event = pynwb.behavior.BehavioralEvents(name='BehavioralEvents') +behavior.add_data_interface(behav_event) + +for trial_event_type in \ + (experiment.TrialEventType & \ + experiment.TrialEvent & session_key).fetch('trial_event_type'): + event_times, trial_starts = \ + (experiment.TrialEvent * experiment.SessionTrial + & session_key & {'trial_event_type': trial_event_type}).fetch( + 'trial_event_time', 'start_time') + + if trial_event_type == 'sample': + description = 'Timestamps: beginning of the sampling on each trial.' + elif trial_event_type == 'delay': + description = 'Timestamps: beginning of the delay on each trial.' + elif trial_event_type == 'go': + description = 'Time stamps of the go cue signal on each trial.' + + if len(event_times) > 0: + event_times = np.hstack(event_times.astype(float) + + trial_starts.astype(float)) + behav_event.create_timeseries( + name=trial_event_type, unit='a.u.', conversion=1.0, + data=np.full_like(event_times, 1), + timestamps=event_times, + description=description) + +""" diff --git a/element_trial/readers/bpod_fields_notes.py b/element_trial/readers/bpod_fields_notes.py index caa667c..f9481c7 100644 --- a/element_trial/readers/bpod_fields_notes.py +++ b/element_trial/readers/bpod_fields_notes.py @@ -100,4 +100,3 @@ TriggerLocOptitrackCircleHitRadius tform ''' -''' From ab9f035670862e42d8b7d274fc484857aa42ac56 Mon Sep 17 00:00:00 2001 From: Chris Brozdowski Date: Wed, 5 Jan 2022 16:27:29 -0600 Subject: [PATCH 04/19] minor edit to readme --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 303e3b1..5f28d33 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,8 @@ This repository is a work in progress not yet ready for public release. It serves as a draft of a DataJoint element for trialized experiments behavior for our U24 itiative. +Work in progress. + ## Schemas `trial` should be activated if all events are tied to specific trials. `event` should be activated if events occur independently of trials. From dfa41c9416659afaade227c2e3309dab05d033d5 Mon Sep 17 00:00:00 2001 From: Chris Broz Date: Fri, 7 Jan 2022 18:26:22 -0600 Subject: [PATCH 05/19] README update pointing to djarchive for example data --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 5f28d33..075326f 100644 --- a/README.md +++ b/README.md @@ -20,8 +20,8 @@ event table. MATLAB is the only officially supported environment. SciPy has a lot of support for loading matlab files. In the current draft, SciPy is used to import Bpod .mat files as embedded dictionaries. Micheal Wulf shared the Bpod files that -are currently in `workflow-trial/user_data/Bpod_files` as a diverse set of -examples. +are currently available via djarchive as `workflow-trial`, revision 0.0.0b1. +They were shared as a diverse set of examples. Under `element_trial/readers/`, there are two files: `Bpod.py` and `Bpod_fields_notes.py`. The former is a draft of an eventual reader that could From 8342d9e22eefceaa56d7f64ebedaecbaec63b09c Mon Sep 17 00:00:00 2001 From: Chris Brozdowski Date: Mon, 24 Jan 2022 12:07:02 -0600 Subject: [PATCH 06/19] Apply suggestions from code review Co-authored-by: Kabilar Gunalan --- element_trial/__init__.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/element_trial/__init__.py b/element_trial/__init__.py index 6ff17f9..8bbacb6 100644 --- a/element_trial/__init__.py +++ b/element_trial/__init__.py @@ -1,12 +1,3 @@ -__author__ = "DataJoint" -__date__ = "December, 2021" -__version__ = "0.1.0c0" - -__all__ = ['__author__', '__version__', '__date__'] - -import datajoint as dj -dj.config['enable_python_native_blobs'] = True - """ Two schemas, trial and event. Trial for fully trialized, segmented. Event for events independent of trials, like an act of naturaistic behavior. From a420a2c8b5f0ff361112f2b6b6cac717fc448ab1 Mon Sep 17 00:00:00 2001 From: Chris Broz Date: Mon, 24 Jan 2022 13:16:40 -0600 Subject: [PATCH 07/19] Revert version to alpha; refactor README --- CHANGELOG.md | 2 +- README.md | 136 ++++++++++++++------------------------- element_trial/version.py | 2 +- 3 files changed, 50 insertions(+), 90 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ed66e14..4d98f3c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,6 @@ Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and ### Added + First beta release -## [0.1.0c0] - 2021-11-15 +## [0.1.0a0] - 2021-11-15 ### Added + First draft begins diff --git a/README.md b/README.md index 075326f..67d295f 100644 --- a/README.md +++ b/README.md @@ -1,102 +1,62 @@ -# element-trial +# DataJoint Element - Experimental trials This repository is a work in progress not yet ready for public release. It serves as a draft of a DataJoint element for trialized experiments behavior for our U24 itiative. Work in progress. -## Schemas -`trial` should be activated if all events are tied to specific trials. -`event` should be activated if events occur independently of trials. - -The current draft of `trial` contains all proposed tables. When we're sure of -the contents, including make functions, one would copy the contents of the -`trial` schema over to events, and then remove the trialized version of the -event table. - -## Filetypes - -### Bpod -MATLAB is the only officially supported environment. SciPy has a lot of support -for loading matlab files. In the current draft, SciPy is used to import Bpod -.mat files as embedded dictionaries. Micheal Wulf shared the Bpod files that -are currently available via djarchive as `workflow-trial`, revision 0.0.0b1. -They were shared as a diverse set of examples. - -Under `element_trial/readers/`, there are two files: `Bpod.py` and -`Bpod_fields_notes.py`. The former is a draft of an eventual reader that could -be folded into `element_data_loader`. It handles general ingestion for fields -that are consisitent across examples. The latter highlights just how different -Bpod files can be. A fully realized loader may need to pull directly from -Bpod 'raw' fields to reconstruct the experiement, as many other fields are -organized differently across trials. - -Development for this reader was partially conducted within -`workflow_trial/notebooks/1_.ipynb`. This contains code snipits for exploring -the loaded Bpod data. - -Ecosystem: The SanWorks team -([git repositories here](https://github.com/sanworks?tab=repositories)) offers -no offical python support, suggesting instead that users -[export to JSON](https://sanworks.io/forum/showthread.php?tid=626&pid=1169). -The PyBpod project ([github](https://github.com/pyBpod/pyBpod), -[docs](https://pyBpod.readthedocs.io/en/v1.8.1/)) offers a python-based GUI -alternative for running Bpod hardware. So far as I could tell, they do not -offer any usefull ingestion functions we would want to incorporate. - - -### Alternate generalized format - -We previously discussed also supporing a univariate csv format that would -contain 4 values: timestamp on, timestamp off, data type and value. This has -not yet been implemented. - -## To do: -- [X] Support functions - - [X] Pull `find_full_path` and `find_root_directory` from `element-data-loader.utils`. - - [X] To find root/session dirs, refer to config file if exists. If not, linking module -- [ ] Table definitions - - [X] Discuss table structure - - [X] Decide supported filetypes: - - Bpod - - Generalizable 'Univariate' CSV: timestamp on, off, data type, value - - [ ] Develop ingestion functions -- [ ] Test tables with example data - - Aeon, bonsai system API - - Kepecs group provided Bpod files -- [X] Contact the [Bpod team](https://github.com/sanworks/) - - [X] Already an implementation of loading to Python - No - - [X] Create joint sustainability roadmap - Unlikely -- [ ] Analysis package - - [ ] Load processed data to table structure - - [ ] Trigger analysis on raw data import -- [ ] Quality control metrics -- [ ] GitHub Actions for PyPI release -- [ ] example workflow - - [ ] Integration tests with pytest - - [ ] Tutorials in text format (i.e. Jupyter notebook) - - [ ] Tutorial in video format - - [ ] Docker for tests - - [X] Example dataset(s) for public release - - [ ] Example dataset loaded to DJ Archive - - [ ] NWB export - - [ ] README -- [ ] RRID +## Element architecture + +In both of the following diagrams, the trial element starts immediately downstream from ***Session***. In one case, Sessions are first segmented into trials, and then segmented into events. This might be appropriate, for example, in a paradigm with repeated conditions and response behaviors associated with different conditions. In the next, Sessions are directly upstream from Events. This might be appropropriate for a paradigm that recorded events within naturalistic free behavior. +We provide an [example workflow](https://github.com/datajoint/workflow-trial/) with a +[pipeline script](https://github.com/datajoint/workflow-trial/blob/main/workflow_trial/pipeline.py) that models combining this Element with the corresponding [Element-Session](https://github.com/datajoint/element-session). + + + ## Installation -``` -pip install element-trial -``` ++ Install `element-trial` + ``` + pip install element-trial + ``` -if you have an older version of ***element-trial*** installed, using `pip`, upgrade with ++ Upgrade `element-trial` previously installed with `pip` + ``` + pip install --upgrade element-trial + ``` -``` -pip install --upgrade element-trial -``` + + +## Usage + +### Element activation + +To activate the `element-trial`, one need to provide: + +1. Schema names for the event or trial module + +2. Upstream Session table: A set of keys identifying a recording session (see [Element-Session](https://github.com/datajoint/element-session)). +3. Utility functions. See example definitions here](https://github.com/datajoint/workflow-trial/blob/main/workflow_trial/paths... + +For more detail, check the docstring of the `element-trial`: +```python +from element_trial import event, trial +help(event.activate) +help(trial.activate) ``` -element-data-loader @ git+https://github.com/datajoint/element-data-loader -``` + +### Example usage + +See [this project](https://github.com/datajoint/workflow-trial) for an example usage of this Array Electrophysiology Element. diff --git a/element_trial/version.py b/element_trial/version.py index 5677c7a..71bd74d 100644 --- a/element_trial/version.py +++ b/element_trial/version.py @@ -1,2 +1,2 @@ """Package metadata.""" -__version__ = '0.1.0c0' \ No newline at end of file +__version__ = '0.1.0a0' From e3bbd89ecafe1de469852469e93d31320d1048cf Mon Sep 17 00:00:00 2001 From: Chris Broz Date: Mon, 24 Jan 2022 13:49:25 -0600 Subject: [PATCH 08/19] git mv for comparison. formatting --- element_trial/export/__init__.py | 7 - element_trial/export/nwb.py | 5 - element_trial/readers/bpod_fields_notes.py | 250 ++++++++++++++------- 3 files changed, 169 insertions(+), 93 deletions(-) diff --git a/element_trial/export/__init__.py b/element_trial/export/__init__.py index 8d41ba5..e69de29 100644 --- a/element_trial/export/__init__.py +++ b/element_trial/export/__init__.py @@ -1,7 +0,0 @@ - - -__author__ = "DataJoint" -__date__ = "November, 2021" -__version__ = "0.1.0c0" - -__all__ = ['__author__', '__version__', '__date__'] diff --git a/element_trial/export/nwb.py b/element_trial/export/nwb.py index 7e89675..fdd484f 100644 --- a/element_trial/export/nwb.py +++ b/element_trial/export/nwb.py @@ -12,11 +12,6 @@ def trial_to_nwb(trial_key): """ From CHEN 2017 https://github.com/vathes/DJ-NWB-Chen-2017 - -# =========================================================================== -# ============================= BEHAVIOR TRIALS ============================= -# =========================================================================== - # =============== TrialSet ==================== # NWB 'trial' (of type dynamic table) by default comes with three mandatory # attributes: 'start_time' and 'stop_time'. Other trial-related information diff --git a/element_trial/readers/bpod_fields_notes.py b/element_trial/readers/bpod_fields_notes.py index f9481c7..3fa5224 100644 --- a/element_trial/readers/bpod_fields_notes.py +++ b/element_trial/readers/bpod_fields_notes.py @@ -1,25 +1,113 @@ '''Temporary file for keeping track of bpod structures The RAW prefix in bpod structures seems to always have data elsewhere - Raw is where it is initially stored, and then ported elsewhere during export + Raw is where it is initially stored, and then ported elsewhere during export Inconsistent across examples: - BalbC top level: LeftValveTime, RightValveTime, LeftAmount, RightAmount,changebridge, bridgepos, subject, delays - 1119a top level: Notes, MarkerCodes + BalbC top level: LeftValveTime, RightValveTime, LeftAmount, RightAmount, + changebridge, bridgepos, subject, delays + 1119a top level: Notes, MarkerCodes Custom is determined by user: - 1119a Custom: contrastLevels, dbSplTrialType3, dbSplTrialType6,startSessionDatetime, startSessionTimeStamp,TQ03_Dual2AFC_Jun18_2021_Session1.mat, BlockNumber,BlockTrial, ChoiceLeft, ChoiceCorrect, Feedback, FeedbackTimeFixBroke, EarlyWithdrawal, FixDur, MT, CatchTrial, OdorFracA,OdorID, OdorPair, ST, ResolutionTime, Rewarded,RewardMagnitude, number_trials, LaserTrial,LaserTrialTrainStart, AuditoryTrial, ClickTask,OlfactometerStartup, PsychtoolboxStartup, AuditoryOmega,LeftClickRate, RightClickRate, LeftClickTrain,RightClickTrain, LeftRewarded, DV, Rig, Subject,PulsePalParamStimulus, PulsePalParamFeedback, StimDelay,FeedbackDelay, MinSampleAud, - TP24 Custom: BlockNumber, BlockTrial, ChoiceLeft, ChoiceCorrect, Feedback,FeedbackTime, FixBroke, EarlyWithdrawal, FixDur, MT,CatchTrial, OdorFracA, OdorID, OdorPair, ST, Rewarded,RewardMagnitude, number_trials, LaserTrial, LaserTrialTrainStart,AuditoryTrial, ClickTask, OlfactometerStartup,PsychtoolboxStartup, AuditoryOmega, LeftClickRate,RightClickRate, LeftClickTrain, RightClickTrain, LeftRewarded,DV, Rig, Subject, PulsePalParamStimulus, PulsePalParamFeedback,StimDelay, FeedbackDelay, MinSampleAud + 1119a Custom: contrastLevels, dbSplTrialType3, dbSplTrialType6,startSessionDatetime, + startSessionTimeStamp,TQ03_Dual2AFC_Jun18_2021_Session1.mat, + BlockNumber, BlockTrial, ChoiceLeft, ChoiceCorrect, Feedback, + FeedbackTimeFixBroke, EarlyWithdrawal, FixDur, MT, CatchTrial, + OdorFracA, OdorID, OdorPair, ST, ResolutionTime, Rewarded, + RewardMagnitude, number_trials, LaserTrial,LaserTrialTrainStart, + AuditoryTrial, ClickTask,OlfactometerStartup, PsychtoolboxStartup, + AuditoryOmega,LeftClickRate, RightClickRate, LeftClickTrain, + RightClickTrain, LeftRewarded, DV, Rig, Subject, + PulsePalParamStimulus, PulsePalParamFeedback, StimDelay, + FeedbackDelay, MinSampleAud, + TP24 Custom: BlockNumber, BlockTrial, ChoiceLeft, ChoiceCorrect, Feedback, + FeedbackTime, FixBroke, EarlyWithdrawal, FixDur, MT,CatchTrial, + OdorFracA, OdorID, OdorPair, ST, Rewarded,RewardMagnitude, + number_trials, LaserTrial, LaserTrialTrainStart,AuditoryTrial, + ClickTask, OlfactometerStartup,PsychtoolboxStartup, AuditoryOmega, + LeftClickRate,RightClickRate, LeftClickTrain, RightClickTrain, + LeftRewarded,DV, Rig, Subject, PulsePalParamStimulus, + PulsePalParamFeedback,StimDelay, FeedbackDelay, MinSampleAud Settings is inconsistently used: - BalcC: Nothing - TQ03 : same as TrialSettings + BalcC: Nothing + TQ03 : same as TrialSettings TrialSettings is also file-specific: - BalbC TrialSettings: Subject, RewardAmount, RewardDelay, ChangeOver, block, depleft, depright, MaxTrials, randomDelay, Lange, Bridge, Drugs - 1119A TrialSettings: MaxTrialNum, StartPhaseLength, TrialType1, TrialType2,TrialType3, TrialType4, TrialType5, TrialType6, TrialType7, RewardAmount, ContrastLevelMin, ContrastLevelMax, UseBonsai, UsePulsePal, PulsePalComPort, PulsePalParameterFile, UseOptoGeneticPulseLength, NumberOfPulses, PulseFrequency,Bpod_BNC_Ch_Opto, PulsePal_Trig_Ch_Opto, PulsePal_Out_Ch_Opto, Opto_Stim_In_TrialType2, Opto_Stim_In_TrialType3, Opto_Stim_In_TrialType5, Opto_Stim_In_TrialType6, UseSoundStimulation, dB_SPL_min, dB_SPL_max, dB_SPL_step, Photometry, DbleFibers, Isobestic405, RedChannel,PhotometryVersion, Modulation, NidaqDuration, NidaqSamplingRate, DecimateFactor, LED1_Name, LED1_Amp, LED1_Freq, LED2_Name, LED2_Amp, LED2_Freq, LED1b_Name, LED1b_Amp, LED1b_Freq - TQ03_ TrialSettings: Aud_Levels, Aud_NoEvidence, Aud_Ramp, Aud_SamplingRate, Aud_ToneDuration, Aud_ToneOverlap, Aud_UseMiddleOctave, Aud_Volume, Aud_maxFreq, Aud_minFreq, Aud_nFreq, AuditoryAlpha, AuditoryStimulusTime, AuditoryStimulusType, BaselineBegin, BaselineEnd, BlockTable, CatchError, CenterWaitMax, ChoiceDeadLine, DbleFibers, DecimateFactor, DrinkingGrace, DrinkingTime, FeedbackDelay, FeedbackDelayDecr, FeedbackDelayGrace, FeedbackDelayIncr, FeedbackDelayMax, FeedbackDelayMin, FeedbackDelaySelection, FeedbackDelayTau, ITI, IncorrectChoiceFeedbackType, Isobestic405, LED1_Amp, LED1_Freq, LED1_Name, LED1b_Amp, LED1b_Freq, LED1b_Name, LED2_Amp, LED2_Freq, LED2_Name, LaserAmp, LaserFeedback, LaserITI, LaserMov, LaserPreStim, LaserPulseDuration_ms, LaserRampDuration_ms, LaserRew, LaserSoftCode, LaserStim, LaserStimFreq, LaserTimeInvestment, LaserTrainDuration_ms, LaserTrainRandStart, LaserTrainStartMax_s, LaserTrainStartMin_s, LaserTrials, LeftBiasAud, MaxSessionTime, MinSampleAud, MinSampleAudAutoincrement, MinSampleAudDecr, MinSampleAudIncr, MinSampleAudMax, MinSampleAudMin, Modulation, NidaqDuration, NidaqMax, NidaqMin, NidaqSamplingRate, OdorA_bank, OdorB_bank, OdorStimulusTimeMin, OdorTable, Percent50Fifty, PercentAuditory, PercentCatch, PhotoPlotReward, PhotoPlotSidePokeIn, PhotoPlotSidePokeLeave, Photometry, PhotometryVersion, PortLEDs, Ports_LMR, PreITI, RedChannel, RewardAmount, ShowFeedback, ShowFix, ShowPsycAud, ShowPsycOlf, ShowST, ShowTrialRate, ShowVevaiometric, SkippedFeedbackFeedbackType, StartEasyTrials, StimDelay, StimDelayAutoincrement, StimDelayDecr, StimDelayIncr, StimDelayMax, StimDelayMin, SumRates, TimeMax, TimeMin, TimeOutBrokeFixation, TimeOutEarlyWithdrawal, TimeOutIncorrectChoice, TimeOutSkippedFeedback, TrialSelection, VevaiometricMinWT, VevaiometricNBin, VevaiometricShowPoints, VideoTrials, Wire1VideoTrigger, nidaqDev - TP24_ TrialSettings: Aud_Levels, Aud_NoEvidence, Aud_Ramp, Aud_SamplingRate, Aud_ToneDuration, Aud_ToneOverlap, Aud_UseMiddleOctave, Aud_Volume, Aud_maxFreq, Aud_minFreq, Aud_nFreq, AuditoryAlpha, AuditoryStimulusTime, AuditoryStimulusType, BlockTable, CatchError, ChoiceDeadLine, FeedbackDelay, FeedbackDelayDecr, FeedbackDelayGrace, FeedbackDelayIncr, FeedbackDelayMax, FeedbackDelayMin, FeedbackDelaySelection, FeedbackDelayTau, ITI, IncorrectChoiceFeedbackType, LaserFeedback, LaserITI, LaserMov, LaserPreStim, LaserPulseDuration_ms, LaserRew, LaserStim, LaserStimFreq, LaserTimeInvestment, LaserTrainDuration_ms, LaserTrainRandStart, LaserTrainStartMax_s, LaserTrainStartMin_s, LaserTrials, LeftBiasAud, MaxSessionTime, MinSampleAud, MinSampleAudAutoincrement, MinSampleAudDecr, MinSampleAudIncr, MinSampleAudMax, MinSampleAudMin, OdorA_bank, OdorB_bank, OdorStimulusTimeMin, OdorTable, Percent50Fifty, PercentAuditory, PercentCatch, PortLEDs, Ports_LMR, RewardAmount, ShowFeedback, ShowFix, ShowPsycAud, ShowPsycOlf, ShowST, ShowTrialRate, ShowVevaiometric, SkippedFeedbackFeedbackType, StartEasyTrials, StimDelay, StimDelayAutoincrement, StimDelayDecr, StimDelayIncr, StimDelayMax, StimDelayMin, SumRates, TimeOutBrokeFixation, TimeOutEarlyWithdrawal, TimeOutIncorrectChoice, TimeOutSkippedFeedback, TrialSelection, VevaiometricMinWT, VevaiometricNBin, VevaiometricShowPoints, Wire1VideoTrigger + BalbC TrialSettings: Subject, RewardAmount, RewardDelay, ChangeOver, block, depleft, + depright, MaxTrials, randomDelay, Lange, Bridge, Drugs + 1119A TrialSettings: MaxTrialNum, StartPhaseLength, TrialType1, TrialType2, + TrialType3, TrialType4, TrialType5, TrialType6, TrialType7, + RewardAmount, ContrastLevelMin, ContrastLevelMax, UseBonsai, + UsePulsePal, PulsePalComPort, PulsePalParameterFile, + UseOptoGeneticPulseLength, NumberOfPulses, PulseFrequency, + Bpod_BNC_Ch_Opto, PulsePal_Trig_Ch_Opto, PulsePal_Out_Ch_Opto, + Opto_Stim_In_TrialType2, Opto_Stim_In_TrialType3, + Opto_Stim_In_TrialType5, Opto_Stim_In_TrialType6, + UseSoundStimulation, dB_SPL_min, dB_SPL_max, dB_SPL_step, + Photometry, DbleFibers, Isobestic405, RedChannel, + PhotometryVersion, Modulation, NidaqDuration, + NidaqSamplingRate, DecimateFactor, LED1_Name, LED1_Amp, + LED1_Freq, LED2_Name, LED2_Amp, LED2_Freq, LED1b_Name, + LED1b_Amp, LED1b_Freq + TQ03_ TrialSettings: Aud_Levels, Aud_NoEvidence, Aud_Ramp, Aud_SamplingRate, + Aud_ToneDuration, Aud_ToneOverlap, Aud_UseMiddleOctave, + Aud_Volume, Aud_maxFreq, Aud_minFreq, Aud_nFreq, + AuditoryAlpha, AuditoryStimulusTime, AuditoryStimulusType, + BaselineBegin, BaselineEnd, BlockTable, CatchError, + CenterWaitMax, ChoiceDeadLine, DbleFibers, DecimateFactor, + DrinkingGrace, DrinkingTime, FeedbackDelay, FeedbackDelayDecr, + FeedbackDelayGrace, FeedbackDelayIncr, FeedbackDelayMax, + FeedbackDelayMin, FeedbackDelaySelection, FeedbackDelayTau, + ITI, IncorrectChoiceFeedbackType, Isobestic405, LED1_Amp, + LED1_Freq, LED1_Name, LED1b_Amp, LED1b_Freq, LED1b_Name, + LED2_Amp, LED2_Freq, LED2_Name, LaserAmp, LaserFeedback, + LaserITI, LaserMov, LaserPreStim, LaserPulseDuration_ms, + LaserRampDuration_ms, LaserRew, LaserSoftCode, LaserStim, + LaserStimFreq, LaserTimeInvestment, LaserTrainDuration_ms, + LaserTrainRandStart, LaserTrainStartMax_s, + LaserTrainStartMin_s, LaserTrials, LeftBiasAud, MaxSessionTime, + MinSampleAud, MinSampleAudAutoincrement, MinSampleAudDecr, + MinSampleAudIncr, MinSampleAudMax, MinSampleAudMin, Modulation, + NidaqDuration, NidaqMax, NidaqMin, NidaqSamplingRate, + OdorA_bank, OdorB_bank, OdorStimulusTimeMin, OdorTable, + Percent50Fifty, PercentAuditory, PercentCatch, PhotoPlotReward, + PhotoPlotSidePokeIn, PhotoPlotSidePokeLeave, Photometry, + PhotometryVersion, PortLEDs, Ports_LMR, PreITI, RedChannel, + RewardAmount, ShowFeedback, ShowFix, ShowPsycAud, ShowPsycOlf, + ShowST, ShowTrialRate, ShowVevaiometric, + SkippedFeedbackFeedbackType, StartEasyTrials, StimDelay, + StimDelayAutoincrement, StimDelayDecr, StimDelayIncr, + StimDelayMax, StimDelayMin, SumRates, TimeMax, TimeMin, + TimeOutBrokeFixation, TimeOutEarlyWithdrawal, + TimeOutIncorrectChoice, TimeOutSkippedFeedback, TrialSelection, + VevaiometricMinWT, VevaiometricNBin, VevaiometricShowPoints, + VideoTrials, Wire1VideoTrigger, nidaqDev + TP24_ TrialSettings: Aud_Levels, Aud_NoEvidence, Aud_Ramp, Aud_SamplingRate, + Aud_ToneDuration, Aud_ToneOverlap, Aud_UseMiddleOctave, + Aud_Volume, Aud_maxFreq, Aud_minFreq, Aud_nFreq, + AuditoryAlpha, AuditoryStimulusTime, AuditoryStimulusType, + BlockTable, CatchError, ChoiceDeadLine, FeedbackDelay, + FeedbackDelayDecr, FeedbackDelayGrace, FeedbackDelayIncr, + FeedbackDelayMax, FeedbackDelayMin, FeedbackDelaySelection, + FeedbackDelayTau, ITI, IncorrectChoiceFeedbackType, + LaserFeedback, LaserITI, LaserMov, LaserPreStim, + LaserPulseDuration_ms, LaserRew, LaserStim, LaserStimFreq, + LaserTimeInvestment, LaserTrainDuration_ms, LaserTrainRandStart, + LaserTrainStartMax_s, LaserTrainStartMin_s, LaserTrials, + LeftBiasAud, MaxSessionTime, MinSampleAud, + MinSampleAudAutoincrement, MinSampleAudDecr, MinSampleAudIncr, + MinSampleAudMax, MinSampleAudMin, OdorA_bank, OdorB_bank, + OdorStimulusTimeMin, OdorTable, Percent50Fifty, + PercentAuditory, PercentCatch, PortLEDs, Ports_LMR, + RewardAmount, ShowFeedback, ShowFix, ShowPsycAud, ShowPsycOlf, + ShowST, ShowTrialRate, ShowVevaiometric, + SkippedFeedbackFeedbackType, StartEasyTrials, StimDelay, + StimDelayAutoincrement, StimDelayDecr, StimDelayIncr, + StimDelayMax, StimDelayMin, SumRates, TimeOutBrokeFixation, + TimeOutEarlyWithdrawal, TimeOutIncorrectChoice, + TimeOutSkippedFeedback, TrialSelection, VevaiometricMinWT, + VevaiometricNBin, VevaiometricShowPoints, Wire1VideoTrigger TrialTypes: array of condition? Not consistent across all trials @@ -29,74 +117,74 @@ RawData Bonn bpod SessionData structure - TrialTypes - 1,2,3,1,2,3 - TrialTypeNames - Visibile,Visible,Fading - Info - StateMachineVersion - SessionDate - SessionStartTime_UTC - SessionStartTime_MATLAB - nTrials (# trials in session, here 54) - RawEvents (timestamps for each trial's state transitions/recorded events) - Trial{1,n}.States #Which of these are important? - WaitForPosTriggerSoftCode - CueDelay - WaitForResponse - Port2RewardDelay - Port2Reward - CloseValves - Drinking - Port1RewardDelay - Port3RewardDelay - Port4RewardDelay - Port5RewardDelay - Port6RewardDelay - Port7RewardDelay - Port8RewardDelay - Port1Reward - Port3Reward - Port4Reward - Port5Reward - Port6Reward - Port7Reward - Port8Reward - Punish - Punishexit - EarlyWithdrawal - Trial{1,n}.Events - Port4In - Port4Out - SoftCode10 - Tup - Port2In - Port2Out - RawData (copy of raw data from state machine) - TrialStartTimestamp (time when trial started on Bpod's clock) - Note: Timestamps in RawEvents are relative to each trial's start - TrialEndTimestamp - SettingsFile (the settings file you selected in the launch manager) - Notes - MarkerCodes - CurrentSubjectName - TrialSettings - GUI - GUIMeta - GUIPanels - polling - debug - debugvis - Data - arm_number - arm_baited_orig - arm_baited - SF - rotation - position - StimAlpha - StimPos - TriggerLocPix - TriggerLocOptitrackHitbox - TriggerLocOptitrackCenter - TriggerLocOptitrackCircleHitRadius - tform + TrialTypes - 1,2,3,1,2,3 + TrialTypeNames - Visibile,Visible,Fading + Info + StateMachineVersion + SessionDate + SessionStartTime_UTC + SessionStartTime_MATLAB + nTrials (# trials in session, here 54) + RawEvents (timestamps for each trial's state transitions/recorded events) + Trial{1,n}.States #Which of these are important? + WaitForPosTriggerSoftCode + CueDelay + WaitForResponse + Port2RewardDelay + Port2Reward + CloseValves + Drinking + Port1RewardDelay + Port3RewardDelay + Port4RewardDelay + Port5RewardDelay + Port6RewardDelay + Port7RewardDelay + Port8RewardDelay + Port1Reward + Port3Reward + Port4Reward + Port5Reward + Port6Reward + Port7Reward + Port8Reward + Punish + Punishexit + EarlyWithdrawal + Trial{1,n}.Events + Port4In + Port4Out + SoftCode10 + Tup + Port2In + Port2Out + RawData (copy of raw data from state machine) + TrialStartTimestamp (time when trial started on Bpod's clock) + Note: Timestamps in RawEvents are relative to each trial's start + TrialEndTimestamp + SettingsFile (the settings file you selected in the launch manager) + Notes + MarkerCodes + CurrentSubjectName + TrialSettings + GUI + GUIMeta + GUIPanels + polling + debug + debugvis + Data + arm_number + arm_baited_orig + arm_baited + SF + rotation + position + StimAlpha + StimPos + TriggerLocPix + TriggerLocOptitrackHitbox + TriggerLocOptitrackCenter + TriggerLocOptitrackCircleHitRadius + tform ''' From 3142726cd9382f707d1b6fdf67b521cdc51cd7df Mon Sep 17 00:00:00 2001 From: Chris Brozdowski Date: Mon, 24 Jan 2022 16:02:30 -0600 Subject: [PATCH 09/19] Apply suggestions from code review Co-authored-by: Kabilar Gunalan --- README.md | 2 +- element_trial/trial.py | 2 +- element_trial/version.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 67d295f..bbb33ab 100644 --- a/README.md +++ b/README.md @@ -59,4 +59,4 @@ help(trial.activate) ### Example usage -See [this project](https://github.com/datajoint/workflow-trial) for an example usage of this Array Electrophysiology Element. +See [this project](https://github.com/datajoint/workflow-trial) for an example usage of this Trial Element. diff --git a/element_trial/trial.py b/element_trial/trial.py index 59d575d..b9dd2b0 100644 --- a/element_trial/trial.py +++ b/element_trial/trial.py @@ -2,7 +2,7 @@ import inspect import importlib import pathlib -from element_data_loader.utils import find_full_path +from element_interface.utils import find_full_path from .readers import bpod schema = dj.schema() diff --git a/element_trial/version.py b/element_trial/version.py index 71bd74d..324779f 100644 --- a/element_trial/version.py +++ b/element_trial/version.py @@ -1,2 +1,2 @@ """Package metadata.""" -__version__ = '0.1.0a0' +__version__ = '0.1.0a1' From 07b69a951c780de199833d381fc8de45c7c9932a Mon Sep 17 00:00:00 2001 From: Chris Brozdowski Date: Mon, 24 Jan 2022 16:19:29 -0600 Subject: [PATCH 10/19] Apply suggestion from code review: dj.io->dj.org Co-authored-by: Kabilar Gunalan --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5836c18..dd14131 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,3 +1,3 @@ # Contribution Guidelines -This project follows the [DataJoint Contribution Guidelines](https://docs.datajoint.io/python/community/02-Contribute.html). Please reference the link for more full details. +This project follows the [DataJoint Contribution Guidelines](https://docs.datajoint.org/python/community/02-Contribute.html). Please reference the link for more full details. From e82adc6e8699a25769af652ad7df8234ad5273a5 Mon Sep 17 00:00:00 2001 From: Chris Broz Date: Mon, 24 Jan 2022 16:20:48 -0600 Subject: [PATCH 11/19] Add Background.md, remove config example --- .gitignore | 2 +- .../bpod_fields_notes.py => Background.md | 28 ++++--- dj_local_conf_example.json | 26 ------- element_trial/__init__.py | 4 - element_trial/event.py | 1 - element_trial/export/nwb.py | 78 ------------------- 6 files changed, 17 insertions(+), 122 deletions(-) rename element_trial/readers/bpod_fields_notes.py => Background.md (93%) delete mode 100644 dj_local_conf_example.json delete mode 100644 element_trial/export/nwb.py diff --git a/.gitignore b/.gitignore index 2742b10..e13e4fa 100644 --- a/.gitignore +++ b/.gitignore @@ -123,4 +123,4 @@ docker-compose.y*ml # notes temp* -*/temp* \ No newline at end of file +*/temp* diff --git a/element_trial/readers/bpod_fields_notes.py b/Background.md similarity index 93% rename from element_trial/readers/bpod_fields_notes.py rename to Background.md index 3fa5224..bb6c9df 100644 --- a/element_trial/readers/bpod_fields_notes.py +++ b/Background.md @@ -1,6 +1,16 @@ -'''Temporary file for keeping track of bpod structures +# Trial Behavior Element -The RAW prefix in bpod structures seems to always have data elsewhere +## Pipeline Development +Initial Element development began in December of 2021 in discussion with the team led +by Adam Kepecs at Washington University School of Medicine in St. Louis. Micheal Wulf +has been instrumental our understanding of the diversity of Bpod files. DataJoint provides +a diverse set of bpod examples via [DJArchive](https://github.com/datajoint/djarchive-client/). + +
+Click to expand notes on the organization of these files + +``` +The RAW prefix in bpod structures always has data elsewhere Raw is where it is initially stored, and then ported elsewhere during export Inconsistent across examples: @@ -108,15 +118,8 @@ TimeOutEarlyWithdrawal, TimeOutIncorrectChoice, TimeOutSkippedFeedback, TrialSelection, VevaiometricMinWT, VevaiometricNBin, VevaiometricShowPoints, Wire1VideoTrigger - -TrialTypes: array of condition? Not consistent across all trials - -['SessionData'].RawEvents.Trial[0].States -BalbC_Ph_W1_LH_Randdelay_changeover_Aug08_2021_Session1.mat -RawEvents -RawData - -Bonn bpod SessionData structure + +Alternate SessionData structure TrialTypes - 1,2,3,1,2,3 TrialTypeNames - Visibile,Visible,Fading Info @@ -187,4 +190,5 @@ TriggerLocOptitrackCenter TriggerLocOptitrackCircleHitRadius tform -''' +``` +
diff --git a/dj_local_conf_example.json b/dj_local_conf_example.json deleted file mode 100644 index 498909d..0000000 --- a/dj_local_conf_example.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "database.host": "", - "database.password": "", - "database.user": "", - "database.port": 3306, - "database.reconnect": true, - "connection.init_function": null, - "connection.charset": "", - "loglevel": "INFO", - "safemode": true, - "fetch_format": "array", - "display.limit": 12, - "display.width": 14, - "display.show_tuple_count": true, - "database.use_tls": null, - "enable_python_native_blobs": true, - "database.ingest_filename_short": "", - "database.ingest_filename_full": "", - "custom": { - "database.prefix": "YourPrefix_", - "beh_root_dir": [ - "/Abolute/Path/Here/", - "/Abolute/Other/Path/" - ] - } -} \ No newline at end of file diff --git a/element_trial/__init__.py b/element_trial/__init__.py index 8bbacb6..e69de29 100644 --- a/element_trial/__init__.py +++ b/element_trial/__init__.py @@ -1,4 +0,0 @@ -""" -Two schemas, trial and event. Trial for fully trialized, segmented. -Event for events independent of trials, like an act of naturaistic behavior. -""" diff --git a/element_trial/event.py b/element_trial/event.py index 0a7f5da..e69de29 100644 --- a/element_trial/event.py +++ b/element_trial/event.py @@ -1 +0,0 @@ -# Copy from trial, remove 'EventTrialized' diff --git a/element_trial/export/nwb.py b/element_trial/export/nwb.py deleted file mode 100644 index fdd484f..0000000 --- a/element_trial/export/nwb.py +++ /dev/null @@ -1,78 +0,0 @@ -from pynwb import NWBFile - -from element_session import session -from element_trial import trial - - -def trial_to_nwb(trial_key): - scan_query = session.Session & trial.TrialEvent & trial_key - # https://github.com/nwb-extensions/ndx-events-record - return NWBFile(scan_query) - - -""" From CHEN 2017 https://github.com/vathes/DJ-NWB-Chen-2017 - -# =============== TrialSet ==================== -# NWB 'trial' (of type dynamic table) by default comes with three mandatory -# attributes: 'start_time' and 'stop_time'. Other trial-related information -# needs to be added in to the trial-table as additional columns (with column -# name and column description) - -dj_trial = experiment.SessionTrial * experiment.BehaviorTrial -skip_adding_columns = experiment.Session.primary_key + ['trial_uid', 'trial'] - -if experiment.SessionTrial & session_key: - # Get trial descriptors from TrialSet.Trial and TrialStimInfo - trial_columns = [{'name': tag, - 'description': re.sub('\s+:|\s+', ' ', re.search( - f'(?<={tag})(.*)', str(dj_trial.heading)).group()).strip()} - for tag in dj_trial.heading.names - if tag not in skip_adding_columns + ['start_time', 'stop_time']] - - # Add new table columns to nwb trial-table for trial-label - for c in trial_columns: - nwbfile.add_trial_column(**c) - - # Add entry to the trial-table - for trial in (dj_trial & session_key).fetch(as_dict=True): - trial['start_time'] = float(trial['start_time']) - trial['stop_time'] = (float(trial['stop_time']) if - trial['stop_time'] else 5.0) - [trial.pop(k) for k in skip_adding_columns] - trial['early_lick'] = True if trial['early_lick'] == 'early' else False - nwbfile.add_trial(**trial) - -# =========================================================================== -# ============================= BEHAVIOR TRIAL EVENTS ======================= -# =========================================================================== - -behavior = nwbfile.create_processing_module( - 'behavior', 'Time of behavioral events in this session') -behav_event = pynwb.behavior.BehavioralEvents(name='BehavioralEvents') -behavior.add_data_interface(behav_event) - -for trial_event_type in \ - (experiment.TrialEventType & \ - experiment.TrialEvent & session_key).fetch('trial_event_type'): - event_times, trial_starts = \ - (experiment.TrialEvent * experiment.SessionTrial - & session_key & {'trial_event_type': trial_event_type}).fetch( - 'trial_event_time', 'start_time') - - if trial_event_type == 'sample': - description = 'Timestamps: beginning of the sampling on each trial.' - elif trial_event_type == 'delay': - description = 'Timestamps: beginning of the delay on each trial.' - elif trial_event_type == 'go': - description = 'Time stamps of the go cue signal on each trial.' - - if len(event_times) > 0: - event_times = np.hstack(event_times.astype(float) - + trial_starts.astype(float)) - behav_event.create_timeseries( - name=trial_event_type, unit='a.u.', conversion=1.0, - data=np.full_like(event_times, 1), - timestamps=event_times, - description=description) - -""" From 4e35af73f7f5a78812b91317b234e75111f27f26 Mon Sep 17 00:00:00 2001 From: Chris Brozdowski Date: Wed, 2 Feb 2022 06:58:25 -0600 Subject: [PATCH 12/19] Apply suggestions from code review Co-authored-by: Thinh Nguyen --- element_trial/trial.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/element_trial/trial.py b/element_trial/trial.py index b9dd2b0..0fe7afc 100644 --- a/element_trial/trial.py +++ b/element_trial/trial.py @@ -155,16 +155,16 @@ class TrialVariable(dj.Part): @schema class BehaviorRecording(dj.Imported): definition = """ - -> session.SessionDirectory + -> Session recording_id: varchar(16) --- recording_notes: varchar(256) """ - class BehFile(dj.Part): + class BehaviorFile(dj.Part): definition = """ -> master - filetype: varchar(16) + filepath: varchar(16) """ def make(self, key): From 9aacfec4bbc7bbce690e0f91b512a146bc2db49e Mon Sep 17 00:00:00 2001 From: Chris Broz Date: Wed, 2 Feb 2022 14:59:04 -0600 Subject: [PATCH 13/19] CHANGELOG clarity --- Background.md | 6 +++--- CHANGELOG.md | 6 +++--- element_trial/version.py | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Background.md b/Background.md index bb6c9df..216f753 100644 --- a/Background.md +++ b/Background.md @@ -1,9 +1,9 @@ # Trial Behavior Element ## Pipeline Development -Initial Element development began in December of 2021 in discussion with the team led -by Adam Kepecs at Washington University School of Medicine in St. Louis. Micheal Wulf -has been instrumental our understanding of the diversity of Bpod files. DataJoint provides +This Element was developed from a series of existing projects featuring trialized +behavior. Micheal Wulf at Washington University School of Medicine in St. Louis has +been instrumental our understanding of the diversity of Bpod files. DataJoint provides a diverse set of bpod examples via [DJArchive](https://github.com/datajoint/djarchive-client/).
diff --git a/CHANGELOG.md b/CHANGELOG.md index 4d98f3c..4484e0c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,10 +2,10 @@ Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) convention. -## [0.1.0b0] - [unreleased] +## [0.0.0b0] - [unreleased] ### Added + First beta release -## [0.1.0a0] - 2021-11-15 +## [0.0.0a0] - 2021-11-15 ### Added -+ First draft begins ++ First draft based on [Cajal](https://github.com/cajal/pipeline) and [Kavli Institute](https://github.com/kavli-ntnu/dj-docs) precursor projects diff --git a/element_trial/version.py b/element_trial/version.py index 324779f..5613088 100644 --- a/element_trial/version.py +++ b/element_trial/version.py @@ -1,2 +1,2 @@ """Package metadata.""" -__version__ = '0.1.0a1' +__version__ = '0.0.0a0' From 38b38ebbd309330a65a47f2ec6cc9ceda326a267 Mon Sep 17 00:00:00 2001 From: Chris Broz Date: Tue, 1 Mar 2022 14:24:42 -0600 Subject: [PATCH 14/19] remove bpod --- Background.md | 194 ---------------------------------- LICENSE | 2 +- README.md | 25 +++-- element_trial/event.py | 156 +++++++++++++++++++++++++++ element_trial/readers/bpod.py | 76 ------------- element_trial/trial.py | 124 +++++++--------------- 6 files changed, 210 insertions(+), 367 deletions(-) delete mode 100644 Background.md delete mode 100644 element_trial/readers/bpod.py diff --git a/Background.md b/Background.md deleted file mode 100644 index 216f753..0000000 --- a/Background.md +++ /dev/null @@ -1,194 +0,0 @@ -# Trial Behavior Element - -## Pipeline Development -This Element was developed from a series of existing projects featuring trialized -behavior. Micheal Wulf at Washington University School of Medicine in St. Louis has -been instrumental our understanding of the diversity of Bpod files. DataJoint provides -a diverse set of bpod examples via [DJArchive](https://github.com/datajoint/djarchive-client/). - -
-Click to expand notes on the organization of these files - -``` -The RAW prefix in bpod structures always has data elsewhere - Raw is where it is initially stored, and then ported elsewhere during export - -Inconsistent across examples: - BalbC top level: LeftValveTime, RightValveTime, LeftAmount, RightAmount, - changebridge, bridgepos, subject, delays - 1119a top level: Notes, MarkerCodes - -Custom is determined by user: - 1119a Custom: contrastLevels, dbSplTrialType3, dbSplTrialType6,startSessionDatetime, - startSessionTimeStamp,TQ03_Dual2AFC_Jun18_2021_Session1.mat, - BlockNumber, BlockTrial, ChoiceLeft, ChoiceCorrect, Feedback, - FeedbackTimeFixBroke, EarlyWithdrawal, FixDur, MT, CatchTrial, - OdorFracA, OdorID, OdorPair, ST, ResolutionTime, Rewarded, - RewardMagnitude, number_trials, LaserTrial,LaserTrialTrainStart, - AuditoryTrial, ClickTask,OlfactometerStartup, PsychtoolboxStartup, - AuditoryOmega,LeftClickRate, RightClickRate, LeftClickTrain, - RightClickTrain, LeftRewarded, DV, Rig, Subject, - PulsePalParamStimulus, PulsePalParamFeedback, StimDelay, - FeedbackDelay, MinSampleAud, - TP24 Custom: BlockNumber, BlockTrial, ChoiceLeft, ChoiceCorrect, Feedback, - FeedbackTime, FixBroke, EarlyWithdrawal, FixDur, MT,CatchTrial, - OdorFracA, OdorID, OdorPair, ST, Rewarded,RewardMagnitude, - number_trials, LaserTrial, LaserTrialTrainStart,AuditoryTrial, - ClickTask, OlfactometerStartup,PsychtoolboxStartup, AuditoryOmega, - LeftClickRate,RightClickRate, LeftClickTrain, RightClickTrain, - LeftRewarded,DV, Rig, Subject, PulsePalParamStimulus, - PulsePalParamFeedback,StimDelay, FeedbackDelay, MinSampleAud - -Settings is inconsistently used: - BalcC: Nothing - TQ03 : same as TrialSettings - -TrialSettings is also file-specific: - BalbC TrialSettings: Subject, RewardAmount, RewardDelay, ChangeOver, block, depleft, - depright, MaxTrials, randomDelay, Lange, Bridge, Drugs - 1119A TrialSettings: MaxTrialNum, StartPhaseLength, TrialType1, TrialType2, - TrialType3, TrialType4, TrialType5, TrialType6, TrialType7, - RewardAmount, ContrastLevelMin, ContrastLevelMax, UseBonsai, - UsePulsePal, PulsePalComPort, PulsePalParameterFile, - UseOptoGeneticPulseLength, NumberOfPulses, PulseFrequency, - Bpod_BNC_Ch_Opto, PulsePal_Trig_Ch_Opto, PulsePal_Out_Ch_Opto, - Opto_Stim_In_TrialType2, Opto_Stim_In_TrialType3, - Opto_Stim_In_TrialType5, Opto_Stim_In_TrialType6, - UseSoundStimulation, dB_SPL_min, dB_SPL_max, dB_SPL_step, - Photometry, DbleFibers, Isobestic405, RedChannel, - PhotometryVersion, Modulation, NidaqDuration, - NidaqSamplingRate, DecimateFactor, LED1_Name, LED1_Amp, - LED1_Freq, LED2_Name, LED2_Amp, LED2_Freq, LED1b_Name, - LED1b_Amp, LED1b_Freq - TQ03_ TrialSettings: Aud_Levels, Aud_NoEvidence, Aud_Ramp, Aud_SamplingRate, - Aud_ToneDuration, Aud_ToneOverlap, Aud_UseMiddleOctave, - Aud_Volume, Aud_maxFreq, Aud_minFreq, Aud_nFreq, - AuditoryAlpha, AuditoryStimulusTime, AuditoryStimulusType, - BaselineBegin, BaselineEnd, BlockTable, CatchError, - CenterWaitMax, ChoiceDeadLine, DbleFibers, DecimateFactor, - DrinkingGrace, DrinkingTime, FeedbackDelay, FeedbackDelayDecr, - FeedbackDelayGrace, FeedbackDelayIncr, FeedbackDelayMax, - FeedbackDelayMin, FeedbackDelaySelection, FeedbackDelayTau, - ITI, IncorrectChoiceFeedbackType, Isobestic405, LED1_Amp, - LED1_Freq, LED1_Name, LED1b_Amp, LED1b_Freq, LED1b_Name, - LED2_Amp, LED2_Freq, LED2_Name, LaserAmp, LaserFeedback, - LaserITI, LaserMov, LaserPreStim, LaserPulseDuration_ms, - LaserRampDuration_ms, LaserRew, LaserSoftCode, LaserStim, - LaserStimFreq, LaserTimeInvestment, LaserTrainDuration_ms, - LaserTrainRandStart, LaserTrainStartMax_s, - LaserTrainStartMin_s, LaserTrials, LeftBiasAud, MaxSessionTime, - MinSampleAud, MinSampleAudAutoincrement, MinSampleAudDecr, - MinSampleAudIncr, MinSampleAudMax, MinSampleAudMin, Modulation, - NidaqDuration, NidaqMax, NidaqMin, NidaqSamplingRate, - OdorA_bank, OdorB_bank, OdorStimulusTimeMin, OdorTable, - Percent50Fifty, PercentAuditory, PercentCatch, PhotoPlotReward, - PhotoPlotSidePokeIn, PhotoPlotSidePokeLeave, Photometry, - PhotometryVersion, PortLEDs, Ports_LMR, PreITI, RedChannel, - RewardAmount, ShowFeedback, ShowFix, ShowPsycAud, ShowPsycOlf, - ShowST, ShowTrialRate, ShowVevaiometric, - SkippedFeedbackFeedbackType, StartEasyTrials, StimDelay, - StimDelayAutoincrement, StimDelayDecr, StimDelayIncr, - StimDelayMax, StimDelayMin, SumRates, TimeMax, TimeMin, - TimeOutBrokeFixation, TimeOutEarlyWithdrawal, - TimeOutIncorrectChoice, TimeOutSkippedFeedback, TrialSelection, - VevaiometricMinWT, VevaiometricNBin, VevaiometricShowPoints, - VideoTrials, Wire1VideoTrigger, nidaqDev - TP24_ TrialSettings: Aud_Levels, Aud_NoEvidence, Aud_Ramp, Aud_SamplingRate, - Aud_ToneDuration, Aud_ToneOverlap, Aud_UseMiddleOctave, - Aud_Volume, Aud_maxFreq, Aud_minFreq, Aud_nFreq, - AuditoryAlpha, AuditoryStimulusTime, AuditoryStimulusType, - BlockTable, CatchError, ChoiceDeadLine, FeedbackDelay, - FeedbackDelayDecr, FeedbackDelayGrace, FeedbackDelayIncr, - FeedbackDelayMax, FeedbackDelayMin, FeedbackDelaySelection, - FeedbackDelayTau, ITI, IncorrectChoiceFeedbackType, - LaserFeedback, LaserITI, LaserMov, LaserPreStim, - LaserPulseDuration_ms, LaserRew, LaserStim, LaserStimFreq, - LaserTimeInvestment, LaserTrainDuration_ms, LaserTrainRandStart, - LaserTrainStartMax_s, LaserTrainStartMin_s, LaserTrials, - LeftBiasAud, MaxSessionTime, MinSampleAud, - MinSampleAudAutoincrement, MinSampleAudDecr, MinSampleAudIncr, - MinSampleAudMax, MinSampleAudMin, OdorA_bank, OdorB_bank, - OdorStimulusTimeMin, OdorTable, Percent50Fifty, - PercentAuditory, PercentCatch, PortLEDs, Ports_LMR, - RewardAmount, ShowFeedback, ShowFix, ShowPsycAud, ShowPsycOlf, - ShowST, ShowTrialRate, ShowVevaiometric, - SkippedFeedbackFeedbackType, StartEasyTrials, StimDelay, - StimDelayAutoincrement, StimDelayDecr, StimDelayIncr, - StimDelayMax, StimDelayMin, SumRates, TimeOutBrokeFixation, - TimeOutEarlyWithdrawal, TimeOutIncorrectChoice, - TimeOutSkippedFeedback, TrialSelection, VevaiometricMinWT, - VevaiometricNBin, VevaiometricShowPoints, Wire1VideoTrigger - -Alternate SessionData structure - TrialTypes - 1,2,3,1,2,3 - TrialTypeNames - Visibile,Visible,Fading - Info - StateMachineVersion - SessionDate - SessionStartTime_UTC - SessionStartTime_MATLAB - nTrials (# trials in session, here 54) - RawEvents (timestamps for each trial's state transitions/recorded events) - Trial{1,n}.States #Which of these are important? - WaitForPosTriggerSoftCode - CueDelay - WaitForResponse - Port2RewardDelay - Port2Reward - CloseValves - Drinking - Port1RewardDelay - Port3RewardDelay - Port4RewardDelay - Port5RewardDelay - Port6RewardDelay - Port7RewardDelay - Port8RewardDelay - Port1Reward - Port3Reward - Port4Reward - Port5Reward - Port6Reward - Port7Reward - Port8Reward - Punish - Punishexit - EarlyWithdrawal - Trial{1,n}.Events - Port4In - Port4Out - SoftCode10 - Tup - Port2In - Port2Out - RawData (copy of raw data from state machine) - TrialStartTimestamp (time when trial started on Bpod's clock) - Note: Timestamps in RawEvents are relative to each trial's start - TrialEndTimestamp - SettingsFile (the settings file you selected in the launch manager) - Notes - MarkerCodes - CurrentSubjectName - TrialSettings - GUI - GUIMeta - GUIPanels - polling - debug - debugvis - Data - arm_number - arm_baited_orig - arm_baited - SF - rotation - position - StimAlpha - StimPos - TriggerLocPix - TriggerLocOptitrackHitbox - TriggerLocOptitrackCenter - TriggerLocOptitrackCircleHitRadius - tform -``` -
diff --git a/LICENSE b/LICENSE index 6bf141b..2f92789 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2021 DataJoint +Copyright (c) 2022 DataJoint Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.md b/README.md index bbb33ab..d5e35a8 100644 --- a/README.md +++ b/README.md @@ -3,13 +3,18 @@ This repository is a work in progress not yet ready for public release. It serves as a draft of a DataJoint element for trialized experiments behavior for our U24 itiative. -Work in progress. - ## Element architecture -In both of the following diagrams, the trial element starts immediately downstream from ***Session***. In one case, Sessions are first segmented into trials, and then segmented into events. This might be appropriate, for example, in a paradigm with repeated conditions and response behaviors associated with different conditions. In the next, Sessions are directly upstream from Events. This might be appropropriate for a paradigm that recorded events within naturalistic free behavior. -We provide an [example workflow](https://github.com/datajoint/workflow-trial/) with a -[pipeline script](https://github.com/datajoint/workflow-trial/blob/main/workflow_trial/pipeline.py) that models combining this Element with the corresponding [Element-Session](https://github.com/datajoint/element-session). +In both of the following diagrams, the trial element starts immediately downstream from +***Session***. In one case, Sessions are first segmented into trials, and then aaa +segmented into events. This might be appropriate, for example, in a paradigm with +repeated conditions and response behaviors associated with different conditions. In the +next, Sessions are directly upstream from Events. This might be appropropriate for a +paradigm that recorded events within naturalistic free behavior. We provide an +[example workflow](https://github.com/datajoint/workflow-trial/) with a +[pipeline script](https://github.com/datajoint/workflow-trial/blob/main/workflow_trial/pipeline.py) +that models combining this Element with the corresponding +[Element-Session](https://github.com/datajoint/element-session). +![trial schema](images/attached_trial_element_trialized.svg) + +### Event Schema +![event schema](images/attached_trial_element_events.svg) ## Installation diff --git a/images/diagram_event.svg b/images/diagram_event.svg new file mode 100644 index 0000000..405f8e9 --- /dev/null +++ b/images/diagram_event.svg @@ -0,0 +1,145 @@ + + + + + +event.Trial.TrialVariable + + +event.Trial.TrialVariable + + + + + +event.Trial + + +event.Trial + + + + + +event.Trial->event.Trial.TrialVariable + + + + +event.TrialEvent + + +event.TrialEvent + + + + + +event.Trial->event.TrialEvent + + + + +event.EventType + + +event.EventType + + + + + +event.Event + + +event.Event + + + + + +event.EventType->event.Event + + + + +event.Event.EventVariable + + +event.Event.EventVariable + + + + + +event.TrialType + + +event.TrialType + + + + + +event.TrialType->event.Trial + + + + +event.BehaviorRecording + + +event.BehaviorRecording + + + + + +event.BehaviorRecording.BehaviorFile + + +event.BehaviorRecording.BehaviorFile + + + + + +event.BehaviorRecording->event.BehaviorRecording.BehaviorFile + + + + +event.Event->event.TrialEvent + + + + +event.Event->event.Event.EventVariable + + + + +session.Session + + +session.Session + + + + + +session.Session->event.Trial + + + + +session.Session->event.BehaviorRecording + + + + +session.Session->event.Event + + + + \ No newline at end of file diff --git a/images/diagram_trial.svg b/images/diagram_trial.svg new file mode 100644 index 0000000..dd77e26 --- /dev/null +++ b/images/diagram_trial.svg @@ -0,0 +1,126 @@ + + + + + +trial.TrialType + + +trial.TrialType + + + + + +trial.Trial + + +trial.Trial + + + + + +trial.TrialType->trial.Trial + + + + +session.Session + + +session.Session + + + + + +trial.BehaviorRecording + + +trial.BehaviorRecording + + + + + +session.Session->trial.BehaviorRecording + + + + +session.Session->trial.Trial + + + + +trial.Event.EventVariable + + +trial.Event.EventVariable + + + + + +trial.BehaviorRecording.BehaviorFile + + +trial.BehaviorRecording.BehaviorFile + + + + + +trial.Event + + +trial.Event + + + + + +trial.Event->trial.Event.EventVariable + + + + +trial.EventType + + +trial.EventType + + + + + +trial.EventType->trial.Event + + + + +trial.Trial.TrialVariable + + +trial.Trial.TrialVariable + + + + + +trial.BehaviorRecording->trial.BehaviorRecording.BehaviorFile + + + + +trial.Trial->trial.Event + + + + +trial.Trial->trial.Trial.TrialVariable + + + + \ No newline at end of file From 62a7062f116f9492c8c93a47ae63e0affd2ae46b Mon Sep 17 00:00:00 2001 From: Chris Broz Date: Tue, 1 Mar 2022 14:34:37 -0600 Subject: [PATCH 16/19] edit readme diagram refs --- README.md | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 165a1e5..e24ae59 100644 --- a/README.md +++ b/README.md @@ -17,10 +17,10 @@ that models combining this Element with the corresponding ### Trial Schema -![trial schema](images/attached_trial_element_trialized.svg) +![trial schema](./images/attached_trial_element_trialized.svg) ### Event Schema -![event schema](images/attached_trial_element_events.svg) +![event schema](./images/attached_trial_element_events.svg) ## Installation @@ -63,7 +63,3 @@ from element_trial import event, trial help(event.activate) help(trial.activate) ``` - -### Example usage - -See [this project](https://github.com/datajoint/workflow-trial) for an example usage of this Trial Element. From 4554d64888a3fcfb2140c6ce2c7faabf8bee3dc2 Mon Sep 17 00:00:00 2001 From: Chris Broz Date: Tue, 1 Mar 2022 14:37:55 -0600 Subject: [PATCH 17/19] edit readme diagram links typo --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e24ae59..265803e 100644 --- a/README.md +++ b/README.md @@ -17,10 +17,10 @@ that models combining this Element with the corresponding ### Trial Schema -![trial schema](./images/attached_trial_element_trialized.svg) +![trial schema](./images/diagram_trial.svg) ### Event Schema -![event schema](./images/attached_trial_element_events.svg) +![event schema](./images/diagram_event.svg) ## Installation From 514b6f4b27c99ad95aaea007aa5d6fbbf91e948e Mon Sep 17 00:00:00 2001 From: Chris Broz Date: Tue, 1 Mar 2022 14:46:15 -0600 Subject: [PATCH 18/19] changelog formatting --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4484e0c..90f0422 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,10 +2,10 @@ Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) convention. -## [0.0.0b0] - [unreleased] +## 0.0.0b0 - Unreleased ### Added + First beta release -## [0.0.0a0] - 2021-11-15 +## 0.0.0a0 - 2021-11-15 ### Added + First draft based on [Cajal](https://github.com/cajal/pipeline) and [Kavli Institute](https://github.com/kavli-ntnu/dj-docs) precursor projects From ee7d0fb64e46ae8339632d5f8b939d0646d20bf6 Mon Sep 17 00:00:00 2001 From: Chris Brozdowski Date: Thu, 3 Mar 2022 11:34:01 -0600 Subject: [PATCH 19/19] Apply suggestions from code review Co-authored-by: Kabilar Gunalan --- CHANGELOG.md | 4 +--- README.md | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 90f0422..4ea2ef8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,10 +2,8 @@ Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) convention. -## 0.0.0b0 - Unreleased +## 0.1.0b0 - Unreleased ### Added + First beta release -## 0.0.0a0 - 2021-11-15 -### Added + First draft based on [Cajal](https://github.com/cajal/pipeline) and [Kavli Institute](https://github.com/kavli-ntnu/dj-docs) precursor projects diff --git a/README.md b/README.md index 265803e..8831670 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ for our U24 itiative. ## Element architecture -In both of the following diagrams, the trial element starts immediately downstream from +In both of the following diagrams, the trial table starts immediately downstream from ***Session***. In one case, Sessions are first segmented into trials, and then segmented into events. This might be appropriate, for example, in a paradigm with repeated conditions and response behaviors associated with different conditions. In the