From 31e46cd30f0467f43aca18ca450ec72667656417 Mon Sep 17 00:00:00 2001 From: Chris Broz Date: Tue, 11 Oct 2022 18:06:51 -0500 Subject: [PATCH 01/19] Add nwb.py from run_kilosort branch --- element_array_ephys/export/nwb/nwb.py | 69 ++++++++++++++++----------- 1 file changed, 40 insertions(+), 29 deletions(-) diff --git a/element_array_ephys/export/nwb/nwb.py b/element_array_ephys/export/nwb/nwb.py index 321159ee..d56f1f38 100644 --- a/element_array_ephys/export/nwb/nwb.py +++ b/element_array_ephys/export/nwb/nwb.py @@ -14,12 +14,14 @@ from spikeinterface import extractors from tqdm import tqdm import warnings -from ... import probe, ephys_no_curation +from ... import probe +from ... import ephys_no_curation as ephys -assert probe.schema.is_activated(), 'probe not yet activated' -assert ephys_no_curation.schema.is_activated, \ - "The ephys module must be activated before export." +ephys_mode = os.getenv('EPHYS_MODE', dj.config['custom'].get('ephys_mode', 'acute')) +if ephys_mode != 'no-curation': + raise NotImplementedError('This export function is designed for the no_curation ' + +'schema') class DecimalEncoder(json.JSONEncoder): @@ -41,14 +43,15 @@ def __init__(self, lfp_electrodes_query, chunk_length: int = 10000): ---------- lfp_electrodes_query: element_array_ephys.ephys.LFP.Electrode chunk_length: int, optional - Chunks are blocks of disk space where data are stored contiguously and compressed + Chunks are blocks of disk space where data are stored contiguously + and compressed """ self.lfp_electrodes_query = lfp_electrodes_query self.electrodes = self.lfp_electrodes_query.fetch("electrode") first_record = ( self.lfp_electrodes_query & dict(electrode=self.electrodes[0]) - ).fetch1(as_dict=True) + ).fetch1() self.n_channels = len(self.electrodes) self.n_tt = len(first_record["lfp"]) @@ -166,7 +169,8 @@ def create_units_table( nwbfile: pynwb.NWBFile, paramset_record, name="units", - desc="data on spiking units"): + desc="data on spiking units" +): """ ephys.CuratedClustering.Unit::unit -> units.id @@ -215,7 +219,8 @@ def create_units_table( ephys.ProbeInsertion * ephys.CuratedClustering.Unit * probe.ProbeType.Electrode - & unit + & dict((k, unit[k]) for k in unit.keys() # excess keys caused errs + if k not in ['spike_times', 'spike_sites', 'spike_depths']) ).fetch1("probe", "shank") waveform_mean = ( @@ -302,8 +307,8 @@ def add_ephys_units_to_nwb( def get_electrodes_mapping(electrodes): """ Create a mapping from the probe and electrode id to the row number of the electrodes - table. This is used in the construction of the DynamicTableRegion that indicates what rows of the electrodes - table correspond to the data in an ElectricalSeries. + table. This is used in the construction of the DynamicTableRegion that indicates + what rows of the electrodes table correspond to the data in an ElectricalSeries. Parameters ---------- @@ -356,18 +361,17 @@ def add_ephys_recording_to_nwb( end_frame: int = None, ): """ - Read voltage data directly from source files and iteratively transfer them to the NWB file. Automatically - applies lossless compression to the data, so the final file might be smaller than the original, without - data loss. Currently supports Neuropixels data acquired with SpikeGLX or Open Ephys, and relies on SpikeInterface to read the data. + Read voltage data directly from source files and iteratively transfer them to the + NWB file. Automatically applies lossless compression to the data, so the final file + might be smaller than the original, without data loss. Currently supports + Neuropixels data acquired with SpikeGLX or Open Ephys, and relies on SpikeInterface + to read the data. source data -> acquisition["ElectricalSeries"] Parameters ---------- - session_key: dict - ephys_root_data_dir: str - nwbfile: NWBFile - end_frame: int, optional + session_key: dict ephys_root_data_dir: str nwbfile: NWBFile end_frame: int, optional Used for small test conversions """ @@ -429,8 +433,12 @@ def add_ephys_lfp_from_dj_to_nwb(session_key: dict, nwbfile: pynwb.NWBFile): """ Read LFP data from the data in element-aray-ephys - ephys.LFP.Electrode::lfp -> processing["ecephys"].lfp.electrical_series["ElectricalSeries{insertion_number}"].data - ephys.LFP::lfp_time_stamps -> processing["ecephys"].lfp.electrical_series["ElectricalSeries{insertion_number}"].timestamps + ephys.LFP.Electrode::lfp -> + processing["ecephys"].lfp.electrical_series["ElectricalSeries{insertion_number}" + ].data + ephys.LFP::lfp_time_stamps -> + processing["ecephys"].lfp.electrical_series["ElectricalSeries{insertion_number}" + ].timestamps Parameters ---------- @@ -475,7 +483,7 @@ def add_ephys_lfp_from_dj_to_nwb(session_key: dict, nwbfile: pynwb.NWBFile): def add_ephys_lfp_from_source_to_nwb( session_key: dict, ephys_root_data_dir, nwbfile: pynwb.NWBFile, end_frame=None): """ - Read the LFP data directly from the source file. Currently, only works for SpikeGLX data. + Read the LFP data from the source file. Currently, only works for SpikeGLX data. ephys.EphysRecording::recording_datetime -> acquisition @@ -514,7 +522,8 @@ def add_ephys_lfp_from_source_to_nwb( extractor = extractors.read_spikeglx(os.path.split(file_path)[0], "imec.lf") else: raise ValueError( - f"unsupported acq_software type: {ephys_recording_record['acq_software']}" + "unsupported acq_software type:" + + f"{ephys_recording_record['acq_software']}" ) if end_frame is not None: @@ -564,7 +573,7 @@ def ecephys_session_to_nwb( ---------- session_key: dict raw: bool - Whether to include the raw data from source. SpikeGLX and OpenEphys are supported + Whether to include the raw data from source. SpikeGLX & OpenEphys are supported spikes: bool Whether to include CuratedClustering lfp: @@ -573,13 +582,14 @@ def ecephys_session_to_nwb( False - do not convert LFP end_frame: int, optional Used to create small test conversions where large datasets are truncated. - lab_key, project_key, and protocol_key: dictionaries used to look up optional additional metadata + lab_key, project_key, and protocol_key: dictionaries to look up optional metadata nwbfile_kwargs: dict, optional - - If element-session is not being used, this argument is required and must be a dictionary containing - 'session_description' (str), 'identifier' (str), and 'session_start_time' (datetime), - the minimal data for instantiating an NWBFile object. - - - If element-session is being used, this argument can optionally be used to add over overwrite NWBFile fields. + - If element-session is not being used, this argument is required and must be a + dictionary containing 'session_description' (str), 'identifier' (str), and + 'session_start_time' (datetime), the required minimal data for instantiating + an NWBFile object. + - If element-session is being used, this argument can optionally be used to + overwrite NWBFile fields. """ session_to_nwb = getattr(ephys._linking_module, 'session_to_nwb', False) @@ -608,7 +618,8 @@ def ecephys_session_to_nwb( add_ephys_lfp_from_dj_to_nwb(session_key, nwbfile) if lfp == "source": - add_ephys_lfp_from_source_to_nwb(session_key, ephys_root_data_dir=ephys_root_data_dir, + add_ephys_lfp_from_source_to_nwb(session_key, + ephys_root_data_dir=ephys_root_data_dir, nwbfile=nwbfile, end_frame=end_frame) return nwbfile From 0ec5e3d135f62013083ce2799db927e23ad1d10e Mon Sep 17 00:00:00 2001 From: Chris Brozdowski Date: Wed, 12 Oct 2022 09:17:34 -0500 Subject: [PATCH 02/19] Update element_array_ephys/export/nwb/nwb.py Co-authored-by: Kabilar Gunalan --- element_array_ephys/export/nwb/nwb.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/element_array_ephys/export/nwb/nwb.py b/element_array_ephys/export/nwb/nwb.py index d56f1f38..3e0e37a4 100644 --- a/element_array_ephys/export/nwb/nwb.py +++ b/element_array_ephys/export/nwb/nwb.py @@ -371,7 +371,10 @@ def add_ephys_recording_to_nwb( Parameters ---------- - session_key: dict ephys_root_data_dir: str nwbfile: NWBFile end_frame: int, optional + session_key: dict + ephys_root_data_dir: str + nwbfile: NWBFile + end_frame: int, optional Used for small test conversions """ From 0e5a1c64d8e7913cdb70ef701c5e53a83225fbcf Mon Sep 17 00:00:00 2001 From: CBroz1 Date: Thu, 12 Jan 2023 14:58:40 -0600 Subject: [PATCH 03/19] blackify --- element_array_ephys/ephys_acute.py | 84 ++++++++++------------------ element_array_ephys/ephys_chronic.py | 78 +++++++++----------------- 2 files changed, 56 insertions(+), 106 deletions(-) diff --git a/element_array_ephys/ephys_acute.py b/element_array_ephys/ephys_acute.py index b32d60ca..21aaa7bb 100644 --- a/element_array_ephys/ephys_acute.py +++ b/element_array_ephys/ephys_acute.py @@ -185,8 +185,7 @@ def auto_generate_entries(cls, session_key): probe_dir = meta_filepath.parent try: - probe_number = re.search( - "(imec)?\d{1}$", probe_dir.name).group() + probe_number = re.search("(imec)?\d{1}$", probe_dir.name).group() probe_number = int(probe_number.replace("imec", "")) except AttributeError: probe_number = meta_fp_idx @@ -215,8 +214,7 @@ def auto_generate_entries(cls, session_key): } ) else: - raise NotImplementedError( - f"Unknown acquisition software: {acq_software}") + raise NotImplementedError(f"Unknown acquisition software: {acq_software}") probe.Probe.insert(probe_list, skip_duplicates=True) cls.insert(probe_insertion_list, skip_duplicates=True) @@ -323,14 +321,12 @@ def make(self, key): break else: raise FileNotFoundError( - "No SpikeGLX data found for probe insertion: {}".format( - key) + "No SpikeGLX data found for probe insertion: {}".format(key) ) if spikeglx_meta.probe_model in supported_probe_types: probe_type = spikeglx_meta.probe_model - electrode_query = probe.ProbeType.Electrode & { - "probe_type": probe_type} + electrode_query = probe.ProbeType.Electrode & {"probe_type": probe_type} probe_electrodes = { (shank, shank_col, shank_row): key @@ -363,11 +359,9 @@ def make(self, key): } ) - root_dir = find_root_directory( - get_ephys_root_data_dir(), meta_filepath) + root_dir = find_root_directory(get_ephys_root_data_dir(), meta_filepath) self.EphysFile.insert1( - {**key, - "file_path": meta_filepath.relative_to(root_dir).as_posix()} + {**key, "file_path": meta_filepath.relative_to(root_dir).as_posix()} ) elif acq_software == "Open Ephys": dataset = openephys.OpenEphys(session_dir) @@ -376,8 +370,7 @@ def make(self, key): break else: raise FileNotFoundError( - "No Open Ephys data found for probe insertion: {}".format( - key) + "No Open Ephys data found for probe insertion: {}".format(key) ) if not probe_data.ap_meta: @@ -387,8 +380,7 @@ def make(self, key): if probe_data.probe_model in supported_probe_types: probe_type = probe_data.probe_model - electrode_query = probe.ProbeType.Electrode & { - "probe_type": probe_type} + electrode_query = probe.ProbeType.Electrode & {"probe_type": probe_type} probe_electrodes = { key["electrode"]: key for key in electrode_query.fetch("KEY") @@ -401,8 +393,7 @@ def make(self, key): else: raise NotImplementedError( "Processing for neuropixels" - " probe model {} not yet implemented".format( - probe_data.probe_model) + " probe model {} not yet implemented".format(probe_data.probe_model) ) self.insert1( @@ -464,7 +455,7 @@ class LFP(dj.Imported): class Electrode(dj.Part): """Saves local field potential data for each electrode. - + Attributes: LFP (foreign key): LFP primary key. probe.ElectrodeConfig.Electrode (foreign key): probe.ElectrodeConfig.Electrode primary key. @@ -484,18 +475,16 @@ class Electrode(dj.Part): def make(self, key): """Populates the LFP tables.""" - acq_software = (EphysRecording * ProbeInsertion & - key).fetch1("acq_software") + acq_software = (EphysRecording * ProbeInsertion & key).fetch1("acq_software") electrode_keys, lfp = [], [] if acq_software == "SpikeGLX": spikeglx_meta_filepath = get_spikeglx_meta_filepath(key) - spikeglx_recording = spikeglx.SpikeGLX( - spikeglx_meta_filepath.parent) + spikeglx_recording = spikeglx.SpikeGLX(spikeglx_meta_filepath.parent) lfp_channel_ind = spikeglx_recording.lfmeta.recording_channels[ - -1:: -self._skip_channel_counts + -1 :: -self._skip_channel_counts ] # Extract LFP data at specified channels and convert to uV @@ -503,8 +492,7 @@ def make(self, key): :, lfp_channel_ind ] # (sample x channel) lfp = ( - lfp * - spikeglx_recording.get_channel_bit_volts("lf")[lfp_channel_ind] + lfp * spikeglx_recording.get_channel_bit_volts("lf")[lfp_channel_ind] ).T # (channel x sample) self.insert1( @@ -536,21 +524,19 @@ def make(self, key): shank, shank_col, shank_row, _ = spikeglx_recording.apmeta.shankmap[ "data" ][recorded_site] - electrode_keys.append( - probe_electrodes[(shank, shank_col, shank_row)]) + electrode_keys.append(probe_electrodes[(shank, shank_col, shank_row)]) elif acq_software == "Open Ephys": oe_probe = get_openephys_probe_data(key) lfp_channel_ind = np.r_[ len(oe_probe.lfp_meta["channels_indices"]) - - 1: 0: -self._skip_channel_counts + - 1 : 0 : -self._skip_channel_counts ] # (sample x channel) lfp = oe_probe.lfp_timeseries[:, lfp_channel_ind] lfp = ( - lfp * - np.array(oe_probe.lfp_meta["channels_gains"])[lfp_channel_ind] + lfp * np.array(oe_probe.lfp_meta["channels_gains"])[lfp_channel_ind] ).T # (channel x sample) lfp_timestamps = oe_probe.lfp_timestamps @@ -778,8 +764,7 @@ def auto_generate_entries(cls, ephys_recording_key: dict, paramset_idx: int = 0) key = {**ephys_recording_key, "paramset_idx": paramset_idx} processed_dir = get_processed_root_data_dir() - output_dir = ClusteringTask.infer_output_dir( - key, relative=False, mkdir=True) + output_dir = ClusteringTask.infer_output_dir(key, relative=False, mkdir=True) try: kilosort.Kilosort( @@ -826,8 +811,7 @@ def make(self, key): ) if not output_dir: - output_dir = ClusteringTask.infer_output_dir( - key, relative=True, mkdir=True) + output_dir = ClusteringTask.infer_output_dir(key, relative=True, mkdir=True) # update clustering_output_dir ClusteringTask.update1( {**key, "clustering_output_dir": output_dir.as_posix()} @@ -1038,8 +1022,7 @@ def make(self, key): "acq_software", "sampling_rate" ) - sample_rate = kilosort_dataset.data["params"].get( - "sample_rate", sample_rate) + sample_rate = kilosort_dataset.data["params"].get("sample_rate", sample_rate) # ---------- Unit ---------- # -- Remove 0-spike units @@ -1051,8 +1034,7 @@ def make(self, key): valid_units = kilosort_dataset.data["cluster_ids"][withspike_idx] valid_unit_labels = kilosort_dataset.data["cluster_groups"][withspike_idx] # -- Get channel and electrode-site mapping - channel2electrodes = get_neuropixels_channel2electrode_map( - key, acq_software) + channel2electrodes = get_neuropixels_channel2electrode_map(key, acq_software) # -- Spike-times -- # spike_times_sec_adj > spike_times_sec > spike_times @@ -1219,8 +1201,7 @@ def yield_unit_waveforms(): else: if acq_software == "SpikeGLX": spikeglx_meta_filepath = get_spikeglx_meta_filepath(key) - neuropixels_recording = spikeglx.SpikeGLX( - spikeglx_meta_filepath.parent) + neuropixels_recording = spikeglx.SpikeGLX(spikeglx_meta_filepath.parent) elif acq_software == "Open Ephys": session_dir = find_full_path( get_ephys_root_data_dir(), get_session_directory(key) @@ -1268,11 +1249,9 @@ def yield_unit_waveforms(): self.insert1(key) for unit_peak_waveform, unit_electrode_waveforms in yield_unit_waveforms(): if unit_peak_waveform: - self.PeakWaveform.insert1( - unit_peak_waveform, ignore_extra_fields=True) + self.PeakWaveform.insert1(unit_peak_waveform, ignore_extra_fields=True) if unit_electrode_waveforms: - self.Waveform.insert( - unit_electrode_waveforms, ignore_extra_fields=True) + self.Waveform.insert(unit_electrode_waveforms, ignore_extra_fields=True) @schema @@ -1417,8 +1396,7 @@ def get_spikeglx_meta_filepath(ephys_recording_key: dict) -> str: ProbeInsertion * probe.Probe & ephys_recording_key ).fetch1("probe") - spikeglx_meta_filepaths = [ - fp for fp in session_dir.rglob("*.ap.meta")] + spikeglx_meta_filepaths = [fp for fp in session_dir.rglob("*.ap.meta")] for meta_filepath in spikeglx_meta_filepaths: spikeglx_meta = spikeglx.SpikeGLXMeta(meta_filepath) if str(spikeglx_meta.probe_SN) == inserted_probe_serial_number: @@ -1458,8 +1436,7 @@ def get_neuropixels_channel2electrode_map( ) -> dict: """Get the channel map for neuropixels probe.""" if acq_software == "SpikeGLX": - spikeglx_meta_filepath = get_spikeglx_meta_filepath( - ephys_recording_key) + spikeglx_meta_filepath = get_spikeglx_meta_filepath(ephys_recording_key) spikeglx_meta = spikeglx.SpikeGLXMeta(spikeglx_meta_filepath) electrode_config_key = ( EphysRecording * probe.ElectrodeConfig & ephys_recording_key @@ -1514,8 +1491,7 @@ def generate_electrode_config(probe_type: str, electrode_keys: list) -> dict: dict: representing a key of the probe.ElectrodeConfig table """ # compute hash for the electrode config (hash of dict of all ElectrodeConfig.Electrode) - electrode_config_hash = dict_to_uuid( - {k["electrode"]: k for k in electrode_keys}) + electrode_config_hash = dict_to_uuid({k["electrode"]: k for k in electrode_keys}) electrode_list = sorted([k["electrode"] for k in electrode_keys]) electrode_gaps = ( @@ -1585,11 +1561,9 @@ def get_recording_channels_details(ephys_recording_key: dict) -> np.array: channels_details["num_channels"] = len(channels_details["channel_ind"]) if acq_software == "SpikeGLX": - spikeglx_meta_filepath = get_spikeglx_meta_filepath( - ephys_recording_key) + spikeglx_meta_filepath = get_spikeglx_meta_filepath(ephys_recording_key) spikeglx_recording = spikeglx.SpikeGLX(spikeglx_meta_filepath.parent) - channels_details["uVPerBit"] = spikeglx_recording.get_channel_bit_volts("ap")[ - 0] + channels_details["uVPerBit"] = spikeglx_recording.get_channel_bit_volts("ap")[0] channels_details["connected"] = np.array( [v for *_, v in spikeglx_recording.apmeta.shankmap["data"]] ) diff --git a/element_array_ephys/ephys_chronic.py b/element_array_ephys/ephys_chronic.py index 636ea6ad..cb0050c0 100644 --- a/element_array_ephys/ephys_chronic.py +++ b/element_array_ephys/ephys_chronic.py @@ -259,8 +259,7 @@ def make(self, key): if spikeglx_meta.probe_model in supported_probe_types: probe_type = spikeglx_meta.probe_model - electrode_query = probe.ProbeType.Electrode & { - "probe_type": probe_type} + electrode_query = probe.ProbeType.Electrode & {"probe_type": probe_type} probe_electrodes = { (shank, shank_col, shank_row): key @@ -293,11 +292,9 @@ def make(self, key): } ) - root_dir = find_root_directory( - get_ephys_root_data_dir(), meta_filepath) + root_dir = find_root_directory(get_ephys_root_data_dir(), meta_filepath) self.EphysFile.insert1( - {**key, - "file_path": meta_filepath.relative_to(root_dir).as_posix()} + {**key, "file_path": meta_filepath.relative_to(root_dir).as_posix()} ) elif acq_software == "Open Ephys": dataset = openephys.OpenEphys(session_dir) @@ -306,8 +303,7 @@ def make(self, key): break else: raise FileNotFoundError( - "No Open Ephys data found for probe insertion: {}".format( - key) + "No Open Ephys data found for probe insertion: {}".format(key) ) if not probe_data.ap_meta: @@ -317,8 +313,7 @@ def make(self, key): if probe_data.probe_model in supported_probe_types: probe_type = probe_data.probe_model - electrode_query = probe.ProbeType.Electrode & { - "probe_type": probe_type} + electrode_query = probe.ProbeType.Electrode & {"probe_type": probe_type} probe_electrodes = { key["electrode"]: key for key in electrode_query.fetch("KEY") @@ -331,8 +326,7 @@ def make(self, key): else: raise NotImplementedError( "Processing for neuropixels" - " probe model {} not yet implemented".format( - probe_data.probe_model) + " probe model {} not yet implemented".format(probe_data.probe_model) ) self.insert1( @@ -394,7 +388,7 @@ class LFP(dj.Imported): class Electrode(dj.Part): """Saves local field potential data for each electrode. - + Attributes: LFP (foreign key): LFP primary key. probe.ElectrodeConfig.Electrode (foreign key): probe.ElectrodeConfig.Electrode primary key. @@ -414,18 +408,16 @@ class Electrode(dj.Part): def make(self, key): """Populates the LFP tables.""" - acq_software = (EphysRecording * ProbeInsertion & - key).fetch1("acq_software") + acq_software = (EphysRecording * ProbeInsertion & key).fetch1("acq_software") electrode_keys, lfp = [], [] if acq_software == "SpikeGLX": spikeglx_meta_filepath = get_spikeglx_meta_filepath(key) - spikeglx_recording = spikeglx.SpikeGLX( - spikeglx_meta_filepath.parent) + spikeglx_recording = spikeglx.SpikeGLX(spikeglx_meta_filepath.parent) lfp_channel_ind = spikeglx_recording.lfmeta.recording_channels[ - -1:: -self._skip_channel_counts + -1 :: -self._skip_channel_counts ] # Extract LFP data at specified channels and convert to uV @@ -433,8 +425,7 @@ def make(self, key): :, lfp_channel_ind ] # (sample x channel) lfp = ( - lfp * - spikeglx_recording.get_channel_bit_volts("lf")[lfp_channel_ind] + lfp * spikeglx_recording.get_channel_bit_volts("lf")[lfp_channel_ind] ).T # (channel x sample) self.insert1( @@ -466,21 +457,19 @@ def make(self, key): shank, shank_col, shank_row, _ = spikeglx_recording.apmeta.shankmap[ "data" ][recorded_site] - electrode_keys.append( - probe_electrodes[(shank, shank_col, shank_row)]) + electrode_keys.append(probe_electrodes[(shank, shank_col, shank_row)]) elif acq_software == "Open Ephys": oe_probe = get_openephys_probe_data(key) lfp_channel_ind = np.r_[ len(oe_probe.lfp_meta["channels_indices"]) - - 1: 0: -self._skip_channel_counts + - 1 : 0 : -self._skip_channel_counts ] # (sample x channel) lfp = oe_probe.lfp_timeseries[:, lfp_channel_ind] lfp = ( - lfp * - np.array(oe_probe.lfp_meta["channels_gains"])[lfp_channel_ind] + lfp * np.array(oe_probe.lfp_meta["channels_gains"])[lfp_channel_ind] ).T # (channel x sample) lfp_timestamps = oe_probe.lfp_timestamps @@ -673,8 +662,7 @@ def infer_output_dir(cls, key, relative=False, mkdir=False): e.g.: sub4/sess1/probe_2/kilosort2_0 """ processed_dir = pathlib.Path(get_processed_root_data_dir()) - sess_dir = find_full_path( - get_ephys_root_data_dir(), get_session_directory(key)) + sess_dir = find_full_path(get_ephys_root_data_dir(), get_session_directory(key)) root_dir = find_root_directory(get_ephys_root_data_dir(), sess_dir) method = ( @@ -707,8 +695,7 @@ def auto_generate_entries(cls, ephys_recording_key: dict, paramset_idx: int = 0) key = {**ephys_recording_key, "paramset_idx": paramset_idx} processed_dir = get_processed_root_data_dir() - output_dir = ClusteringTask.infer_output_dir( - key, relative=False, mkdir=True) + output_dir = ClusteringTask.infer_output_dir(key, relative=False, mkdir=True) try: kilosort.Kilosort( @@ -755,8 +742,7 @@ def make(self, key): ) if not output_dir: - output_dir = ClusteringTask.infer_output_dir( - key, relative=True, mkdir=True) + output_dir = ClusteringTask.infer_output_dir(key, relative=True, mkdir=True) # update clustering_output_dir ClusteringTask.update1( {**key, "clustering_output_dir": output_dir.as_posix()} @@ -967,8 +953,7 @@ def make(self, key): "acq_software", "sampling_rate" ) - sample_rate = kilosort_dataset.data["params"].get( - "sample_rate", sample_rate) + sample_rate = kilosort_dataset.data["params"].get("sample_rate", sample_rate) # ---------- Unit ---------- # -- Remove 0-spike units @@ -980,8 +965,7 @@ def make(self, key): valid_units = kilosort_dataset.data["cluster_ids"][withspike_idx] valid_unit_labels = kilosort_dataset.data["cluster_groups"][withspike_idx] # -- Get channel and electrode-site mapping - channel2electrodes = get_neuropixels_channel2electrode_map( - key, acq_software) + channel2electrodes = get_neuropixels_channel2electrode_map(key, acq_software) # -- Spike-times -- # spike_times_sec_adj > spike_times_sec > spike_times @@ -1148,8 +1132,7 @@ def yield_unit_waveforms(): else: if acq_software == "SpikeGLX": spikeglx_meta_filepath = get_spikeglx_meta_filepath(key) - neuropixels_recording = spikeglx.SpikeGLX( - spikeglx_meta_filepath.parent) + neuropixels_recording = spikeglx.SpikeGLX(spikeglx_meta_filepath.parent) elif acq_software == "Open Ephys": session_dir = find_full_path( get_ephys_root_data_dir(), get_session_directory(key) @@ -1197,11 +1180,9 @@ def yield_unit_waveforms(): self.insert1(key) for unit_peak_waveform, unit_electrode_waveforms in yield_unit_waveforms(): if unit_peak_waveform: - self.PeakWaveform.insert1( - unit_peak_waveform, ignore_extra_fields=True) + self.PeakWaveform.insert1(unit_peak_waveform, ignore_extra_fields=True) if unit_electrode_waveforms: - self.Waveform.insert( - unit_electrode_waveforms, ignore_extra_fields=True) + self.Waveform.insert(unit_electrode_waveforms, ignore_extra_fields=True) @schema @@ -1346,8 +1327,7 @@ def get_spikeglx_meta_filepath(ephys_recording_key: dict) -> str: ProbeInsertion * probe.Probe & ephys_recording_key ).fetch1("probe") - spikeglx_meta_filepaths = [ - fp for fp in session_dir.rglob("*.ap.meta")] + spikeglx_meta_filepaths = [fp for fp in session_dir.rglob("*.ap.meta")] for meta_filepath in spikeglx_meta_filepaths: spikeglx_meta = spikeglx.SpikeGLXMeta(meta_filepath) if str(spikeglx_meta.probe_SN) == inserted_probe_serial_number: @@ -1387,8 +1367,7 @@ def get_neuropixels_channel2electrode_map( ) -> dict: """Get the channel map for neuropixels probe.""" if acq_software == "SpikeGLX": - spikeglx_meta_filepath = get_spikeglx_meta_filepath( - ephys_recording_key) + spikeglx_meta_filepath = get_spikeglx_meta_filepath(ephys_recording_key) spikeglx_meta = spikeglx.SpikeGLXMeta(spikeglx_meta_filepath) electrode_config_key = ( EphysRecording * probe.ElectrodeConfig & ephys_recording_key @@ -1443,8 +1422,7 @@ def generate_electrode_config(probe_type: str, electrode_keys: list) -> dict: dict: representing a key of the probe.ElectrodeConfig table """ # compute hash for the electrode config (hash of dict of all ElectrodeConfig.Electrode) - electrode_config_hash = dict_to_uuid( - {k["electrode"]: k for k in electrode_keys}) + electrode_config_hash = dict_to_uuid({k["electrode"]: k for k in electrode_keys}) electrode_list = sorted([k["electrode"] for k in electrode_keys]) electrode_gaps = ( @@ -1514,11 +1492,9 @@ def get_recording_channels_details(ephys_recording_key: dict) -> np.array: channels_details["num_channels"] = len(channels_details["channel_ind"]) if acq_software == "SpikeGLX": - spikeglx_meta_filepath = get_spikeglx_meta_filepath( - ephys_recording_key) + spikeglx_meta_filepath = get_spikeglx_meta_filepath(ephys_recording_key) spikeglx_recording = spikeglx.SpikeGLX(spikeglx_meta_filepath.parent) - channels_details["uVPerBit"] = spikeglx_recording.get_channel_bit_volts("ap")[ - 0] + channels_details["uVPerBit"] = spikeglx_recording.get_channel_bit_volts("ap")[0] channels_details["connected"] = np.array( [v for *_, v in spikeglx_recording.apmeta.shankmap["data"]] ) From b3c0786329d6c60c00e092aa0f3a10e920970c20 Mon Sep 17 00:00:00 2001 From: CBroz1 Date: Thu, 12 Jan 2023 15:01:27 -0600 Subject: [PATCH 04/19] Remove unused import --- element_array_ephys/plotting/qc.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/element_array_ephys/plotting/qc.py b/element_array_ephys/plotting/qc.py index 4910918e..f7b62f3b 100644 --- a/element_array_ephys/plotting/qc.py +++ b/element_array_ephys/plotting/qc.py @@ -6,8 +6,6 @@ import plotly.graph_objs as go from scipy.ndimage import gaussian_filter1d -from .. import ephys_report - logger = logging.getLogger("datajoint") From ea980e9d35a7582ff1953651812e61a736931c9d Mon Sep 17 00:00:00 2001 From: CBroz1 Date: Thu, 12 Jan 2023 15:35:13 -0600 Subject: [PATCH 05/19] Spellcheck pass --- cspell.json | 121 +++++++++++++++++++++++ docs/mkdocs.yaml | 2 +- element_array_ephys/ephys_acute.py | 2 +- element_array_ephys/ephys_chronic.py | 2 +- element_array_ephys/ephys_no_curation.py | 2 +- element_array_ephys/ephys_precluster.py | 4 +- element_array_ephys/plotting/qc.py | 10 +- 7 files changed, 132 insertions(+), 11 deletions(-) create mode 100644 cspell.json diff --git a/cspell.json b/cspell.json new file mode 100644 index 00000000..242291f0 --- /dev/null +++ b/cspell.json @@ -0,0 +1,121 @@ +// cSpell Settings +//https://github.com/streetsidesoftware/vscode-spell-checker +{ + "version": "0.2", // Version of the setting file. Always 0.2 + "language": "en", // language - current active spelling language + "enabledLanguageIds": [ + "markdown", "yaml", "python" + ], + // flagWords - list of words to be always considered incorrect + // This is useful for offensive words and common spelling errors. + // For example "hte" should be "the" + "flagWords": [], + "allowCompoundWords": true, + "ignorePaths": [ + "./element_array_ephys.egg-info/*", + "./images/*" + ], + "words": [ + "aggr", + "apmeta", + "arange", + "asarray", + "astype", + "autocorrelogram", + "bbins", + "bdist", + "Binarize", + "catgt", + "cbar", + "cbin", + "cdat", + "Chans", + "chans", + "chns", + "cmap", + "correlogram", + "correlograms", + "decomp", + "DISTRO", + "djbase", + "dtype", + "ecephys", + "elif", + "Ephys", + "gblcar", + "gfix", + "hstack", + "ibllib", + "ifnull", + "Imax", + "IMAX", + "imax", + "imec", + "imread", + "imro", + "imrotbl", + "imshow", + "inlinehilite", + "ipywidgets", + "kcoords", + "lfmeta", + "linenums", + "mdict", + "mkdocs", + "mkdocstrings", + "mtscomp", + "Nchan", + "nchan", + "ndarray", + "ndim", + "ndimage", + "NEURO", + "Neuropix", + "NeuroPixels", + "oebin", + "openephys", + "openephys", + "openpyxl", + "phylog", + "plotly", + "pykilosort", + "pymdownx", + "pynwb", + "pyopenephys", + "pyplot", + "pytest", + "repolarization", + "Roboto", + "scipy", + "sdist", + "sess", + "SGLX", + "spikeglx", + "spkcount", + "Stereotaxic","acorr", + "tcat", + "tickvals", + "tofile", + "vline", + "Vmax", + "vmax", + "xanchor", + "xaxes", + "xaxis", + "xcoords", + "xcorr", + "xlabel", + "xlim", + "XPOS", + "xtick", + "yaxes", + "yaxis", + "ycoord", + "ycoords", + "ylabel", + "ylim", + "YPOS", + "yref", + "yticks" + ] +} diff --git a/docs/mkdocs.yaml b/docs/mkdocs.yaml index f600e7df..373ad738 100644 --- a/docs/mkdocs.yaml +++ b/docs/mkdocs.yaml @@ -43,7 +43,7 @@ nav: # HOST_UID=$(id -u) docker compose -f docs/docker-compose.yaml up --build # ``` # 02. Site analytics depend on a local environment variable GOOGLE_ANALYTICS_KEY -# You can find this in LastPass or declare with any string to suprress errors +# You can find this in LastPass or declare with any string to suppress errors # 03. The API section will pull docstrings. # A. Follow google styleguide e.g., # https://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html diff --git a/element_array_ephys/ephys_acute.py b/element_array_ephys/ephys_acute.py index 21aaa7bb..974ef8f3 100644 --- a/element_array_ephys/ephys_acute.py +++ b/element_array_ephys/ephys_acute.py @@ -32,7 +32,7 @@ def activate( Args: ephys_schema_name (str): A string containing the name of the ephys schema. - probe_schema_name (str): A string containing the name of the probe scehma. + probe_schema_name (str): A string containing the name of the probe schema. create_schema (bool): If True, schema will be created in the database. create_tables (bool): If True, tables related to the schema will be created in the database. linking_module (str): A string containing the module name or module containing the required dependencies to activate the schema. diff --git a/element_array_ephys/ephys_chronic.py b/element_array_ephys/ephys_chronic.py index cb0050c0..fe65f735 100644 --- a/element_array_ephys/ephys_chronic.py +++ b/element_array_ephys/ephys_chronic.py @@ -31,7 +31,7 @@ def activate( Args: ephys_schema_name (str): A string containing the name of the ephys schema. - probe_schema_name (str): A string containing the name of the probe scehma. + probe_schema_name (str): A string containing the name of the probe schema. create_schema (bool): If True, schema will be created in the database. create_tables (bool): If True, tables related to the schema will be created in the database. linking_module (str): A string containing the module name or module containing the required dependencies to activate the schema. diff --git a/element_array_ephys/ephys_no_curation.py b/element_array_ephys/ephys_no_curation.py index ce36d87b..c93a6cd1 100644 --- a/element_array_ephys/ephys_no_curation.py +++ b/element_array_ephys/ephys_no_curation.py @@ -32,7 +32,7 @@ def activate( Args: ephys_schema_name (str): A string containing the name of the ephys schema. - probe_schema_name (str): A string containing the name of the probe scehma. + probe_schema_name (str): A string containing the name of the probe schema. create_schema (bool): If True, schema will be created in the database. create_tables (bool): If True, tables related to the schema will be created in the database. linking_module (str): A string containing the module name or module containing the required dependencies to activate the schema. diff --git a/element_array_ephys/ephys_precluster.py b/element_array_ephys/ephys_precluster.py index a9912ae6..1d367f6c 100644 --- a/element_array_ephys/ephys_precluster.py +++ b/element_array_ephys/ephys_precluster.py @@ -27,7 +27,7 @@ def activate( Args: ephys_schema_name (str): A string containing the name of the ephys schema. - probe_schema_name (str): A string containing the name of the probe scehma. + probe_schema_name (str): A string containing the name of the probe schema. create_schema (bool): If True, schema will be created in the database. create_tables (bool): If True, tables related to the schema will be created in the database. linking_module (str): A string containing the module name or module containing the required dependencies to activate the schema. @@ -436,7 +436,7 @@ class Step(dj.Part): @schema class PreClusterTask(dj.Manual): - """Defines a pre-clusting task ready to be run. + """Defines a pre-clustering task ready to be run. Attributes: EphysRecording (foreign key): EphysRecording primary key. diff --git a/element_array_ephys/plotting/qc.py b/element_array_ephys/plotting/qc.py index f7b62f3b..eb5d7709 100644 --- a/element_array_ephys/plotting/qc.py +++ b/element_array_ephys/plotting/qc.py @@ -28,7 +28,7 @@ def __init__( key (dict, optional): key from ephys.QualityMetric table. Defaults to None. scale (float, optional): Scale at which to render figure. Defaults to 1.4. fig_width (int, optional): Figure width in pixels. Defaults to 800. - amplitude_cutoff_maximum (float, optional): Cutoff for unit ampliude in + amplitude_cutoff_maximum (float, optional): Cutoff for unit amplitude in visualizations. Defaults to None. presence_ratio_minimum (float, optional): Cutoff for presence ratio in visualizations. Defaults to None. @@ -61,7 +61,7 @@ def key(self) -> dict: def key(self, key: dict): """Use class_instance.key = your_key to reset key""" if key not in self._ephys.QualityMetrics.fetch("KEY"): - # If not already full key, check if unquely identifies entry + # If not already full key, check if uniquely identifies entry key = (self._ephys.QualityMetrics & key).fetch1("KEY") self._key = key @@ -127,7 +127,7 @@ def units(self) -> pd.DataFrame: def _format_fig( self, fig: go.Figure = None, scale: float = None, ratio: float = 1.0 ) -> go.Figure: - """Return formatted figure or apply prmatting to existing figure + """Return formatted figure or apply formatting to existing figure Args: fig (go.Figure, optional): Apply formatting to this plotly graph object @@ -252,7 +252,7 @@ def get_grid(self, n_columns: int = 4, scale: float = 1.0) -> go.Figure: """Plot grid of histograms as subplots in go.Figure using n_columns Args: - n_columns (int, optional): Number of colums in grid. Defaults to 4. + n_columns (int, optional): Number of column in grid. Defaults to 4. scale (float, optional): Scale to render fig. Defaults to scale at class init, 1. @@ -334,7 +334,7 @@ def get_grid(self, n_columns: int = 4, scale: float = 1.0) -> go.Figure: @property def plot_list(self): - """List of plots that can be rendered inidividually by name or as grid""" + """List of plots that can be rendered individually by name or as grid""" if not self._plots: _ = self.plots return [plot for plot in self._plots] From 4ef0b6db9a27fd7ee68fcc48744ee98981947156 Mon Sep 17 00:00:00 2001 From: CBroz1 Date: Thu, 12 Jan 2023 15:36:43 -0600 Subject: [PATCH 06/19] Update License 2023 --- LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index d394fe35..386e2980 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2021 DataJoint NEURO +Copyright (c) 2023 DataJoint Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal From 64e7dc690c7cbf1f4051d4cbd3e5d29f3bfd9218 Mon Sep 17 00:00:00 2001 From: CBroz1 Date: Thu, 12 Jan 2023 15:57:11 -0600 Subject: [PATCH 07/19] More spellcheck; Markdown linting --- .markdownlint.yaml | 16 ++++ CHANGELOG.md | 23 +++-- CODE_OF_CONDUCT.md | 1 - CONTRIBUTING.md | 4 +- cspell.json | 31 ++++++- docs/src/changelog.md | 0 docs/src/citation.md | 10 ++- docs/src/concepts.md | 109 +++++++++++++++++------ docs/src/index.md | 8 +- element_array_ephys/export/nwb/README.md | 30 +++---- 10 files changed, 170 insertions(+), 62 deletions(-) create mode 100644 .markdownlint.yaml mode change 100644 => 120000 docs/src/changelog.md diff --git a/.markdownlint.yaml b/.markdownlint.yaml new file mode 100644 index 00000000..7420359b --- /dev/null +++ b/.markdownlint.yaml @@ -0,0 +1,16 @@ +# Markdown Linter configuration for docs +# https://github.com/DavidAnson/markdownlint +# https://github.com/DavidAnson/markdownlint/blob/main/doc/Rules.md +MD009: false # permit trailing spaces +MD007: false # List indenting - permit 4 spaces +MD013: + line_length: "88" # Line length limits + tables: false # disable for tables + headings: false # disable for headings +MD030: false # Number of spaces after a list +MD033: # HTML elements allowed + allowed_elements: + - "br" +MD034: false # Permit bare URLs +MD031: false # Spacing w/code blocks. Conflicts with `??? Note` and code tab styling +MD046: false # Spacing w/code blocks. Conflicts with `??? Note` and code tab styling diff --git a/CHANGELOG.md b/CHANGELOG.md index ea0a1407..c2ad3716 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,19 +6,23 @@ Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and ## [0.2.2] - 2022-01-11 + Bugfix - Revert import order in `__init__.py` to avoid circular import error. -+ Update - `.pre-commit-config.yaml` to disable automatic positioning of import statement at the top. ++ Update - `.pre-commit-config.yaml` to disable automatic positioning of import + statement at the top. + Bugfix - Update docstrings to render API for documentation website. ## [0.2.1] - 2022-01-06 -+ Add - `build_electrode_layouts` function in `probe.py` to compute the electrode layout for all types of probes. -+ Update - parameterize run_CatGT step from parameters retrieved from `ClusteringParamSet` table ++ Add - `build_electrode_layouts` function in `probe.py` to compute the electrode layout + for all types of probes. ++ Update - parameterize run_CatGT step from parameters retrieved from + `ClusteringParamSet` table + Update - clustering step, update duration for "median_subtraction" step + Bugfix - handles single probe recording in "Neuropix-PXI" format + Update - safeguard in creating/inserting probe types upon probe activation + Add - quality control metric dashboard + Update & fix docstrings -+ Update - `ephys_report.UnitLevelReport` to add `ephys.ClusterQualityLabel` as a foreign key reference ++ Update - `ephys_report.UnitLevelReport` to add `ephys.ClusterQualityLabel` as a + foreign key reference + Add - `.pre-commit-config.yaml` ## [0.2.0] - 2022-10-28 @@ -28,7 +32,8 @@ Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and + Add - Add `ephys_no_curation` and routines to trigger spike-sorting analysis using Kilosort (2.0, 2.5) + Add - mkdocs for Element Documentation -+ Add - New `QualityMetrics` table to store clusters' and waveforms' metrics after the spike sorting analysis. ++ Add - New `QualityMetrics` table to store clusters' and waveforms' metrics after the + spike sorting analysis. ## [0.1.4] - 2022-07-11 @@ -36,7 +41,8 @@ Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and ## [0.1.3] - 2022-06-16 -+ Update - Allow for the `precluster_output_dir` attribute to be nullable when no pre-clustering is performed. ++ Update - Allow for the `precluster_output_dir` attribute to be nullable when no + pre-clustering is performed. ## [0.1.2] - 2022-06-09 @@ -49,7 +55,8 @@ Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and ## [0.1.0] - 2022-05-26 + Update - Rename module for acute probe insertions from `ephys.py` to `ephys_acute.py`. -+ Add - Module for pre-clustering steps (`ephys_precluster.py`), which is built off of `ephys_acute.py`. ++ Add - Module for pre-clustering steps (`ephys_precluster.py`), which is built off of + `ephys_acute.py`. + Add - Module for chronic probe insertions (`ephys_chronic.py`). + Bugfix - Missing `fileTimeSecs` key in SpikeGLX meta file. + Update - Move common functions to `element-interface` package. @@ -59,7 +66,6 @@ Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and + Add - Processing with Kilosort and pyKilosort for Open Ephys and SpikeGLX - ## [0.1.0b0] - 2021-05-07 + Update - First beta release @@ -72,7 +78,6 @@ Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and + Add - Probe table supporting: Neuropixels probes 1.0 - 3A, 1.0 - 3B, 2.0 - SS, 2.0 - MS - [0.2.2]: https://github.com/datajoint/element-array-ephys/releases/tag/0.2.2 [0.2.1]: https://github.com/datajoint/element-array-ephys/releases/tag/0.2.1 [0.2.0]: https://github.com/datajoint/element-array-ephys/releases/tag/0.2.0 diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index 684cf81d..05025283 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -1,4 +1,3 @@ - # Contributor Covenant Code of Conduct ## Our Pledge diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b2f05e64..e04d1708 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,3 +1,5 @@ # Contribution Guidelines -This project follows the [DataJoint Contribution Guidelines](https://datajoint.com/docs/community/contribute/). Please reference the link for more full details. +This project follows the +[DataJoint Contribution Guidelines](https://datajoint.com/docs/community/contribute/). +Please reference the link for more full details. diff --git a/cspell.json b/cspell.json index 242291f0..09417178 100644 --- a/cspell.json +++ b/cspell.json @@ -16,15 +16,21 @@ "./images/*" ], "words": [ + "acorr", "aggr", + "Alessio", + "Andreas", "apmeta", "arange", "asarray", "astype", "autocorrelogram", + "Axona", "bbins", "bdist", "Binarize", + "Brody", + "Buccino", "catgt", "cbar", "cbin", @@ -32,18 +38,24 @@ "Chans", "chans", "chns", + "Clust", + "clusterings", "cmap", "correlogram", "correlograms", + "curations", + "DANDI", "decomp", "DISTRO", "djbase", "dtype", "ecephys", + "electrophysiogical", "elif", "Ephys", "gblcar", "gfix", + "HHMI", "hstack", "ibllib", "ifnull", @@ -58,19 +70,26 @@ "inlinehilite", "ipywidgets", "kcoords", + "Klusta", + "Kwik", "lfmeta", "linenums", "mdict", + "Mesoscale", "mkdocs", "mkdocstrings", + "Moser", "mtscomp", "Nchan", "nchan", "ndarray", "ndim", "ndimage", + "Neuralynx", "NEURO", + "Neurodata", "Neuropix", + "neuropixel", "NeuroPixels", "oebin", "openephys", @@ -78,24 +97,33 @@ "openpyxl", "phylog", "plotly", + "PSTH", "pykilosort", "pymdownx", "pynwb", "pyopenephys", "pyplot", "pytest", + "Reimer", "repolarization", "Roboto", + "RRID", + "Rxiv", + "Sasaki", "scipy", "sdist", + "Siegle", "sess", "SGLX", + "Shen", + "Sitonic", "spikeglx", "spkcount", - "Stereotaxic","acorr", + "Stereotaxic", "tcat", "tickvals", "tofile", + "Tolias", "vline", "Vmax", "vmax", @@ -108,6 +136,7 @@ "xlim", "XPOS", "xtick", + "Yatsenko", "yaxes", "yaxis", "ycoord", diff --git a/docs/src/changelog.md b/docs/src/changelog.md deleted file mode 100644 index 699cc9e7..00000000 --- a/docs/src/changelog.md +++ /dev/null @@ -1 +0,0 @@ -../../CHANGELOG.md \ No newline at end of file diff --git a/docs/src/changelog.md b/docs/src/changelog.md new file mode 120000 index 00000000..699cc9e7 --- /dev/null +++ b/docs/src/changelog.md @@ -0,0 +1 @@ +../../CHANGELOG.md \ No newline at end of file diff --git a/docs/src/citation.md b/docs/src/citation.md index 734ec9cb..34ea0ab0 100644 --- a/docs/src/citation.md +++ b/docs/src/citation.md @@ -1,7 +1,11 @@ # Citation -If your work uses this Element, please cite the following manuscript and Research Resource Identifier (RRID): +If your work uses this Element, please cite the following manuscript and Research +Resource Identifier (RRID): -+ Yatsenko D, Nguyen T, Shen S, Gunalan K, Turner CA, Guzman R, Sasaki M, Sitonic D, Reimer J, Walker EY, Tolias AS. DataJoint Elements: Data Workflows for Neurophysiology. bioRxiv. 2021 Jan 1. doi: https://doi.org/10.1101/2021.03.30.437358 ++ Yatsenko D, Nguyen T, Shen S, Gunalan K, Turner CA, Guzman R, Sasaki M, Sitonic D, + Reimer J, Walker EY, Tolias AS. DataJoint Elements: Data Workflows for + Neurophysiology. bioRxiv. 2021 Jan 1. doi: https://doi.org/10.1101/2021.03.30.437358 -+ DataJoint Elements ([RRID:SCR_021894](https://scicrunch.org/resolver/SCR_021894)) - Element Array Electrophysiology (version {{ PATCH_VERSION }}) \ No newline at end of file ++ DataJoint Elements ([RRID:SCR_021894](https://scicrunch.org/resolver/SCR_021894)) - + Element Array Electrophysiology (version {{ PATCH_VERSION }}) diff --git a/docs/src/concepts.md b/docs/src/concepts.md index 21a2d16f..cee2df32 100644 --- a/docs/src/concepts.md +++ b/docs/src/concepts.md @@ -3,12 +3,28 @@ ## Acquisition Tools for Electrophysiology ### Neuropixels Probes -Neuropixels probes were developed by a collaboration between HHMI Janelia, industry partners, and others[1](#references). Since their initial release in October 2018, 300 labs have ordered 1200 probes. Since the rollout of Neuropixels 2.0 in October 2020, IMEC has been shipping 100+ probes monthly (correspondence with Tim Harris). -Neuropixels probes offer 960 electrode sites along a 10mm long shank, with 384 recordable channels per probe that can record hundreds of units spanning multiple brain regions (Neuropixels 2.0 version is a 4-shank probe with 1280 electrode sites per shank). Such large recording capacity has offered tremendous opportunities for the field of neurophysiology research, yet this is accompanied by an equally great challenge in terms of data and computation management. +Neuropixels probes were developed by a collaboration between HHMI Janelia, industry +partners, and others[^1]. Since their initial release in October +2018, 300 labs have ordered 1200 probes. Since the rollout of Neuropixels 2.0 in October +2020, IMEC has been shipping 100+ probes monthly (correspondence with Tim Harris). + +Neuropixels probes offer 960 electrode sites along a 10mm long shank, with 384 +recordable channels per probe that can record hundreds of units spanning multiple brain +regions (Neuropixels 2.0 version is a 4-shank probe with 1280 electrode sites per +shank). Such large recording capacity has offered tremendous opportunities for the field +of neurophysiology research, yet this is accompanied by an equally great challenge in +terms of data and computation management. + +[^1]: + Jun, J., Steinmetz, N., Siegle, J. et al. Fully integrated silicon probes for + high-density recording of neural activity. *Nature* 551, 232–236 (2017). + [https://doi.org/10.1038/nature24636](https://doi.org/10.1038/nature24636). ### Data Acquisition Tools -Some commonly used acquisiton tools and systems by the neuroscience research community include: + +Some commonly used acquisition tools and systems by the neuroscience research community +include: + [Neuropixels probes](https://www.neuropixels.org) + Tetrodes @@ -19,9 +35,16 @@ Some commonly used acquisiton tools and systems by the neuroscience research com + ... ### Data Preprocessing Tools -The preprocessing pipeline includes bandpass filtering for LFP extraction, bandpass filtering for spike sorting, spike sorting, manual curation of the spike sorting results, and calculation of quality control metrics. In trial-based experiments, the spike trains are aligned and separated into trials. Standard processing may include PSTH computation aligned to trial onset or other events, and often grouped by different trial types. Neuroscience groups have traditionally developed custom home-made toolchains. -In recent years, several leaders have been emerging as de facto standards with significant community uptake: +The preprocessing pipeline includes bandpass filtering for LFP extraction, bandpass +filtering for spike sorting, spike sorting, manual curation of the spike sorting +results, and calculation of quality control metrics. In trial-based experiments, the +spike trains are aligned and separated into trials. Standard processing may include PSTH +computation aligned to trial onset or other events, and often grouped by different trial +types. Neuroscience groups have traditionally developed custom home-made toolchains. + +In recent years, several leaders have been emerging as de facto standards with +significant community uptake: + [Kilosort](https://github.com/MouseLand/Kilosort) + [pyKilosort](https://github.com/MouseLand/pykilosort) @@ -33,27 +56,44 @@ In recent years, several leaders have been emerging as de facto standards with s + [spikeforest](https://spikeforest.flatironinstitute.org/) + ... -Kilosort provides most automation and has gained significant popularity, being adopted as one of the key spike sorting methods in the majority of the teams/collaborations we have worked with. As part of our Year-1 NIH U24 effort, we provide support for data ingestion of spike sorting results from Kilosort. Further effort will be devoted for the ingestion support of other spike sorting methods. On this end, a framework for unifying existing spike sorting methods, named [SpikeInterface](https://github.com/SpikeInterface/spikeinterface), has been developed by Alessio Buccino, et al. SpikeInterface provides a convenient Python-based wrapper to invoke, extract, compare spike sorting results from different sorting algorithms. +Kilosort provides most automation and has gained significant popularity, being adopted +as one of the key spike sorting methods in the majority of the teams/collaborations we +have worked with. As part of our Year-1 NIH U24 effort, we provide support for data +ingestion of spike sorting results from Kilosort. Further effort will be devoted for the +ingestion support of other spike sorting methods. On this end, a framework for unifying +existing spike sorting methods, named +[SpikeInterface](https://github.com/SpikeInterface/spikeinterface), has been developed +by Alessio Buccino, et al. SpikeInterface provides a convenient Python-based wrapper to +invoke, extract, compare spike sorting results from different sorting algorithms. ## Key Partnerships -Over the past few years, several labs have developed DataJoint-based data management and processing pipelines for Neuropixels probes. Our team collaborated with several of them during their projects. Additionally, we interviewed these teams to understand their experimental workflow, pipeline design, associated tools, and interfaces. These teams include: +Over the past few years, several labs have developed DataJoint-based data management and +processing pipelines for Neuropixels probes. Our team collaborated with several of them +during their projects. Additionally, we interviewed these teams to understand their +experimental workflow, pipeline design, associated tools, and interfaces. These teams +include: -- [International Brain Lab](https://internationalbrainlab.org) - [https://github.com/int-brain-lab/IBL-pipeline](https://github.com/int-brain-lab/IBL-pipeline) ++ [International Brain Lab](https://internationalbrainlab.org): + [https://github.com/int-brain-lab/IBL-pipeline](https://github.com/int-brain-lab/IBL-pipeline) -- [Mesoscale Activity Project (HHMI Janelia)](https://github.com/mesoscale-activity-map) - [https://github.com/mesoscale-activity-map/map-ephys](https://github.com/mesoscale-activity-map/map-ephys) ++ [Mesoscale Activity Project (HHMI Janelia)](https://github.com/mesoscale-activity-map): + [https://github.com/mesoscale-activity-map/map-ephys](https://github.com/mesoscale-activity-map/map-ephys) -- Moser Group (Norwegian University of Science and Technology) - see [pipeline design](https://moser-pipelines.readthedocs.io/en/latest/ephys/overview.html) ++ Moser Group (Norwegian University of Science and Technology): see + [pipeline design](https://moser-pipelines.readthedocs.io/en/latest/ephys/overview.html) -- Andreas Tolias Lab (Baylor College of Medicine) ++ Andreas Tolias Lab (Baylor College of Medicine) -- BrainCoGs (Princeton Neuroscience Institute) ++ BrainCoGs (Princeton Neuroscience Institute) -- Brody Lab (Princeton University) ++ Brody Lab (Princeton University) ## Element Architecture -Each of the DataJoint Elements creates a set of tables for common neuroscience data modalities to organize, preprocess, and analyze data. Each node in the following diagram is a table within the Element or a table connected to the Element. +Each of the DataJoint Elements creates a set of tables for common neuroscience data +modalities to organize, preprocess, and analyze data. Each node in the following diagram +is a table within the Element or a table connected to the Element. ### `ephys_acute` module @@ -68,7 +108,8 @@ Each of the DataJoint Elements creates a set of tables for common neuroscience d ![diagram](https://raw.githubusercontent.com/datajoint/element-array-ephys/main/images/attached_array_ephys_element_precluster.svg) ### `subject` schema ([API docs](https://datajoint.com/docs/elements/element-animal/api/element_animal/subject)) -- Although not required, most choose to connect the `Session` table to a `Subject` table. + +Although not required, most choose to connect the `Session` table to a `Subject` table. | Table | Description | | --- | --- | @@ -81,6 +122,7 @@ Each of the DataJoint Elements creates a set of tables for common neuroscience d | Session | A table for unique experimental session identifiers. | ### `probe` schema ([API docs](../api/element_array_ephys/probe)) + Tables related to the Neuropixels probe and electrode configuration. | Table | Description | @@ -92,7 +134,9 @@ Tables related to the Neuropixels probe and electrode configuration. | ElectrodeConfig.Electrode | A record of electrodes out of those in `ProbeType.Electrode` that are used for recording. | ### `ephys` schema ([API docs](../api/element_array_ephys/ephys)) -Tables related to information about physiological recordings and automatic ingestion of spike sorting results. + +Tables related to information about physiological recordings and automatic ingestion of +spike sorting results. | Table | Description | | --- | --- | @@ -101,10 +145,11 @@ Tables related to information about physiological recordings and automatic inges | Clustering | A table with clustering data for spike sorting extracellular electrophysiology data. | | Curation | A table to declare optional manual curation of spike sorting results. | | CuratedClustering | A table with metadata for sorted data generated after each curation. | -| CuratedClusting.Unit | A part table containing single unit information after spike sorting and optional curation. | +| CuratedClustering.Unit | A part table containing single unit information after spike sorting and optional curation. | | WaveformSet | A table containing spike waveforms for single units. | ### `ephys_report` schema ([API docs](../api/element_array_ephys/ephys_report)) + Tables for storing probe or unit-level visualization results. | Table | Description | @@ -114,17 +159,23 @@ Tables for storing probe or unit-level visualization results. ## Element Development -Through our interviews and direct collaboration on the precursor projects, we identified the common motifs to create the [Array Electrophysiology Element](https://github.com/datajoint/element-array-ephys). +Through our interviews and direct collaboration on the precursor projects, we identified +the common motifs to create the +[Array ElectrophysiologyElement](https://github.com/datajoint/element-array-ephys). Major features of the Array Electrophysiology Element include: + Pipeline architecture detailed by: - + Probe, electrode configuration compatible with Neuropixels probes and generalizable to other types of probes (e.g. tetrodes) - supporting both `chronic` and `acute` probe insertion modes. + + Probe, electrode configuration compatible with Neuropixels probes and + generalizable to other types of probes (e.g. tetrodes) - supporting both `chronic` + and `acute` probe insertion modes. - + Probe-insertion, ephys-recordings, LFP extraction, clusterings, curations, sorted units and the associated data (e.g. spikes, waveforms, etc.). + + Probe-insertion, ephys-recordings, LFP extraction, clusterings, curations, sorted + units and the associated data (e.g. spikes, waveforms, etc.). - + Store/track/manage different curations of the spike sorting results - supporting both curated clustering and kilosort triggered clustering (i.e., `no_curation`). + + Store/track/manage different curations of the spike sorting results - supporting + both curated clustering and kilosort triggered clustering (i.e., `no_curation`). + Ingestion support for data acquired with SpikeGLX and OpenEphys acquisition systems. + Ingestion support for spike sorting outputs from Kilosort. @@ -133,7 +184,14 @@ Major features of the Array Electrophysiology Element include: ## Data Export and Publishing -Element Array Electrophysiology supports exporting of all data into standard Neurodata Without Borders (NWB) files. This makes it easy to share files with collaborators and publish results on [DANDI Archive](https://dandiarchive.org/). [NWB](https://www.nwb.org/), as an organization, is dedicated to standardizing data formats and maximizing interoperability across tools for neurophysiology. For more information on uploading NWB files to DANDI within the DataJoint Elements ecosystem, visit our documentation for the DANDI upload feature of [Element Array Electrophysiology](datajoint.com/docs/elements/element-array-ephys/). +Element Array Electrophysiology supports exporting of all data into standard Neurodata +Without Borders (NWB) files. This makes it easy to share files with collaborators and +publish results on [DANDI Archive](https://dandiarchive.org/). +[NWB](https://www.nwb.org/), as an organization, is dedicated to standardizing data +formats and maximizing interoperability across tools for neurophysiology. For more +information on uploading NWB files to DANDI within the DataJoint Elements ecosystem, +visit our documentation for the DANDI upload feature of [Element Array +Electrophysiology](datajoint.com/docs/elements/element-array-ephys/). ## Roadmap @@ -146,8 +204,5 @@ NeurodataWithoutBorders format integrated Future additions to this element will add functionality to support large (> 48 hours) neuropixel recordings via an overlapping segmented processing approach. -Further development of this Element is community driven. Upon user requests we will continue adding features to this Element. - -## References - -[1]: Jun, J., Steinmetz, N., Siegle, J. et al. Fully integrated silicon probes for high-density recording of neural activity. *Nature* 551, 232–236 (2017). [https://doi.org/10.1038/nature24636](https://doi.org/10.1038/nature24636). \ No newline at end of file +Further development of this Element is community driven. Upon user requests we will +continue adding features to this Element. diff --git a/docs/src/index.md b/docs/src/index.md index 9476c2da..c8cc8a01 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -9,12 +9,12 @@ fully functional pipeline. ![diagram](https://raw.githubusercontent.com/datajoint/element-array-ephys/main/images/diagram_flowchart.svg) The Element is comprised of `probe` and `ephys` schemas. Several `ephys` schemas are -developed to handle various use cases of this pipeline and workflow: +developed to handle various use cases of this pipeline and workflow: -+ `ephys_acute`: A probe is inserted into a new location during each session. ++ `ephys_acute`: A probe is inserted into a new location during each session. + `ephys_chronic`: A probe is inserted once and used to record across multiple - sessions. + sessions. + `ephys_precluster`: A probe is inserted into a new location during each session. Pre-clustering steps are performed on the data from each probe prior to Kilosort @@ -22,7 +22,7 @@ developed to handle various use cases of this pipeline and workflow: + `ephys_no_curation`: A probe is inserted into a new location during each session and Kilosort-triggered clustering is performed without the option to manually curate the - results. + results. Visit the [Concepts page](./concepts.md) for more information about the use cases of `ephys` schemas and an explanation of the tables. To get started with building your own diff --git a/element_array_ephys/export/nwb/README.md b/element_array_ephys/export/nwb/README.md index d6b5b2e5..b5985498 100644 --- a/element_array_ephys/export/nwb/README.md +++ b/element_array_ephys/export/nwb/README.md @@ -1,15 +1,13 @@ # Exporting data to NWB The `export/nwb/nwb.py` module maps from the element-array-ephys data structure to NWB. -The main function is `ecephys_session_to_nwb`, which contains flags to control calling the following functions, -which can be called independently: +The main function is `ecephys_session_to_nwb`, which contains flags to control calling +the following functions, which can be called independently: 1. `session.export.nwb.session_to_nwb`: Gathers session-level metadata - -2. `add_electrodes_to_nwb`: Add electrodes table to NWBFile. This is needed for any ElectricalSeries, including - raw source data and LFP. - +2. `add_electrodes_to_nwb`: Add electrodes table to NWBFile. This is needed for any + ElectricalSeries, including raw source data and LFP. ephys.InsertionLocation -> ElectrodeGroup.location @@ -24,16 +22,16 @@ which can be called independently: probe.ProbeType.Electrode::shank_col -> electrodes["shank_col"] probe.ProbeType.Electrode::shank_row -> electrodes["shank_row"] -3. `add_ephys_recording_to_nwb`: Read voltage data directly from source files and iteratively transfer them to the - NWB file. Automatically applies lossless compression to the data, so the final file might be smaller than the original, but there is no - data loss. Currently supports Neuropixels data acquired with SpikeGLX or Open Ephys, and relies on SpikeInterface to read the data. - +3. `add_ephys_recording_to_nwb`: Read voltage data directly from source files and + iteratively transfer them to the NWB file. Automatically applies lossless compression + to the data, so the final file might be smaller than the original, but there is no + data loss. Currently supports Neuropixels data acquired with SpikeGLX or Open Ephys, + and relies on SpikeInterface to read the data. source data -> acquisition["ElectricalSeries"] 4. `add_ephys_units_to_nwb`: Add spiking data from CuratedClustering to NWBFile. - ephys.CuratedClustering.Unit::unit -> units.id ephys.CuratedClustering.Unit::spike_times -> units["spike_times"] ephys.CuratedClustering.Unit::spike_depths -> units["spike_depths"] @@ -41,15 +39,15 @@ which can be called independently: ephys.WaveformSet.PeakWaveform::peak_electrode_waveform -> units["waveform_mean"] -5. `add_ephys_lfp_from_dj_to_nwb`: Read LFP data from the data in element-array-ephys and convert to NWB. - +5. `add_ephys_lfp_from_dj_to_nwb`: Read LFP data from the data in element-array-ephys + and convert to NWB. ephys.LFP.Electrode::lfp -> processing["ecephys"].lfp.electrical_series["ElectricalSeries{insertion_number}"].data ephys.LFP::lfp_time_stamps -> processing["ecephys"].lfp.electrical_series["ElectricalSeries{insertion_number}"].timestamps -6. `add_ephys_lfp_from_source_to_nwb`: Read the LFP data directly from the source file. Currently, only works for - SpikeGLX data. Can be used instead of `add_ephys_lfp_from_dj_to_nwb`. - +6. `add_ephys_lfp_from_source_to_nwb`: Read the LFP data directly from the source file. + Currently, only works for SpikeGLX data. Can be used instead of + `add_ephys_lfp_from_dj_to_nwb`. source data -> processing["ecephys"].lfp.electrical_series["ElectricalSeries{insertion_number}"].data source data -> processing["ecephys"].lfp.electrical_series["ElectricalSeries{insertion_number}"].timestamps From 96bb6fa10207782e2d5249eda76007ebc453567d Mon Sep 17 00:00:00 2001 From: CBroz1 Date: Thu, 12 Jan 2023 17:37:24 -0600 Subject: [PATCH 08/19] Add tutorial notebook renders --- .gitignore | 1 + cspell.json | 1 + docs/.docker/pip_requirements.txt | 3 +- docs/docker-compose.yaml | 10 +++++- docs/mkdocs.yaml | 23 ++++++++++++- docs/src/api/make_pages.py | 14 -------- docs/src/index.md | 2 +- docs/src/tutorials.md | 3 -- docs/src/tutorials/index.md | 54 +++++++++++++++++++++++++++++++ 9 files changed, 90 insertions(+), 21 deletions(-) delete mode 100644 docs/src/tutorials.md create mode 100644 docs/src/tutorials/index.md diff --git a/.gitignore b/.gitignore index 2b997a95..e25ed045 100644 --- a/.gitignore +++ b/.gitignore @@ -102,6 +102,7 @@ ENV/ # mkdocs documentation docs/site +docs/src/tutorials/*ipynb # mypy .mypy_cache/ diff --git a/cspell.json b/cspell.json index 09417178..860b72d0 100644 --- a/cspell.json +++ b/cspell.json @@ -68,6 +68,7 @@ "imrotbl", "imshow", "inlinehilite", + "ipynb", "ipywidgets", "kcoords", "Klusta", diff --git a/docs/.docker/pip_requirements.txt b/docs/.docker/pip_requirements.txt index 9a1b23f5..5b7b2f4c 100644 --- a/docs/.docker/pip_requirements.txt +++ b/docs/.docker/pip_requirements.txt @@ -7,4 +7,5 @@ mdx-truly-sane-lists mkdocs-gen-files mkdocs-literate-nav mkdocs-exclude-search -mkdocs-markdownextradata-plugin \ No newline at end of file +mkdocs-markdownextradata-plugin +mkdocs-jupyter \ No newline at end of file diff --git a/docs/docker-compose.yaml b/docs/docker-compose.yaml index 8b1b10ef..12149475 100644 --- a/docs/docker-compose.yaml +++ b/docs/docker-compose.yaml @@ -28,9 +28,17 @@ services: - | git config --global --add safe.directory /main set -e + export ELEMENT_UNDERSCORE=$$(echo $${PACKAGE} | sed 's/element_//g') + export ELEMENT_HYPHEN=$$(echo $${ELEMENT_UNDERSCORE} | sed 's/_/-/g') export PATCH_VERSION=$$(cat /main/$${PACKAGE}/version.py | grep -oE '\d+\.\d+\.[a-z0-9]+') + git clone https://github.com/datajoint/workflow-$${ELEMENT_HYPHEN}.git /main/delete || true + if [ -d /main/delete/ ]; then + mv /main/delete/workflow_$${ELEMENT_UNDERSCORE} /main/ + mv /main/delete/notebooks/*ipynb /main/docs/src/tutorials/ + rm -fR /main/delete + fi if echo "$${MODE}" | grep -i live &>/dev/null; then - mkdocs serve --config-file ./docs/mkdocs.yaml -a 0.0.0.0:80 + mkdocs serve --config-file ./docs/mkdocs.yaml -a 0.0.0.0:80 2>&1 | tee docs/temp_mkdocs.log elif echo "$${MODE}" | grep -iE "qa|push" &>/dev/null; then echo "INFO::Delete gh-pages branch" git branch -D gh-pages || true diff --git a/docs/mkdocs.yaml b/docs/mkdocs.yaml index 373ad738..40c98b10 100644 --- a/docs/mkdocs.yaml +++ b/docs/mkdocs.yaml @@ -7,7 +7,19 @@ repo_name: datajoint/element-array-ephys nav: - Element Array Ephys: index.md - Concepts: concepts.md - - Tutorials: tutorials.md + - Tutorials: + - Overview: tutorials/index.md + - Data Download: tutorials/00-data-download-optional.ipynb + - Configure: tutorials/01-configure.ipynb + - Workflow Structure: tutorials/02-workflow-structure-optional.ipynb + - Process: tutorials/03-process.ipynb + - Automate: tutorials/04-automate-optional.ipynb + - Explore: tutorials/05-explore.ipynb + - Drop: tutorials/06-drop-optional.ipynb + - Downstream Analysis: tutorials/07-downstream-analysis.ipynb + - Visualizations: tutorials/10-data_visualization.ipynb + - Electrode Localization: tutorials/08-electrode-localization.ipynb + - NWB Export: tutorials/09-NWB-export.ipynb - Citation: citation.md - API: api/ # defer to gen-files + literate-nav - Changelog: changelog.md @@ -92,6 +104,12 @@ plugins: # "index.md": "getting_started.md" - mkdocstrings: default_handler: python + handlers: + python: + options: + members_order: source + group_by_category: false + line_length: 88 - gen-files: scripts: - ./src/api/make_pages.py @@ -100,6 +118,8 @@ plugins: - exclude-search: exclude: - "*/navigation.md" + - mkdocs-jupyter: + ignore_h1_titles: True markdown_extensions: - attr_list - toc: @@ -120,6 +140,7 @@ markdown_extensions: linenums: true - pymdownx.inlinehilite - pymdownx.snippets + - footnotes extra: PATCH_VERSION: !ENV PATCH_VERSION diff --git a/docs/src/api/make_pages.py b/docs/src/api/make_pages.py index 32ca57fe..ab88d3f2 100644 --- a/docs/src/api/make_pages.py +++ b/docs/src/api/make_pages.py @@ -7,24 +7,10 @@ import mkdocs_gen_files from pathlib import Path import os -import subprocess package = os.getenv("PACKAGE") element = package.split("_", 1)[1] -if not Path(f"workflow_{element}").is_dir(): - try: - subprocess.run( - f"git clone https://github.com/datajoint/workflow-{element.replace('_','-')}.git /main/delete".split( - " " - ), - check=True, - timeout=5, - ) - os.system(f"mv /main/delete/workflow_{element} /main/") - os.system("rm -fR /main/delete") - except subprocess.CalledProcessError: - pass # no repo found nav = mkdocs_gen_files.Nav() for path in sorted(Path(package).glob("**/*.py")) + sorted( diff --git a/docs/src/index.md b/docs/src/index.md index c8cc8a01..b21edcfc 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -26,4 +26,4 @@ developed to handle various use cases of this pipeline and workflow: Visit the [Concepts page](./concepts.md) for more information about the use cases of `ephys` schemas and an explanation of the tables. To get started with building your own -data pipeline, visit the [Tutorials page](./tutorials.md). +data pipeline, visit the [Tutorials page](./tutorials/index.md). diff --git a/docs/src/tutorials.md b/docs/src/tutorials.md deleted file mode 100644 index 87ba953f..00000000 --- a/docs/src/tutorials.md +++ /dev/null @@ -1,3 +0,0 @@ -# Tutorials - -Coming soon! \ No newline at end of file diff --git a/docs/src/tutorials/index.md b/docs/src/tutorials/index.md new file mode 100644 index 00000000..5f367cd9 --- /dev/null +++ b/docs/src/tutorials/index.md @@ -0,0 +1,54 @@ +# Tutorials + +## Installation + +Installation of the Element requires an integrated development environment and database. +Instructions to setup each of the components can be found on the +[User Instructions](https://datajoint.com/docs/elements/user-guide/) page. These +instructions use the example +[workflow for Element Array Ephys](https://github.com/datajoint/workflow-array-ephys), +which can be modified for a user's specific experimental requirements. This example +workflow uses several Elements (Lab, Animal, Session, Event, and Electrophysiology) to construct +a complete pipeline, and is able to ingest experimental metadata and run model training +and inference. + +### Videos + +The [Element Array Ephys tutorial](https://youtu.be/KQlGYOBq7ow?t=3658) gives an +overview of the workflow files and notebooks as well as core concepts related to +Electrophysiology. + +[![YouTube tutorial](https://img.youtube.com/vi/KQlGYOBq7ow/0.jpg)](https://youtu.be/KQlGYOBq7ow?t=3658) + +### Notebooks + +Each of the notebooks in the workflow +([download here](https://github.com/datajoint/workflow-array-ephys/tree/main/notebooks) +steps through ways to interact with the Element itself. For convenience, these notebooks +are also rendered as part of this site. To try out the Elements notebooks in an online +Jupyter environment with access to example data, visit +[CodeBook](https://codebook.datajoint.io/). (Electrophysiology notebooks coming soon!) + +- [Data Download](./00-data-download-optional.ipynb) highlights how to use DataJoint + tools to download a sample model for trying out the Element. +- [Configure](./01-configure.ipynb) helps configure your local DataJoint installation to + point to the correct database. +- [Workflow Structure](./02-workflow-structure-optional.ipynb) demonstrates the table + architecture of the Element and key DataJoint basics for interacting with these + tables. +- [Process](./03-process.ipynb) steps through adding data to these tables and launching + key Electrophysiology features, like model training. +- [Automate](./04-automate-optional.ipynb) highlights the same steps as above, but + utilizing all built-in automation tools. +- [Explore](./05-explore.ipynb) demonstrates how to fetch data from the Element. +- [Drop schemas](./06-drop-optional.ipynb) provides the steps for dropping all the + tables to start fresh. +- [Downstream Analysis](./07-downstream-analysis.ipynb) highlights how to link + this Element to Element Event for event-based analyses. +- [Visualizations](./10-data_visualization.ipynb) highlights how to use a built-in module + for visualizing units, probes and quality metrics. +- [Electrode Localization](./08-electrode-localization.ipynb) demonstrates how to link + this Element to + [Element Electrode Localization](https://datajoint.com/docs/elements/element-electrode-localization/). +- [NWB Export](./09-NWB-export.ipynb) highlights the export functionality available for the + `no-curation` schema. From 0980242d1502d09612b424ce7b9f06a250d11342 Mon Sep 17 00:00:00 2001 From: CBroz1 Date: Thu, 12 Jan 2023 17:37:41 -0600 Subject: [PATCH 09/19] Fix docstrings --- element_array_ephys/ephys_acute.py | 11 +- element_array_ephys/ephys_chronic.py | 9 +- element_array_ephys/ephys_no_curation.py | 13 +- element_array_ephys/ephys_precluster.py | 6 +- element_array_ephys/export/nwb/nwb.py | 310 +++++++++++------------ 5 files changed, 171 insertions(+), 178 deletions(-) diff --git a/element_array_ephys/ephys_acute.py b/element_array_ephys/ephys_acute.py index 974ef8f3..b9f75845 100644 --- a/element_array_ephys/ephys_acute.py +++ b/element_array_ephys/ephys_acute.py @@ -718,15 +718,18 @@ class ClusteringTask(dj.Manual): """ @classmethod - def infer_output_dir(cls, key: dict, relative: bool = False, mkdir: bool = False): + def infer_output_dir( + cls, key: dict, relative: bool = False, mkdir: bool = False + ) -> pathlib.Path: """Infer output directory if it is not provided. Args: key (dict): ClusteringTask primary key. Returns: - Pathlib.Path: Expected clustering_output_dir based on the following convention: processed_dir / session_dir / probe_{insertion_number} / {clustering_method}_{paramset_idx} - e.g.: sub4/sess1/probe_2/kilosort2_0 + Expected clustering_output_dir based on the following convention: + processed_dir / session_dir / probe_{insertion_number} / {clustering_method}_{paramset_idx} + e.g.: sub4/sess1/probe_2/kilosort2_0 """ processed_dir = pathlib.Path(get_processed_root_data_dir()) session_dir = find_full_path( @@ -1325,7 +1328,7 @@ class Waveform(dj.Part): recovery_slope (float): Slope of the regression line fit to first 30 microseconds from peak to tail. spread (float): The range with amplitude over 12-percent of maximum amplitude along the probe. velocity_above (float): inverse velocity of waveform propagation from soma to the top of the probe. - velocity_below (float) inverse velocity of waveform propagation from soma toward the bottom of the probe. + velocity_below (float): inverse velocity of waveform propagation from soma toward the bottom of the probe. """ definition = """ diff --git a/element_array_ephys/ephys_chronic.py b/element_array_ephys/ephys_chronic.py index fe65f735..a15d56eb 100644 --- a/element_array_ephys/ephys_chronic.py +++ b/element_array_ephys/ephys_chronic.py @@ -651,15 +651,16 @@ class ClusteringTask(dj.Manual): """ @classmethod - def infer_output_dir(cls, key, relative=False, mkdir=False): + def infer_output_dir(cls, key, relative=False, mkdir=False) -> pathlib.Path: """Infer output directory if it is not provided. Args: key (dict): ClusteringTask primary key. Returns: - Pathlib.Path: Expected clustering_output_dir based on the following convention: processed_dir / session_dir / probe_{insertion_number} / {clustering_method}_{paramset_idx} - e.g.: sub4/sess1/probe_2/kilosort2_0 + Expected clustering_output_dir based on the following convention: + processed_dir / session_dir / probe_{insertion_number} / {clustering_method}_{paramset_idx} + e.g.: sub4/sess1/probe_2/kilosort2_0 """ processed_dir = pathlib.Path(get_processed_root_data_dir()) sess_dir = find_full_path(get_ephys_root_data_dir(), get_session_directory(key)) @@ -1256,7 +1257,7 @@ class Waveform(dj.Part): recovery_slope (float): Slope of the regression line fit to first 30 microseconds from peak to tail. spread (float): The range with amplitude over 12-percent of maximum amplitude along the probe. velocity_above (float): inverse velocity of waveform propagation from soma to the top of the probe. - velocity_below (float) inverse velocity of waveform propagation from soma toward the bottom of the probe. + velocity_below (float): inverse velocity of waveform propagation from soma toward the bottom of the probe. """ definition = """ diff --git a/element_array_ephys/ephys_no_curation.py b/element_array_ephys/ephys_no_curation.py index c93a6cd1..6b89f6c8 100644 --- a/element_array_ephys/ephys_no_curation.py +++ b/element_array_ephys/ephys_no_curation.py @@ -614,7 +614,7 @@ class ClusteringParamSet(dj.Lookup): ClusteringMethod (dict): ClusteringMethod primary key. paramset_desc (varchar(128) ): Description of the clustering parameter set. param_set_hash (uuid): UUID hash for the parameter set. - params (longblob) + params (longblob): Set of clustering parameters """ definition = """ @@ -724,15 +724,18 @@ class ClusteringTask(dj.Manual): """ @classmethod - def infer_output_dir(cls, key, relative: bool = False, mkdir: bool = False): + def infer_output_dir( + cls, key, relative: bool = False, mkdir: bool = False + ) -> pathlib.Path: """Infer output directory if it is not provided. Args: key (dict): ClusteringTask primary key. Returns: - Pathlib.Path: Expected clustering_output_dir based on the following convention: processed_dir / session_dir / probe_{insertion_number} / {clustering_method}_{paramset_idx} - e.g.: sub4/sess1/probe_2/kilosort2_0 + Expected clustering_output_dir based on the following convention: + processed_dir / session_dir / probe_{insertion_number} / {clustering_method}_{paramset_idx} + e.g.: sub4/sess1/probe_2/kilosort2_0 """ processed_dir = pathlib.Path(get_processed_root_data_dir()) session_dir = find_full_path( @@ -1265,7 +1268,7 @@ class Waveform(dj.Part): recovery_slope (float): Slope of the regression line fit to first 30 microseconds from peak to tail. spread (float): The range with amplitude over 12-percent of maximum amplitude along the probe. velocity_above (float): inverse velocity of waveform propagation from soma to the top of the probe. - velocity_below (float) inverse velocity of waveform propagation from soma toward the bottom of the probe. + velocity_below (float): inverse velocity of waveform propagation from soma toward the bottom of the probe. """ definition = """ diff --git a/element_array_ephys/ephys_precluster.py b/element_array_ephys/ephys_precluster.py index 1d367f6c..a17f3cc8 100644 --- a/element_array_ephys/ephys_precluster.py +++ b/element_array_ephys/ephys_precluster.py @@ -702,7 +702,7 @@ class ClusteringParamSet(dj.Lookup): ClusteringMethod (dict): ClusteringMethod primary key. paramset_desc (varchar(128) ): Description of the clustering parameter set. param_set_hash (uuid): UUID hash for the parameter set. - params (longblob) + params (longblob): Paramset, dictionary of all applicable parameters. """ definition = """ @@ -723,7 +723,7 @@ def insert_new_params( """Inserts new parameters into the ClusteringParamSet table. Args: - clustering_method (str): name of the clustering method. + processing_method (str): name of the clustering method. paramset_desc (str): description of the parameter set params (dict): clustering parameters paramset_idx (int, optional): Unique parameter set ID. Defaults to None. @@ -1245,7 +1245,7 @@ class Waveform(dj.Part): recovery_slope (float): Slope of the regression line fit to first 30 microseconds from peak to tail. spread (float): The range with amplitude over 12-percent of maximum amplitude along the probe. velocity_above (float): inverse velocity of waveform propagation from soma to the top of the probe. - velocity_below (float) inverse velocity of waveform propagation from soma toward the bottom of the probe. + velocity_below (float): inverse velocity of waveform propagation from soma toward the bottom of the probe. """ definition = """ diff --git a/element_array_ephys/export/nwb/nwb.py b/element_array_ephys/export/nwb/nwb.py index df30bf88..919a1ebb 100644 --- a/element_array_ephys/export/nwb/nwb.py +++ b/element_array_ephys/export/nwb/nwb.py @@ -6,11 +6,12 @@ import datajoint as dj import numpy as np import pynwb +from datajoint.table import Table from element_interface.utils import find_full_path from hdmf.backends.hdf5 import H5DataIO from hdmf.data_utils import GenericDataChunkIterator -from nwb_conversion_tools.tools.nwb_helpers import get_module -from nwb_conversion_tools.tools.spikeinterface.spikeinterfacerecordingdatachunkiterator import ( +from neuroconv.tools.nwb_helpers import get_module +from neuroconv.tools.spikeinterface.spikeinterfacerecordingdatachunkiterator import ( SpikeInterfaceRecordingDataChunkIterator, ) from spikeinterface import extractors @@ -27,6 +28,8 @@ class DecimalEncoder(json.JSONEncoder): + """Extension of json.JSONEncoder class""" + def default(self, o): if isinstance(o, decimal.Decimal): return str(o) @@ -34,19 +37,18 @@ def default(self, o): class LFPDataChunkIterator(GenericDataChunkIterator): - """ - DataChunkIterator for LFP data that pulls data one channel at a time. Used when - reading LFP data from the database (as opposed to directly from source files) + """DataChunkIterator for LFP data that pulls data one channel at a time. + + Used when reading LFP data from the database (as opposed to directly from source + files). """ - def __init__(self, lfp_electrodes_query, chunk_length: int = 10000): + def __init__(self, lfp_electrodes_query: Table, chunk_length: int = 10000): """ - Parameters - ---------- - lfp_electrodes_query: element_array_ephys.ephys.LFP.Electrode - chunk_length: int, optional - Chunks are blocks of disk space where data are stored contiguously - and compressed + Arguments: + lfp_electrodes_query (datajoint table): element_array_ephys.ephys.LFP.Electrode + chunk_length (int): Optional. Default 10,000. Chunks are blocks of disk + space where data are stored contiguously and compressed. """ self.lfp_electrodes_query = lfp_electrodes_query self.electrodes = self.lfp_electrodes_query.fetch("electrode") @@ -62,7 +64,6 @@ def __init__(self, lfp_electrodes_query, chunk_length: int = 10000): super().__init__(buffer_shape=(self.n_tt, 1), chunk_shape=(chunk_length, 1)) def _get_data(self, selection): - electrode = self.electrodes[selection[1]][0] return (self.lfp_electrodes_query & dict(electrode=electrode)).fetch1("lfp")[ selection[0], np.newaxis @@ -76,27 +77,27 @@ def _get_maxshape(self): def add_electrodes_to_nwb(session_key: dict, nwbfile: pynwb.NWBFile): - """ - Add electrodes table to NWBFile. This is needed for any ElectricalSeries, including - raw source data and LFP. - - ephys.InsertionLocation -> ElectrodeGroup.location - - probe.Probe::probe -> device.name - probe.Probe::probe_comment -> device.description - probe.Probe::probe_type -> device.manufacturer - - probe.ProbeType.Electrode::electrode -> electrodes["id_in_probe"] - probe.ProbeType.Electrode::y_coord -> electrodes["rel_y"] - probe.ProbeType.Electrode::x_coord -> electrodes["rel_x"] - probe.ProbeType.Electrode::shank -> electrodes["shank"] - probe.ProbeType.Electrode::shank_col -> electrodes["shank_col"] - probe.ProbeType.Electrode::shank_row -> electrodes["shank_row"] - - Parameters - ---------- - session_key: dict - nwbfile: pynwb.NWBFile + """Add electrodes table to NWBFile. + + This is needed for any ElectricalSeries, including raw source data and LFP. + + Mapping: + ephys.InsertionLocation -> ElectrodeGroup.location + + probe.Probe::probe -> device.name + probe.Probe::probe_comment -> device.description + probe.Probe::probe_type -> device.manufacturer + + probe.ProbeType.Electrode::electrode -> electrodes["id_in_probe"] + probe.ProbeType.Electrode::y_coord -> electrodes["rel_y"] + probe.ProbeType.Electrode::x_coord -> electrodes["rel_x"] + probe.ProbeType.Electrode::shank -> electrodes["shank"] + probe.ProbeType.Electrode::shank_col -> electrodes["shank_col"] + probe.ProbeType.Electrode::shank_row -> electrodes["shank_row"] + + Arguments: + session_key (dict): key from Session table + nwbfile (pynwb.NWBFile): nwb file """ electrodes_query = probe.ProbeType.Electrode * probe.ElectrodeConfig.Electrode @@ -177,23 +178,20 @@ def create_units_table( desc="data on spiking units", ): """ - - ephys.CuratedClustering.Unit::unit -> units.id - ephys.CuratedClustering.Unit::spike_times -> units["spike_times"] - ephys.CuratedClustering.Unit::spike_depths -> units["spike_depths"] - ephys.CuratedClustering.Unit::cluster_quality_label -> units["cluster_quality_label"] - - ephys.WaveformSet.PeakWaveform::peak_electrode_waveform -> units["waveform_mean"] - - Parameters - ---------- - session_key: dict - nwbfile: pynwb.NWBFile - paramset_record: int - name: str, optional - default="units" - desc: str, optional - default="data on spiking units" + Mapping: + ephys.CuratedClustering.Unit::unit -> units.id + ephys.CuratedClustering.Unit::spike_times -> units["spike_times"] + ephys.CuratedClustering.Unit::spike_depths -> units["spike_depths"] + ephys.CuratedClustering.Unit::cluster_quality_label -> units["cluster_quality_label"] + + ephys.WaveformSet.PeakWaveform::peak_electrode_waveform -> units["waveform_mean"] + + Arguments: + session_key (dict): key from Session table + nwbfile (pynwb.NWBFile): nwb file + paramset_record (int): paramset id from ephys schema + name (str): Optional table name. Default "units" + desc (str): Optional table description. Default "data on spiking units" """ # electrode id mapping @@ -255,8 +253,7 @@ def create_units_table( def add_ephys_units_to_nwb( session_key: dict, nwbfile: pynwb.NWBFile, primary_clustering_paramset_idx: int = 0 ): - """ - Add spiking data to NWBFile. + """Add spiking data to NWBFile. In NWB, spiking data is stored in a Units table. The primary Units table is stored at /units. The spiking data in /units is generally the data used in @@ -269,18 +266,18 @@ def add_ephys_units_to_nwb( Use `primary_clustering_paramset_idx` to indicate which clustering is primary. All others will be stored in /processing/ecephys/. - ephys.CuratedClustering.Unit::unit -> units.id - ephys.CuratedClustering.Unit::spike_times -> units["spike_times"] - ephys.CuratedClustering.Unit::spike_depths -> units["spike_depths"] - ephys.CuratedClustering.Unit::cluster_quality_label -> units["cluster_quality_label"] + Mapping: + ephys.CuratedClustering.Unit::unit -> units.id + ephys.CuratedClustering.Unit::spike_times -> units["spike_times"] + ephys.CuratedClustering.Unit::spike_depths -> units["spike_depths"] + ephys.CuratedClustering.Unit::cluster_quality_label -> units["cluster_quality_label"] - ephys.WaveformSet.PeakWaveform::peak_electrode_waveform -> units["waveform_mean"] + ephys.WaveformSet.PeakWaveform::peak_electrode_waveform -> units["waveform_mean"] - Parameters - ---------- - session_key: dict - nwbfile: pynwb.NWBFile - primary_clustering_paramset_idx: int, optional + Arguments: + session_key (dict): key from Session table + nwbfile (pynwb.NWBFile): nwb file + primary_clustering_paramset_idx (int): Optional. Default 0 """ if not ephys.ClusteringTask & session_key: @@ -314,19 +311,18 @@ def add_ephys_units_to_nwb( ecephys_module.add(units_table) -def get_electrodes_mapping(electrodes): - """ - Create a mapping from the probe and electrode id to the row number of the electrodes - table. This is used in the construction of the DynamicTableRegion that indicates - what rows of the electrodes table correspond to the data in an ElectricalSeries. +def get_electrodes_mapping(electrodes) -> dict: + """Create mapping from probe and electrode id to row number in the electrodes table + + This is used in the construction of the DynamicTableRegion that indicates what rows + of the electrodes table correspond to the data in an ElectricalSeries. - Parameters - ---------- - electrodes: hdmf.common.table.DynamicTable + Arguments: + electrodes ( hdmf.common.table.DynamicTable ): hdmf Dynamic Table - Returns - ------- - dict + Returns: + dict using tuple (electrodes device name, probe id index) as key and index + electrode index as value """ return { @@ -338,26 +334,23 @@ def get_electrodes_mapping(electrodes): } -def gains_helper(gains): - """ - This handles three different cases for gains: - 1. gains are all 1. In this case, return conversion=1e-6, which applies to all - channels and converts from microvolts to volts. - 2. Gains are all equal, but not 1. In this case, multiply this by 1e-6 to apply this - gain to all channels and convert units to volts. - 3. Gains are different for different channels. In this case use the - `channel_conversion` field in addition to the `conversion` field so that each - channel can be converted to volts using its own individual gain. - - Parameters - ---------- - gains: np.ndarray - - Returns - ------- - dict - conversion : float - channel_conversion : np.ndarray +def gains_helper(gains) -> dict: + """This handles three different cases for gains. See below + + Cases: + 1. gains are all 1. In this case, return conversion=1e-6, which applies to all + channels and converts from microvolts to volts. + 2. Gains are all equal, but not 1. In this case, multiply this by 1e-6 to apply + this gain to all channels and convert units to volts. + 3. Gains are different for different channels. In this case use the + `channel_conversion` field in addition to the `conversion` field so that + each channel can be converted to volts using its own individual gain. + + Arguments: + gains ( np.ndarray ): array of gains + + Returns: + dict with conversion float as key and channel_conversion np.ndarray value """ if all(x == 1 for x in gains): @@ -373,19 +366,20 @@ def add_ephys_recording_to_nwb( nwbfile: pynwb.NWBFile, end_frame: int = None, ): - """ - Read voltage data directly from source files and iteratively transfer them to the - NWB file. Automatically applies lossless compression to the data, so the final file - might be smaller than the original, without data loss. Currently supports - Neuropixels data acquired with SpikeGLX or Open Ephys, and relies on SpikeInterface - to read the data. - - source data -> acquisition["ElectricalSeries"] - - Parameters - ---------- - session_key: dict ephys_root_data_dir: str nwbfile: NWBFile end_frame: int, optional - Used for small test conversions + """Read voltage data from source files and iteratively transfer to the NWB file. + + Automatically applies lossless compression to the data, so the final file might be + smaller than the original, without data loss. Currently supports Neuropixels data + acquired with SpikeGLX or Open Ephys, and relies on SpikeInterface to read the data. + + Mapping: + source data -> acquisition["ElectricalSeries"] + + Arguments: + session_key (dict): key from Session table + ephys_root_data_dir (str): root data directory + nwbfile (NWBFile): nwb file + end_frame (int): Optional limit on frames for small test conversions """ if nwbfile.electrodes is None: @@ -443,20 +437,21 @@ def add_ephys_recording_to_nwb( def add_ephys_lfp_from_dj_to_nwb(session_key: dict, nwbfile: pynwb.NWBFile): - """ - Read LFP data from the data in element-aray-ephys - - ephys.LFP.Electrode::lfp -> - processing["ecephys"].lfp.electrical_series["ElectricalSeries{insertion_number}" - ].data - ephys.LFP::lfp_time_stamps -> - processing["ecephys"].lfp.electrical_series["ElectricalSeries{insertion_number}" - ].timestamps - - Parameters - ---------- - session_key: dict - nwbfile: NWBFile + """Read LFP data from the data in element-aray-ephys + + Mapping: + ephys.LFP.Electrode::lfp -> + processing["ecephys"].lfp.electrical_series[ + "ElectricalSeries{insertion_number}" + ].data + ephys.LFP::lfp_time_stamps -> + processing["ecephys"].lfp.electrical_series[ + "ElectricalSeries{insertion_number}" + ].timestamps + + Arguments: + session_key (dict): key from Session table + nwbfile (NWBFile): nwb file """ if nwbfile.electrodes is None: @@ -501,13 +496,10 @@ def add_ephys_lfp_from_source_to_nwb( ephys.EphysRecording::recording_datetime -> acquisition - Parameters - ---------- - session_key: dict - nwbfile: pynwb.NWBFile - end_frame: int, optional - use for small test conversions - + Arguments: + session_key (dict): key from Session table + nwbfile (NWBFile): nwb file + end_frame (int): Optional limit on frames for small test conversions """ if nwbfile.electrodes is None: add_electrodes_to_nwb(session_key, nwbfile) @@ -580,30 +572,27 @@ def ecephys_session_to_nwb( protocol_key=None, nwbfile_kwargs=None, ): - """ - Main function for converting ephys data to NWB - - Parameters - ---------- - session_key: dict - raw: bool - Whether to include the raw data from source. SpikeGLX & OpenEphys are supported - spikes: bool - Whether to include CuratedClustering - lfp: - "dj" - read LFP data from ephys.LFP - "source" - read LFP data from source (SpikeGLX supported) - False - do not convert LFP - end_frame: int, optional - Used to create small test conversions where large datasets are truncated. - lab_key, project_key, and protocol_key: dictionaries to look up optional metadata - nwbfile_kwargs: dict, optional - - If element-session is not being used, this argument is required and must be a - dictionary containing 'session_description' (str), 'identifier' (str), and - 'session_start_time' (datetime), the required minimal data for instantiating - an NWBFile object. - - If element-session is being used, this argument can optionally be used to - overwrite NWBFile fields. + """Main function for converting ephys data to NWB + + Arguments: + session_key (dict): key from Session table + raw (bool): Optional. Default True. Include the raw data from source. + SpikeGLX & OpenEphys are supported + spikes (bool): Optional. Default True. Whether to include CuratedClustering + lfp (str): One of the following. + "dj", read LFP data from ephys.LFP. + "source", read LFP data from source (SpikeGLX supported). + False, do not convert LFP. + end_frame (int): Optional limit on frames for small test conversions. + lab_key (dict): Optional key to add metadata from other Element Lab. + project_key (dict): Optional key to add metadata from other Element Lab. + protocol_key (dict): Optional key to add metadata from other Element Lab. + nwbfile_kwargs (dict): Optional. If Element Session is not used, this argument + is required and must be a dictionary containing 'session_description' (str), + 'identifier' (str), and 'session_start_time' (datetime), the required + minimal data for instantiating an NWBFile object. If element-session is + being used, this argument can optionally be used to overwrite NWBFile + fields. """ session_to_nwb = getattr(ephys._linking_module, "session_to_nwb", False) @@ -647,16 +636,13 @@ def ecephys_session_to_nwb( def write_nwb(nwbfile, fname, check_read=True): - """ - Export NWBFile - - Parameters - ---------- - nwbfile: pynwb.NWBFile - fname: str Absolute path including `*.nwb` extension. - check_read: bool - If True, PyNWB will try to read the produced NWB file and ensure that it can be - read. + """Export NWBFile + + Arguments: + nwbfile (NWBFile): nwb file + fname (str): Absolute path including `*.nwb` extension. + check_read (bool): If True, PyNWB will try to read the produced NWB file and + ensure that it can be read. """ with pynwb.NWBHDF5IO(fname, "w") as io: io.write(nwbfile) From 56eb68a96f1552aaeed967d4c46ccad45c8eabcd Mon Sep 17 00:00:00 2001 From: CBroz1 Date: Thu, 12 Jan 2023 17:39:15 -0600 Subject: [PATCH 10/19] Spelling --- docs/mkdocs.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/mkdocs.yaml b/docs/mkdocs.yaml index 40c98b10..3607eee5 100644 --- a/docs/mkdocs.yaml +++ b/docs/mkdocs.yaml @@ -26,12 +26,12 @@ nav: # --------------------- NOTES TO CONTRIBUTORS ----------------------- # Markdown in mkdocs -# 01. Redering concatenates across single line breaks. This means... +# 01. Rendering concatenates across single line breaks. This means... # A. We have to be careful to add extra line breaks around paragraphs, -# including between the end of a pgf and the beginnign of bullets. +# including between the end of a pgf and the beginning of bullets. # B. We can use hard wrapping to make github reviews easier to read. # VSCode Rewrap extension offers a keyboard shortcut for hard wrap -# at the ruler, but don't add breaks in [multiword links](example.com) +# at the ruler, but don't add breaks in [multi-word links](example.com) # 02. Instead of designating codeblocks with bash, use console. For example.. # ```console # cd ../my_dir @@ -57,7 +57,7 @@ nav: # 02. Site analytics depend on a local environment variable GOOGLE_ANALYTICS_KEY # You can find this in LastPass or declare with any string to suppress errors # 03. The API section will pull docstrings. -# A. Follow google styleguide e.g., +# A. Follow google style guide e.g., # https://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html # With typing suggestions: https://docs.python.org/3/library/typing.html # B. To pull a specific workflow fork, change ./docs/src/api/make_pages.py#L19 From e01683ca2241e457ccfa6f4d61914808400d663e Mon Sep 17 00:00:00 2001 From: CBroz1 Date: Fri, 13 Jan 2023 10:23:36 -0600 Subject: [PATCH 11/19] Add extras_require for dev and nwb --- element_array_ephys/export/nwb/requirements.txt | 3 ++- requirements.txt | 1 - setup.py | 7 +++++++ 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/element_array_ephys/export/nwb/requirements.txt b/element_array_ephys/export/nwb/requirements.txt index 995a4556..0c236436 100644 --- a/element_array_ephys/export/nwb/requirements.txt +++ b/element_array_ephys/export/nwb/requirements.txt @@ -1,4 +1,5 @@ dandi neo==0.10.2 -nwb-conversion-tools==0.11.1 +neuroconv +pynwb>=2.0.0 spikeinterface==0.93.0 diff --git a/requirements.txt b/requirements.txt index 99b40458..6d67407b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,6 +2,5 @@ datajoint>=0.13 element-interface @ git+https://github.com/datajoint/element-interface.git openpyxl plotly -pynwb>=2.0.0 pyopenephys @ git+https://github.com/datajoint-company/pyopenephys.git seaborn diff --git a/setup.py b/setup.py index e819517c..31b9be61 100644 --- a/setup.py +++ b/setup.py @@ -11,6 +11,12 @@ with open(path.join(here, "requirements.txt")) as f: requirements = f.read().splitlines() +with open(path.join(here, "requirements_dev.txt")) as f: + requirements_dev = f.read().splitlines() + +with open(path.join(here, "element_array_ephys/export/nwb/requirements.txt")) as f: + requirements_nwb = f.read().splitlines() + with open(path.join(here, pkg_name, "version.py")) as f: exec(f.read()) @@ -26,6 +32,7 @@ url=f'https://github.com/datajoint/{pkg_name.replace("_", "-")}', keywords="neuroscience electrophysiology science datajoint", packages=find_packages(exclude=["contrib", "docs", "tests*"]), + extras_require={"dev": requirements_dev, "nwb": requirements_nwb}, scripts=[], install_requires=requirements, ) From 9d0335047fe67e02ebf5ccc6da395d5bdbeda3df Mon Sep 17 00:00:00 2001 From: CBroz1 Date: Fri, 13 Jan 2023 10:30:40 -0600 Subject: [PATCH 12/19] Version bump, changelog --- CHANGELOG.md | 9 +++++++++ element_array_ephys/version.py | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c2ad3716..a6167bd5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,14 @@ Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) convention. +## [0.2.3] - Unreleased + ++ Add - extras_require install options for nwb and development requirement sets ++ Add - mkdocs notebook rendering ++ Add - markdown linting and spellcheck config files, with implementation edits ++ Update - license for 2023 ++ Update - blackify previous updates + ## [0.2.2] - 2022-01-11 + Bugfix - Revert import order in `__init__.py` to avoid circular import error. @@ -78,6 +86,7 @@ Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and + Add - Probe table supporting: Neuropixels probes 1.0 - 3A, 1.0 - 3B, 2.0 - SS, 2.0 - MS +[0.2.3]: https://github.com/datajoint/element-array-ephys/releases/tag/0.2.3 [0.2.2]: https://github.com/datajoint/element-array-ephys/releases/tag/0.2.2 [0.2.1]: https://github.com/datajoint/element-array-ephys/releases/tag/0.2.1 [0.2.0]: https://github.com/datajoint/element-array-ephys/releases/tag/0.2.0 diff --git a/element_array_ephys/version.py b/element_array_ephys/version.py index e427d333..fd93127d 100644 --- a/element_array_ephys/version.py +++ b/element_array_ephys/version.py @@ -1,2 +1,2 @@ """Package metadata.""" -__version__ = "0.2.2" +__version__ = "0.2.3" From 80458796ebda66e81312211677a0f4aa93100295 Mon Sep 17 00:00:00 2001 From: CBroz1 Date: Fri, 13 Jan 2023 11:25:06 -0600 Subject: [PATCH 13/19] add extras_require nwb install option to docs --- docs/src/concepts.md | 12 +++++++++--- element_array_ephys/export/nwb/README.md | 6 ++++++ 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/docs/src/concepts.md b/docs/src/concepts.md index cee2df32..06c57944 100644 --- a/docs/src/concepts.md +++ b/docs/src/concepts.md @@ -189,9 +189,15 @@ Without Borders (NWB) files. This makes it easy to share files with collaborator publish results on [DANDI Archive](https://dandiarchive.org/). [NWB](https://www.nwb.org/), as an organization, is dedicated to standardizing data formats and maximizing interoperability across tools for neurophysiology. For more -information on uploading NWB files to DANDI within the DataJoint Elements ecosystem, -visit our documentation for the DANDI upload feature of [Element Array -Electrophysiology](datajoint.com/docs/elements/element-array-ephys/). +information on uploading NWB files to DANDI within the DataJoint Elements ecosystem see +the corresponding notebook on the [tutorials page](./tutorials/index.md). + +To use the export functionality with additional related dependencies, install the +Element with the `nwb` option as follows: + +```console +pip install element-array-ephys[nwb] +``` ## Roadmap diff --git a/element_array_ephys/export/nwb/README.md b/element_array_ephys/export/nwb/README.md index b5985498..b084872d 100644 --- a/element_array_ephys/export/nwb/README.md +++ b/element_array_ephys/export/nwb/README.md @@ -1,5 +1,11 @@ # Exporting data to NWB +To use the export functionality, install the Element with the `nwb` option as follows: + +```console +pip install element-array-ephys[nwb] +``` + The `export/nwb/nwb.py` module maps from the element-array-ephys data structure to NWB. The main function is `ecephys_session_to_nwb`, which contains flags to control calling the following functions, which can be called independently: From 9ff2e04cb879fe8aae0fe985a661f7fe8761f79b Mon Sep 17 00:00:00 2001 From: CBroz1 Date: Fri, 13 Jan 2023 11:56:49 -0600 Subject: [PATCH 14/19] interface requirement to pip installable --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 6d67407b..187df3b7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ datajoint>=0.13 -element-interface @ git+https://github.com/datajoint/element-interface.git +element-interface>=0.2.0 openpyxl plotly pyopenephys @ git+https://github.com/datajoint-company/pyopenephys.git From 888e7f743fcfc10dc190c3020523aeb1547c8380 Mon Sep 17 00:00:00 2001 From: CBroz1 Date: Tue, 17 Jan 2023 11:25:54 -0600 Subject: [PATCH 15/19] Fix typing bug --- element_array_ephys/probe.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/element_array_ephys/probe.py b/element_array_ephys/probe.py index b26812fe..d9cd4db3 100644 --- a/element_array_ephys/probe.py +++ b/element_array_ephys/probe.py @@ -1,6 +1,7 @@ """ Neuropixels Probes """ +from typing import List import datajoint as dj import numpy as np @@ -216,7 +217,7 @@ def build_electrode_layouts( shank_count: int = 1, shank_spacing: float = None, y_origin="bottom", -) -> list[dict]: +) -> List[dict]: """Builds electrode layouts. From 45f846cf75004b9e5ea6e6580f6f16209515c6f0 Mon Sep 17 00:00:00 2001 From: CBroz1 Date: Wed, 18 Jan 2023 11:13:44 -0600 Subject: [PATCH 16/19] Adjust dependencies --- cspell.json | 5 +++++ element_array_ephys/export/nwb/nwb.py | 10 +++++++--- element_array_ephys/export/nwb/requirements.txt | 2 +- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/cspell.json b/cspell.json index 860b72d0..524ca0da 100644 --- a/cspell.json +++ b/cspell.json @@ -55,6 +55,7 @@ "Ephys", "gblcar", "gfix", + "hdmf", "HHMI", "hstack", "ibllib", @@ -88,10 +89,13 @@ "ndimage", "Neuralynx", "NEURO", + "neuroconv", "Neurodata", "Neuropix", "neuropixel", "NeuroPixels", + "nwbfile", + "NWBHDF", "oebin", "openephys", "openephys", @@ -125,6 +129,7 @@ "tickvals", "tofile", "Tolias", + "tqdm", "vline", "Vmax", "vmax", diff --git a/element_array_ephys/export/nwb/nwb.py b/element_array_ephys/export/nwb/nwb.py index 919a1ebb..d498d468 100644 --- a/element_array_ephys/export/nwb/nwb.py +++ b/element_array_ephys/export/nwb/nwb.py @@ -399,7 +399,9 @@ def add_ephys_recording_to_nwb( file_path = find_full_path(ephys_root_data_dir, relative_path) if ephys_recording_record["acq_software"] == "SpikeGLX": - extractor = extractors.read_spikeglx(os.path.split(file_path)[0], "imec.ap") + extractor = extractors.read_spikeglx( + os.path.split(file_path)[0], stream_id="imec.ap" + ) elif ephys_recording_record["acq_software"] == "OpenEphys": extractor = extractors.read_openephys(file_path, stream_id="0") else: @@ -437,7 +439,7 @@ def add_ephys_recording_to_nwb( def add_ephys_lfp_from_dj_to_nwb(session_key: dict, nwbfile: pynwb.NWBFile): - """Read LFP data from the data in element-aray-ephys + """Read LFP data from the data in element-array-ephys Mapping: ephys.LFP.Electrode::lfp -> @@ -525,7 +527,9 @@ def add_ephys_lfp_from_source_to_nwb( file_path = find_full_path(ephys_root_data_dir, relative_path) if ephys_recording_record["acq_software"] == "SpikeGLX": - extractor = extractors.read_spikeglx(os.path.split(file_path)[0], "imec.lf") + extractor = extractors.read_spikeglx( + os.path.split(file_path)[0], stream_id="imec.lf" + ) else: raise ValueError( "unsupported acq_software type:" diff --git a/element_array_ephys/export/nwb/requirements.txt b/element_array_ephys/export/nwb/requirements.txt index 0c236436..5611474e 100644 --- a/element_array_ephys/export/nwb/requirements.txt +++ b/element_array_ephys/export/nwb/requirements.txt @@ -1,5 +1,5 @@ dandi neo==0.10.2 -neuroconv +neuroconv[ecephys]>=0.2.0 pynwb>=2.0.0 spikeinterface==0.93.0 From a28cf13118f15fbec515171852027965b8c433ad Mon Sep 17 00:00:00 2001 From: CBroz1 Date: Wed, 18 Jan 2023 12:17:13 -0600 Subject: [PATCH 17/19] Adjust dependencies 2 --- element_array_ephys/export/nwb/requirements.txt | 2 -- requirements.txt | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/element_array_ephys/export/nwb/requirements.txt b/element_array_ephys/export/nwb/requirements.txt index 5611474e..251b16ec 100644 --- a/element_array_ephys/export/nwb/requirements.txt +++ b/element_array_ephys/export/nwb/requirements.txt @@ -1,5 +1,3 @@ dandi -neo==0.10.2 neuroconv[ecephys]>=0.2.0 pynwb>=2.0.0 -spikeinterface==0.93.0 diff --git a/requirements.txt b/requirements.txt index 187df3b7..fca73db6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ datajoint>=0.13 -element-interface>=0.2.0 +element-interface>=0.4.0 openpyxl plotly pyopenephys @ git+https://github.com/datajoint-company/pyopenephys.git From 4d9ab28609b161be986cda2872778d7a403277f3 Mon Sep 17 00:00:00 2001 From: JaerongA Date: Wed, 18 Jan 2023 14:55:21 -0600 Subject: [PATCH 18/19] fix: :bug: import from __future__ module this supports backward compability with typing --- element_array_ephys/ephys_report.py | 2 ++ element_array_ephys/plotting/unit_level.py | 2 ++ element_array_ephys/probe.py | 1 + 3 files changed, 5 insertions(+) diff --git a/element_array_ephys/ephys_report.py b/element_array_ephys/ephys_report.py index 986e3d40..5b5aee2d 100644 --- a/element_array_ephys/ephys_report.py +++ b/element_array_ephys/ephys_report.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import datetime import pathlib from uuid import UUID diff --git a/element_array_ephys/plotting/unit_level.py b/element_array_ephys/plotting/unit_level.py index bda2b74b..a19b0fbe 100644 --- a/element_array_ephys/plotting/unit_level.py +++ b/element_array_ephys/plotting/unit_level.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from modulefinder import Module from typing import Any diff --git a/element_array_ephys/probe.py b/element_array_ephys/probe.py index b26812fe..417fa3bc 100644 --- a/element_array_ephys/probe.py +++ b/element_array_ephys/probe.py @@ -1,6 +1,7 @@ """ Neuropixels Probes """ +from __future__ import annotations import datajoint as dj import numpy as np From 4796056c1bdb98039cc52e43a4491dc43f6bcfef Mon Sep 17 00:00:00 2001 From: Chris Brozdowski Date: Wed, 18 Jan 2023 16:54:54 -0600 Subject: [PATCH 19/19] Apply suggestions from code review Co-authored-by: JaerongA --- element_array_ephys/probe.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/element_array_ephys/probe.py b/element_array_ephys/probe.py index 3c147d3f..417fa3bc 100644 --- a/element_array_ephys/probe.py +++ b/element_array_ephys/probe.py @@ -2,7 +2,6 @@ Neuropixels Probes """ from __future__ import annotations -from typing import List import datajoint as dj import numpy as np @@ -218,7 +217,7 @@ def build_electrode_layouts( shank_count: int = 1, shank_spacing: float = None, y_origin="bottom", -) -> List[dict]: +) -> list[dict]: """Builds electrode layouts.