From f9f18d0864615f24420b3feb982b68dfd85dd6d7 Mon Sep 17 00:00:00 2001 From: Thinh Nguyen Date: Tue, 30 Aug 2022 16:15:11 -0500 Subject: [PATCH 1/8] Update kilosort_triggering.py --- element_array_ephys/readers/kilosort_triggering.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/element_array_ephys/readers/kilosort_triggering.py b/element_array_ephys/readers/kilosort_triggering.py index ccb21312..46625387 100644 --- a/element_array_ephys/readers/kilosort_triggering.py +++ b/element_array_ephys/readers/kilosort_triggering.py @@ -303,7 +303,9 @@ class OpenEphysKilosortPipeline: https://github.com/AllenInstitute/ecephys_spike_sorting """ - _modules = ['kilosort_helper', + _modules = ['depth_estimation', + 'median_subtraction', + 'kilosort_helper', 'kilosort_postprocessing', 'noise_templates', 'mean_waveforms', @@ -477,7 +479,7 @@ def _update_total_duration(self): def run_pykilosort(continuous_file, kilosort_output_directory, params, - channel_ind, x_coords, y_coords, shank_ind, connected, sample_rate): + channel_ind, x_coords, y_coords, shank_ind, connected, sample_rate): dat_path = pathlib.Path(continuous_file) probe = pykilosort.Bunch() From 59053923a9968df6f2ab7f90d9324c7502dcaf0e Mon Sep 17 00:00:00 2001 From: Thinh Nguyen Date: Fri, 2 Sep 2022 15:02:56 -0500 Subject: [PATCH 2/8] bugfix in running catgt --- .../readers/kilosort_triggering.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/element_array_ephys/readers/kilosort_triggering.py b/element_array_ephys/readers/kilosort_triggering.py index 46625387..f1f2cbf8 100644 --- a/element_array_ephys/readers/kilosort_triggering.py +++ b/element_array_ephys/readers/kilosort_triggering.py @@ -90,7 +90,7 @@ def generate_CatGT_input_json(self): print('run_CatGT is set to False, skipping...') return - session_str, gate_str, _, probe_str = self.parse_input_filename() + session_str, gate_str, trig_str, probe_str = self.parse_input_filename() first_trig, last_trig = SpikeGLX_utils.ParseTrigStr( 'start,end', probe_str, gate_str, self._npx_input_dir.as_posix()) @@ -111,6 +111,18 @@ def generate_CatGT_input_json(self): input_meta_fullpath, continuous_file = self._get_raw_data_filepaths() + # create symbolic link to the actual data files - as CatGT expects files to follow a certain naming convention + continuous_file_symlink = (continuous_file.parent / f'{session_str}_g{gate_str}' + / f'{session_str}_g{gate_str}_imec{probe_str}' + / f'{session_str}_g{gate_str}_t{trig_str}.imec{probe_str}.ap.bin') + continuous_file_symlink.parent.mkdir(parents=True, exist_ok=True) + continuous_file_symlink.symlink_to(continuous_file) + input_meta_fullpath_symlink = (input_meta_fullpath.parent / f'{session_str}_g{gate_str}' + / f'{session_str}_g{gate_str}_imec{probe_str}' + / f'{session_str}_g{gate_str}_t{trig_str}.imec{probe_str}.ap.meta') + input_meta_fullpath_symlink.parent.mkdir(parents=True, exist_ok=True) + input_meta_fullpath_symlink.symlink_to(input_meta_fullpath) + createInputJson(self._catGT_input_json.as_posix(), KS2ver=self._KS2ver, npx_directory=self._npx_input_dir.as_posix(), @@ -121,7 +133,7 @@ def generate_CatGT_input_json(self): probe_string=probe_str, continuous_file=continuous_file.as_posix(), input_meta_path=input_meta_fullpath.as_posix(), - extracted_data_directory=self._ks_output_dir.parent.as_posix(), + extracted_data_directory=self._ks_output_dir.as_posix(), kilosort_output_directory=self._ks_output_dir.as_posix(), kilosort_repository=_get_kilosort_repository(self._KS2ver), **{k: v for k, v in catgt_params.items() if k in self._input_json_args} From 987231be7d386351a043901d19f55f76b6bbf90d Mon Sep 17 00:00:00 2001 From: Thinh Nguyen Date: Fri, 2 Sep 2022 15:03:07 -0500 Subject: [PATCH 3/8] enable catgt run --- element_array_ephys/ephys_acute.py | 2 +- element_array_ephys/ephys_chronic.py | 2 +- element_array_ephys/ephys_no_curation.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/element_array_ephys/ephys_acute.py b/element_array_ephys/ephys_acute.py index 568d6323..df7ebd73 100644 --- a/element_array_ephys/ephys_acute.py +++ b/element_array_ephys/ephys_acute.py @@ -629,7 +629,7 @@ def make(self, key): ks_output_dir=kilosort_dir, params=params, KS2ver=f'{Decimal(clustering_method.replace("kilosort", "")):.1f}', - run_CatGT=False) + run_CatGT=True) run_kilosort.run_modules() elif acq_software == 'Open Ephys': oe_probe = get_openephys_probe_data(key) diff --git a/element_array_ephys/ephys_chronic.py b/element_array_ephys/ephys_chronic.py index cd599f89..2f6948e6 100644 --- a/element_array_ephys/ephys_chronic.py +++ b/element_array_ephys/ephys_chronic.py @@ -576,7 +576,7 @@ def make(self, key): ks_output_dir=kilosort_dir, params=params, KS2ver=f'{Decimal(clustering_method.replace("kilosort", "")):.1f}', - run_CatGT=False) + run_CatGT=True) run_kilosort.run_modules() elif acq_software == 'Open Ephys': oe_probe = get_openephys_probe_data(key) diff --git a/element_array_ephys/ephys_no_curation.py b/element_array_ephys/ephys_no_curation.py index 1e4fdd5c..f0988917 100644 --- a/element_array_ephys/ephys_no_curation.py +++ b/element_array_ephys/ephys_no_curation.py @@ -628,7 +628,7 @@ def make(self, key): ks_output_dir=kilosort_dir, params=params, KS2ver=f'{Decimal(clustering_method.replace("kilosort", "")):.1f}', - run_CatGT=False) + run_CatGT=True) run_kilosort.run_modules() elif acq_software == 'Open Ephys': oe_probe = get_openephys_probe_data(key) From aed42ca1dddd15a6b2a03c329f9278122846c55b Mon Sep 17 00:00:00 2001 From: Thinh Nguyen Date: Fri, 2 Sep 2022 15:07:11 -0500 Subject: [PATCH 4/8] bugfix --- element_array_ephys/readers/kilosort_triggering.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/element_array_ephys/readers/kilosort_triggering.py b/element_array_ephys/readers/kilosort_triggering.py index f1f2cbf8..d38e9a70 100644 --- a/element_array_ephys/readers/kilosort_triggering.py +++ b/element_array_ephys/readers/kilosort_triggering.py @@ -116,11 +116,13 @@ def generate_CatGT_input_json(self): / f'{session_str}_g{gate_str}_imec{probe_str}' / f'{session_str}_g{gate_str}_t{trig_str}.imec{probe_str}.ap.bin') continuous_file_symlink.parent.mkdir(parents=True, exist_ok=True) + continuous_file_symlink.unlink(missing_ok=False) continuous_file_symlink.symlink_to(continuous_file) input_meta_fullpath_symlink = (input_meta_fullpath.parent / f'{session_str}_g{gate_str}' / f'{session_str}_g{gate_str}_imec{probe_str}' / f'{session_str}_g{gate_str}_t{trig_str}.imec{probe_str}.ap.meta') input_meta_fullpath_symlink.parent.mkdir(parents=True, exist_ok=True) + input_meta_fullpath_symlink.unlink(missing_ok=False) input_meta_fullpath_symlink.symlink_to(input_meta_fullpath) createInputJson(self._catGT_input_json.as_posix(), From fbdbe24adc55c6fb51f4e524760528f27589f37b Mon Sep 17 00:00:00 2001 From: Thinh Nguyen Date: Fri, 2 Sep 2022 15:08:54 -0500 Subject: [PATCH 5/8] bugfix --- element_array_ephys/readers/kilosort_triggering.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/element_array_ephys/readers/kilosort_triggering.py b/element_array_ephys/readers/kilosort_triggering.py index d38e9a70..8c063825 100644 --- a/element_array_ephys/readers/kilosort_triggering.py +++ b/element_array_ephys/readers/kilosort_triggering.py @@ -116,13 +116,13 @@ def generate_CatGT_input_json(self): / f'{session_str}_g{gate_str}_imec{probe_str}' / f'{session_str}_g{gate_str}_t{trig_str}.imec{probe_str}.ap.bin') continuous_file_symlink.parent.mkdir(parents=True, exist_ok=True) - continuous_file_symlink.unlink(missing_ok=False) + continuous_file_symlink.unlink(missing_ok=True) continuous_file_symlink.symlink_to(continuous_file) input_meta_fullpath_symlink = (input_meta_fullpath.parent / f'{session_str}_g{gate_str}' / f'{session_str}_g{gate_str}_imec{probe_str}' / f'{session_str}_g{gate_str}_t{trig_str}.imec{probe_str}.ap.meta') input_meta_fullpath_symlink.parent.mkdir(parents=True, exist_ok=True) - input_meta_fullpath_symlink.unlink(missing_ok=False) + input_meta_fullpath_symlink.unlink(missing_ok=True) input_meta_fullpath_symlink.symlink_to(input_meta_fullpath) createInputJson(self._catGT_input_json.as_posix(), From 35da39bb1c5c80c18ade40fed296cfb47b158123 Mon Sep 17 00:00:00 2001 From: Thinh Nguyen Date: Fri, 2 Sep 2022 15:14:03 -0500 Subject: [PATCH 6/8] `missing_ok` arg only available in python 3.8+ --- element_array_ephys/readers/kilosort_triggering.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/element_array_ephys/readers/kilosort_triggering.py b/element_array_ephys/readers/kilosort_triggering.py index 8c063825..a1797721 100644 --- a/element_array_ephys/readers/kilosort_triggering.py +++ b/element_array_ephys/readers/kilosort_triggering.py @@ -116,13 +116,15 @@ def generate_CatGT_input_json(self): / f'{session_str}_g{gate_str}_imec{probe_str}' / f'{session_str}_g{gate_str}_t{trig_str}.imec{probe_str}.ap.bin') continuous_file_symlink.parent.mkdir(parents=True, exist_ok=True) - continuous_file_symlink.unlink(missing_ok=True) + if continuous_file_symlink.exists(): + continuous_file_symlink.unlink() continuous_file_symlink.symlink_to(continuous_file) input_meta_fullpath_symlink = (input_meta_fullpath.parent / f'{session_str}_g{gate_str}' / f'{session_str}_g{gate_str}_imec{probe_str}' / f'{session_str}_g{gate_str}_t{trig_str}.imec{probe_str}.ap.meta') input_meta_fullpath_symlink.parent.mkdir(parents=True, exist_ok=True) - input_meta_fullpath_symlink.unlink(missing_ok=True) + if input_meta_fullpath_symlink.exists(): + input_meta_fullpath_symlink.unlink() input_meta_fullpath_symlink.symlink_to(input_meta_fullpath) createInputJson(self._catGT_input_json.as_posix(), From c33d1b02811dbaf7e492da60e70819d5102798ff Mon Sep 17 00:00:00 2001 From: Thinh Nguyen Date: Fri, 2 Sep 2022 15:20:22 -0500 Subject: [PATCH 7/8] bugfix - path search for catgt output --- element_array_ephys/readers/kilosort_triggering.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/element_array_ephys/readers/kilosort_triggering.py b/element_array_ephys/readers/kilosort_triggering.py index a1797721..3645dc1a 100644 --- a/element_array_ephys/readers/kilosort_triggering.py +++ b/element_array_ephys/readers/kilosort_triggering.py @@ -232,7 +232,7 @@ def _get_raw_data_filepaths(self): session_str, gate_str, _, probe_str = self.parse_input_filename() if self._CatGT_finished: - catGT_dest = self._ks_output_dir.parent + catGT_dest = self._ks_output_dir run_str = session_str + '_g' + gate_str run_folder = 'catgt_' + run_str prb_folder = run_str + '_imec' + probe_str From f60ba3d6a0c0776d1f7b97d02b9d1608a5788a0e Mon Sep 17 00:00:00 2001 From: Thinh Nguyen Date: Fri, 2 Sep 2022 15:26:11 -0500 Subject: [PATCH 8/8] improve error handling --- element_array_ephys/readers/kilosort_triggering.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/element_array_ephys/readers/kilosort_triggering.py b/element_array_ephys/readers/kilosort_triggering.py index 3645dc1a..4bdd1fed 100644 --- a/element_array_ephys/readers/kilosort_triggering.py +++ b/element_array_ephys/readers/kilosort_triggering.py @@ -239,9 +239,11 @@ def _get_raw_data_filepaths(self): data_directory = catGT_dest / run_folder / prb_folder else: data_directory = self._npx_input_dir - - meta_fp = next(data_directory.glob(f'{session_str}*.ap.meta')) - bin_fp = next(data_directory.glob(f'{session_str}*.ap.bin')) + try: + meta_fp = next(data_directory.glob(f'{session_str}*.ap.meta')) + bin_fp = next(data_directory.glob(f'{session_str}*.ap.bin')) + except StopIteration: + raise RuntimeError(f'No ap meta/bin files found in {data_directory} - CatGT error?') return meta_fp, bin_fp