From be0ba255423da3e0a2f806ce82aa54005fa1f04b Mon Sep 17 00:00:00 2001 From: Miles Wells Date: Thu, 11 May 2023 13:59:43 +0300 Subject: [PATCH 1/6] Remove deprecated usage; subjectTrials documentation --- brainbox/io/spikeglx.py | 11 ++-- .../loading_data/loading_trials_data.ipynb | 56 ++++++++++++++++--- 2 files changed, 54 insertions(+), 13 deletions(-) diff --git a/brainbox/io/spikeglx.py b/brainbox/io/spikeglx.py index 333009427..9c0618c11 100644 --- a/brainbox/io/spikeglx.py +++ b/brainbox/io/spikeglx.py @@ -7,7 +7,7 @@ import random import numpy as np -from one.alf.io import remove_uuid_file +from one.alf.files import remove_uuid_string import spikeglx @@ -46,13 +46,13 @@ def extract_waveforms(ephys_file, ts, ch, t=2.0, sr=30000, n_ch_probe=385, car=T 1) Extract all the waveforms for unit1 with and without CAR. >>> import numpy as np >>> import brainbox as bb - >>> import alf.io as aio + >>> import one.alf.io as alfio >>> import ibllib.ephys.spikes as e_spks (*Note, if there is no 'alf' directory, make 'alf' directory from 'ks2' output directory): >>> e_spks.ks2_to_alf(path_to_ks_out, path_to_alf_out) # Get a clusters bunch and a units bunch from a spikes bunch from an alf directory. - >>> clstrs_b = aio.load_object(path_to_alf_out, 'clusters') - >>> spks_b = aio.load_object(path_to_alf_out, 'spikes') + >>> clstrs_b = alfio.load_object(path_to_alf_out, 'clusters') + >>> spks_b = alfio.load_object(path_to_alf_out, 'spikes') >>> units_b = bb.processing.get_units_bunch(spks, ['times']) # Get the timestamps and 20 channels around the max amp channel for unit1, and extract the # two sets of waveforms. @@ -237,8 +237,7 @@ def _download_raw_partial(self, first_chunk=0, last_chunk=0): raise e _logger.warning(f'Failed to download chunk {first_chunk} to {last_chunk}, retrying') time.sleep(1) - cbin_local_path = remove_uuid_file(cbin_local_path) - cbin_local_path_renamed = cbin_local_path.with_suffix('.stream.cbin') + cbin_local_path_renamed = remove_uuid_string(cbin_local_path).with_suffix('.stream.cbin') cbin_local_path.replace(cbin_local_path_renamed) assert cbin_local_path_renamed.exists() diff --git a/examples/loading_data/loading_trials_data.ipynb b/examples/loading_data/loading_trials_data.ipynb index 05c3f715a..8ea7bdb62 100644 --- a/examples/loading_data/loading_trials_data.ipynb +++ b/examples/loading_data/loading_trials_data.ipynb @@ -37,15 +37,45 @@ "metadata": {}, "source": [ "## Relevant Alf objects\n", - "* trials" + "* trials\n", + "* subjectTrials\n", + "* subjectTraining" ] }, + { + "cell_type": "markdown", + "source": [ + "## Loading a session's trials" + ], + "metadata": { + "collapsed": false + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [ + "from one.api import ONE\n", + "one = ONE()\n", + "eid = '4ecb5d24-f5cc-402c-be28-9d0f7cb14b3a'\n", + "trials = one.load_object(eid, 'trials', collection='alf')" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, { "cell_type": "markdown", "id": "0514237a", "metadata": {}, "source": [ - "## Loading" + "## Loading a subject's trials\n", + "This loads all trials data for a given subject (all session trials concatenated) into a DataFrame.\n", + "The subjectTraining table contains the training statuses." ] }, { @@ -57,8 +87,16 @@ "source": [ "from one.api import ONE\n", "one = ONE()\n", - "eid = '4ecb5d24-f5cc-402c-be28-9d0f7cb14b3a'\n", - "trials = one.load_object(eid, 'trials', collection='alf')" + "subject = 'SWC_043'\n", + "trials = one.load_aggregate('subjects', subject, '_ibl_subjectTrials.table')\n", + "\n", + "# Load training status and join to trials table\n", + "training = one.load_aggregate('subjects', subject, '_ibl_subjectTraining.table')\n", + "trials = (trials\n", + " .set_index('session')\n", + " .join(training.set_index('session'))\n", + " .sort_values(by='session_start_time')\n", + " .fillna(method='ffill'))" ] }, { @@ -108,7 +146,7 @@ "source": [ "from brainbox.behavior.training import compute_performance\n", "\n", - "# compute performance \n", + "# compute performance\n", "performance, contrasts, n_contrasts = compute_performance(trials)\n", "\n", "# compute performance expressed as probability of choosing right\n", @@ -214,7 +252,11 @@ { "cell_type": "markdown", "id": "5738f9fb", - "metadata": {}, + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, "source": [ "## Other relevant examples\n", "* COMING SOON" @@ -243,4 +285,4 @@ }, "nbformat": 4, "nbformat_minor": 5 -} +} \ No newline at end of file From 87ced2656abe4bb2f2e50bde445ff8fdf0711c57 Mon Sep 17 00:00:00 2001 From: Miles Wells Date: Fri, 12 May 2023 13:06:55 +0300 Subject: [PATCH 2/6] Skip cell execution --- examples/loading_data/loading_trials_data.ipynb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/examples/loading_data/loading_trials_data.ipynb b/examples/loading_data/loading_trials_data.ipynb index 8ea7bdb62..fa7f2d736 100644 --- a/examples/loading_data/loading_trials_data.ipynb +++ b/examples/loading_data/loading_trials_data.ipynb @@ -82,7 +82,9 @@ "cell_type": "code", "execution_count": null, "id": "544f4e8d", - "metadata": {}, + "metadata": { + "ibl_execute": false + }, "outputs": [], "source": [ "from one.api import ONE\n", From 9a69d18c2a12da26bb19c458110fc693282ae319 Mon Sep 17 00:00:00 2001 From: Miles Wells Date: Fri, 12 May 2023 16:19:04 +0300 Subject: [PATCH 3/6] Update requirements --- requirements.txt | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index 012e5b4da..a8d0eade1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,8 +10,7 @@ numba>=0.56 numpy>=1.18 nptdms opencv-python-headless -# pandas>=0.24.2 -pandas<2.0 +pandas phylib>=2.4 pyarrow pynrrd>=0.4.0 @@ -25,6 +24,6 @@ tqdm>=4.32.1 ibl-neuropixel>=0.4.0 iblutil>=1.5.0 labcams # widefield extractor -ONE-api>=1.19.1 +ONE-api>=2.0 slidingRP>=1.0.0 # steinmetz lab refractory period metrics wfield>=0.3.6 # widefield extractor From 7d97e486d3fda2af366317a4967ededb59f82400 Mon Sep 17 00:00:00 2001 From: Miles Wells Date: Fri, 12 May 2023 17:42:39 +0300 Subject: [PATCH 4/6] Test on py v3.11 --- .github/workflows/ibllib_ci.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ibllib_ci.yml b/.github/workflows/ibllib_ci.yml index a799ac431..64a98c108 100644 --- a/.github/workflows/ibllib_ci.yml +++ b/.github/workflows/ibllib_ci.yml @@ -18,7 +18,12 @@ jobs: max-parallel: 4 matrix: os: ["windows-latest", "ubuntu-latest"] - python-version: ["3.8"] + python-version: ["3.8", "3.11"] + exclude: + - os: windows-latest + python-version: 3.8 + - os: ubuntu-latest + python-version: 3.11 steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} From dbb2c4f25c2c113647e7559c7dfc8f6097e3583d Mon Sep 17 00:00:00 2001 From: Olivier Winter Date: Mon, 15 May 2023 18:48:12 +0100 Subject: [PATCH 5/6] add quiescence period extracted file (#606) * add quiescence period extracted file * Extract quiescence period for ephys sessions * add quiescence period to FPGA extractor * Always create new tag in registration test --------- Co-authored-by: Miles Wells --- ibllib/io/extractors/biased_trials.py | 2 +- ibllib/io/extractors/ephys_fpga.py | 2 +- ibllib/io/extractors/training_trials.py | 2 +- ibllib/tests/extractors/test_ephys_trials.py | 2 +- ibllib/tests/test_oneibl.py | 16 ++++++---------- 5 files changed, 10 insertions(+), 14 deletions(-) diff --git a/ibllib/io/extractors/biased_trials.py b/ibllib/io/extractors/biased_trials.py index de3637277..2d3e92e8e 100644 --- a/ibllib/io/extractors/biased_trials.py +++ b/ibllib/io/extractors/biased_trials.py @@ -34,7 +34,7 @@ class ProbaContrasts(BaseBpodTrialsExtractor): Bpod pre-generated values for probabilityLeft, contrastLR, phase, quiescence """ save_names = ('_ibl_trials.contrastLeft.npy', '_ibl_trials.contrastRight.npy', None, None, - '_ibl_trials.probabilityLeft.npy', None) + '_ibl_trials.probabilityLeft.npy', '_ibl_trials.quiescencePeriod.npy') var_names = ('contrastLeft', 'contrastRight', 'phase', 'position', 'probabilityLeft', 'quiescence') diff --git a/ibllib/io/extractors/ephys_fpga.py b/ibllib/io/extractors/ephys_fpga.py index 73b57d8d0..7fe9e4758 100644 --- a/ibllib/io/extractors/ephys_fpga.py +++ b/ibllib/io/extractors/ephys_fpga.py @@ -694,7 +694,7 @@ def get_protocol_period(session_path, protocol_number, bpod_sync): class FpgaTrials(extractors_base.BaseExtractor): save_names = ('_ibl_trials.intervals_bpod.npy', '_ibl_trials.goCueTrigger_times.npy', None, None, None, None, None, None, None, - '_ibl_trials.stimOff_times.npy', None, None, None, None, + '_ibl_trials.stimOff_times.npy', None, None, None, '_ibl_trials.quiescencePeriod.npy', '_ibl_trials.table.pqt', '_ibl_wheel.timestamps.npy', '_ibl_wheel.position.npy', '_ibl_wheelMoves.intervals.npy', '_ibl_wheelMoves.peakAmplitude.npy') diff --git a/ibllib/io/extractors/training_trials.py b/ibllib/io/extractors/training_trials.py index c260f39ff..e18f9e30a 100644 --- a/ibllib/io/extractors/training_trials.py +++ b/ibllib/io/extractors/training_trials.py @@ -649,7 +649,7 @@ class PhasePosQuiescence(BaseBpodTrialsExtractor): """Extracts stimulus phase, position and quiescence from Bpod data. For extraction of pre-generated events, use the ProbaContrasts extractor instead. """ - save_names = (None, None, None) + save_names = (None, None, '_ibl_trials.quiescencePeriod.npy') var_names = ('phase', 'position', 'quiescence') def _extract(self, **kwargs): diff --git a/ibllib/tests/extractors/test_ephys_trials.py b/ibllib/tests/extractors/test_ephys_trials.py index a310eb32f..d22665137 100644 --- a/ibllib/tests/extractors/test_ephys_trials.py +++ b/ibllib/tests/extractors/test_ephys_trials.py @@ -127,5 +127,5 @@ def test_get_probabilityLeft(self): self.assertTrue(all([x in [0.2, 0.5, 0.8] for x in np.unique(pLeft1)])) -if __name__ == "__main__": +if __name__ == '__main__': unittest.main(exit=False, verbosity=2) diff --git a/ibllib/tests/test_oneibl.py b/ibllib/tests/test_oneibl.py index fcf37ba63..3042fb98f 100644 --- a/ibllib/tests/test_oneibl.py +++ b/ibllib/tests/test_oneibl.py @@ -210,13 +210,9 @@ def setUp(self) -> None: self.rev = self.one.alyx.rest('revisions', 'read', id=self.revision) except HTTPError: self.rev = self.one.alyx.rest('revisions', 'create', data={'name': self.revision}) - # Create a tag if doesn't already exist - try: - self.tag = next(x for x in self.one.alyx.rest('tags', 'list') - if x['name'] == 'test_tag') - except StopIteration: - self.tag = self.one.alyx.rest('tags', 'create', - data={'name': 'test_tag', 'protected': True}) + # Create a new tag + tag_data = {'name': f'test_tag_{np.random.randint(0, 1e3)}', 'protected': True} + self.tag = self.one.alyx.rest('tags', 'create', data=tag_data) def test_registration_datasets(self): # registers a single file @@ -252,7 +248,7 @@ def test_registration_datasets(self): dsets = self.one.alyx.rest('datasets', 'list', session=ses['url'][-36:]) for d in dsets: self.one.alyx.rest('datasets', 'partial_update', - id=d['url'][-36:], data={'tags': ['test_tag']}) + id=d['url'][-36:], data={'tags': [self.tag['name']]}) # Test registering with a revision already in the file path, should use this rather than create one with today's date flist = list(self.rev_path.glob('*.npy')) @@ -265,7 +261,7 @@ def test_registration_datasets(self): dsets = self.one.alyx.rest('datasets', 'list', session=ses['url'][-36:]) for d in dsets: self.one.alyx.rest('datasets', 'partial_update', - id=d['url'][-36:], data={'tags': ['test_tag']}) + id=d['url'][-36:], data={'tags': [self.tag['name']]}) # Register again with revision in file path, it should register to self.revision + a flist = list(self.rev_path.glob('*.npy')) @@ -290,7 +286,7 @@ def test_registration_datasets(self): dsets = self.one.alyx.rest('datasets', 'list', session=ses['url'][-36:], no_cache=True) for d in dsets: self.one.alyx.rest('datasets', 'partial_update', - id=d['url'][-36:], data={'tags': ['test_tag']}) + id=d['url'][-36:], data={'tags': [self.tag['name']]}) # Same day revision # Need to remake the original files From 24d29510e784d93e9753c9f527ff0a3367a9f930 Mon Sep 17 00:00:00 2001 From: olivier Date: Fri, 19 May 2023 15:17:31 +0100 Subject: [PATCH 6/6] add release notes and bump version number --- ibllib/__init__.py | 2 +- release_notes.md | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/ibllib/__init__.py b/ibllib/__init__.py index 18c710c25..a626765a6 100644 --- a/ibllib/__init__.py +++ b/ibllib/__init__.py @@ -2,7 +2,7 @@ import logging import warnings -__version__ = '2.22.3' +__version__ = '2.23.0' warnings.filterwarnings('always', category=DeprecationWarning, module='ibllib') # if this becomes a full-blown library we should let the logging configuration to the discretion of the dev diff --git a/release_notes.md b/release_notes.md index a61c9fc17..35db1d74f 100644 --- a/release_notes.md +++ b/release_notes.md @@ -1,3 +1,8 @@ +## Release Notes 2.23 +### Release Notes 2.23.0 2023-05-19 +- quiescence period extraction +- ONEv2 requirement + ## Release Notes 2.22 ### Release Notes 2.22.3 2023-05-03