Merge branch 'release/2.23.0'

.github/workflows/ibllib_ci.yml

@@ -18,7 +18,12 @@ jobs:
       max-parallel: 4
       matrix:
         os: ["windows-latest", "ubuntu-latest"]
-        python-version: ["3.8"]
+        python-version: ["3.8", "3.11"]
+        exclude:
+          - os: windows-latest
+            python-version: 3.8
+          - os: ubuntu-latest
+            python-version: 3.11
     steps:
     - uses: actions/checkout@v2
     - name: Set up Python ${{ matrix.python-version }}

brainbox/io/spikeglx.py

@@ -7,7 +7,7 @@
 import random
 
 import numpy as np
-from one.alf.io import remove_uuid_file
+from one.alf.files import remove_uuid_string
 
 import spikeglx
 
@@ -46,13 +46,13 @@ def extract_waveforms(ephys_file, ts, ch, t=2.0, sr=30000, n_ch_probe=385, car=T
     1) Extract all the waveforms for unit1 with and without CAR.
         >>> import numpy as np
         >>> import brainbox as bb
-        >>> import alf.io as aio
+        >>> import one.alf.io as alfio
         >>> import ibllib.ephys.spikes as e_spks
         (*Note, if there is no 'alf' directory, make 'alf' directory from 'ks2' output directory):
         >>> e_spks.ks2_to_alf(path_to_ks_out, path_to_alf_out)
         # Get a clusters bunch and a units bunch from a spikes bunch from an alf directory.
-        >>> clstrs_b = aio.load_object(path_to_alf_out, 'clusters')
-        >>> spks_b = aio.load_object(path_to_alf_out, 'spikes')
+        >>> clstrs_b = alfio.load_object(path_to_alf_out, 'clusters')
+        >>> spks_b = alfio.load_object(path_to_alf_out, 'spikes')
         >>> units_b = bb.processing.get_units_bunch(spks, ['times'])
         # Get the timestamps and 20 channels around the max amp channel for unit1, and extract the
         # two sets of waveforms.
@@ -237,8 +237,7 @@ def _download_raw_partial(self, first_chunk=0, last_chunk=0):
                     raise e
                 _logger.warning(f'Failed to download chunk {first_chunk} to {last_chunk}, retrying')
                 time.sleep(1)
-        cbin_local_path = remove_uuid_file(cbin_local_path)
-        cbin_local_path_renamed = cbin_local_path.with_suffix('.stream.cbin')
+        cbin_local_path_renamed = remove_uuid_string(cbin_local_path).with_suffix('.stream.cbin')
         cbin_local_path.replace(cbin_local_path_renamed)
         assert cbin_local_path_renamed.exists()
 

examples/loading_data/loading_trials_data.ipynb

@@ -37,28 +37,68 @@
    "metadata": {},
    "source": [
     "## Relevant Alf objects\n",
-    "* trials"
+    "* trials\n",
+    "* subjectTrials\n",
+    "* subjectTraining"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "source": [
+    "## Loading a session's trials"
+   ],
+   "metadata": {
+    "collapsed": false
+   }
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "outputs": [],
+   "source": [
+    "from one.api import ONE\n",
+    "one = ONE()\n",
+    "eid = '4ecb5d24-f5cc-402c-be28-9d0f7cb14b3a'\n",
+    "trials = one.load_object(eid, 'trials', collection='alf')"
+   ],
+   "metadata": {
+    "collapsed": false,
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   }
+  },
   {
    "cell_type": "markdown",
    "id": "0514237a",
    "metadata": {},
    "source": [
-    "## Loading"
+    "## Loading a subject's trials\n",
+    "This loads all trials data for a given subject (all session trials concatenated) into a DataFrame.\n",
+    "The subjectTraining table contains the training statuses."
    ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
    "id": "544f4e8d",
-   "metadata": {},
+   "metadata": {
+    "ibl_execute": false
+   },
    "outputs": [],
    "source": [
     "from one.api import ONE\n",
     "one = ONE()\n",
-    "eid = '4ecb5d24-f5cc-402c-be28-9d0f7cb14b3a'\n",
-    "trials = one.load_object(eid, 'trials', collection='alf')"
+    "subject = 'SWC_043'\n",
+    "trials = one.load_aggregate('subjects', subject, '_ibl_subjectTrials.table')\n",
+    "\n",
+    "# Load training status and join to trials table\n",
+    "training = one.load_aggregate('subjects', subject, '_ibl_subjectTraining.table')\n",
+    "trials = (trials\n",
+    "          .set_index('session')\n",
+    "          .join(training.set_index('session'))\n",
+    "          .sort_values(by='session_start_time')\n",
+    "          .fillna(method='ffill'))"
    ]
   },
   {
@@ -108,7 +148,7 @@
    "source": [
     "from brainbox.behavior.training import compute_performance\n",
     "\n",
-    "# compute performance \n",
+    "# compute performance\n",
     "performance, contrasts, n_contrasts = compute_performance(trials)\n",
     "\n",
     "# compute performance expressed as probability of choosing right\n",
@@ -214,7 +254,11 @@
   {
    "cell_type": "markdown",
    "id": "5738f9fb",
-   "metadata": {},
+   "metadata": {
+    "pycharm": {
+     "name": "#%% md\n"
+    }
+   },
    "source": [
     "## Other relevant examples\n",
     "* COMING SOON"
@@ -243,4 +287,4 @@
  },
  "nbformat": 4,
  "nbformat_minor": 5
-}
+}

ibllib/__init__.py

@@ -2,7 +2,7 @@
 import logging
 import warnings
 
-__version__ = '2.22.3'
+__version__ = '2.23.0'
 warnings.filterwarnings('always', category=DeprecationWarning, module='ibllib')
 
 # if this becomes a full-blown library we should let the logging configuration to the discretion of the dev

ibllib/io/extractors/biased_trials.py

@@ -34,7 +34,7 @@ class ProbaContrasts(BaseBpodTrialsExtractor):
     Bpod pre-generated values for probabilityLeft, contrastLR, phase, quiescence
     """
     save_names = ('_ibl_trials.contrastLeft.npy', '_ibl_trials.contrastRight.npy', None, None,
-                  '_ibl_trials.probabilityLeft.npy', None)
+                  '_ibl_trials.probabilityLeft.npy', '_ibl_trials.quiescencePeriod.npy')
     var_names = ('contrastLeft', 'contrastRight', 'phase',
                  'position', 'probabilityLeft', 'quiescence')
 

ibllib/io/extractors/ephys_fpga.py

@@ -694,7 +694,7 @@ def get_protocol_period(session_path, protocol_number, bpod_sync):
 class FpgaTrials(extractors_base.BaseExtractor):
     save_names = ('_ibl_trials.intervals_bpod.npy',
                   '_ibl_trials.goCueTrigger_times.npy', None, None, None, None, None, None, None,
-                  '_ibl_trials.stimOff_times.npy', None, None, None, None,
+                  '_ibl_trials.stimOff_times.npy', None, None, None, '_ibl_trials.quiescencePeriod.npy',
                   '_ibl_trials.table.pqt', '_ibl_wheel.timestamps.npy',
                   '_ibl_wheel.position.npy', '_ibl_wheelMoves.intervals.npy',
                   '_ibl_wheelMoves.peakAmplitude.npy')

ibllib/io/extractors/training_trials.py

@@ -649,7 +649,7 @@ class PhasePosQuiescence(BaseBpodTrialsExtractor):
     """Extracts stimulus phase, position and quiescence from Bpod data.
     For extraction of pre-generated events, use the ProbaContrasts extractor instead.
     """
-    save_names = (None, None, None)
+    save_names = (None, None, '_ibl_trials.quiescencePeriod.npy')
     var_names = ('phase', 'position', 'quiescence')
 
     def _extract(self, **kwargs):

ibllib/tests/extractors/test_ephys_trials.py

@@ -127,5 +127,5 @@ def test_get_probabilityLeft(self):
         self.assertTrue(all([x in [0.2, 0.5, 0.8] for x in np.unique(pLeft1)]))
 
 
-if __name__ == "__main__":
+if __name__ == '__main__':
     unittest.main(exit=False, verbosity=2)

ibllib/tests/test_oneibl.py

@@ -210,13 +210,9 @@ def setUp(self) -> None:
             self.rev = self.one.alyx.rest('revisions', 'read', id=self.revision)
         except HTTPError:
             self.rev = self.one.alyx.rest('revisions', 'create', data={'name': self.revision})
-        # Create a tag if doesn't already exist
-        try:
-            self.tag = next(x for x in self.one.alyx.rest('tags', 'list')
-                            if x['name'] == 'test_tag')
-        except StopIteration:
-            self.tag = self.one.alyx.rest('tags', 'create',
-                                          data={'name': 'test_tag', 'protected': True})
+        # Create a new tag
+        tag_data = {'name': f'test_tag_{np.random.randint(0, 1e3)}', 'protected': True}
+        self.tag = self.one.alyx.rest('tags', 'create', data=tag_data)
 
     def test_registration_datasets(self):
         # registers a single file
@@ -252,7 +248,7 @@ def test_registration_datasets(self):
         dsets = self.one.alyx.rest('datasets', 'list', session=ses['url'][-36:])
         for d in dsets:
             self.one.alyx.rest('datasets', 'partial_update',
-                               id=d['url'][-36:], data={'tags': ['test_tag']})
+                               id=d['url'][-36:], data={'tags': [self.tag['name']]})
 
         # Test registering with a revision already in the file path, should use this rather than create one with today's date
         flist = list(self.rev_path.glob('*.npy'))
@@ -265,7 +261,7 @@ def test_registration_datasets(self):
         dsets = self.one.alyx.rest('datasets', 'list', session=ses['url'][-36:])
         for d in dsets:
             self.one.alyx.rest('datasets', 'partial_update',
-                               id=d['url'][-36:], data={'tags': ['test_tag']})
+                               id=d['url'][-36:], data={'tags': [self.tag['name']]})
 
         # Register again with revision in file path, it should register to self.revision + a
         flist = list(self.rev_path.glob('*.npy'))
@@ -290,7 +286,7 @@ def test_registration_datasets(self):
         dsets = self.one.alyx.rest('datasets', 'list', session=ses['url'][-36:], no_cache=True)
         for d in dsets:
             self.one.alyx.rest('datasets', 'partial_update',
-                               id=d['url'][-36:], data={'tags': ['test_tag']})
+                               id=d['url'][-36:], data={'tags': [self.tag['name']]})
 
         # Same day revision
         # Need to remake the original files

release_notes.md

@@ -1,3 +1,8 @@
+## Release Notes 2.23
+### Release Notes 2.23.0 2023-05-19
+- quiescence period extraction
+- ONEv2 requirement
+
 ## Release Notes 2.22
 
 ### Release Notes 2.22.3 2023-05-03

requirements.txt

@@ -10,8 +10,7 @@ numba>=0.56
 numpy>=1.18
 nptdms
 opencv-python-headless
-# pandas>=0.24.2
-pandas<2.0
+pandas
 phylib>=2.4
 pyarrow
 pynrrd>=0.4.0
@@ -25,6 +24,6 @@ tqdm>=4.32.1
 ibl-neuropixel>=0.4.0
 iblutil>=1.5.0
 labcams  # widefield extractor
-ONE-api>=1.19.1
+ONE-api>=2.0
 slidingRP>=1.0.0  # steinmetz lab refractory period metrics
 wfield>=0.3.6  # widefield extractor