Merge pull request #608 from int-brain-lab/ONEv2

ONEv2

.github/workflows/ibllib_ci.yml

@@ -18,7 +18,12 @@ jobs:
       max-parallel: 4
       matrix:
         os: ["windows-latest", "ubuntu-latest"]
-        python-version: ["3.8"]
+        python-version: ["3.8", "3.11"]
+        exclude:
+          - os: windows-latest
+            python-version: 3.8
+          - os: ubuntu-latest
+            python-version: 3.11
     steps:
     - uses: actions/checkout@v2
     - name: Set up Python ${{ matrix.python-version }}

brainbox/io/spikeglx.py

@@ -7,7 +7,7 @@
 import random
 
 import numpy as np
-from one.alf.io import remove_uuid_file
+from one.alf.files import remove_uuid_string
 
 import spikeglx
 
@@ -46,13 +46,13 @@ def extract_waveforms(ephys_file, ts, ch, t=2.0, sr=30000, n_ch_probe=385, car=T
     1) Extract all the waveforms for unit1 with and without CAR.
         >>> import numpy as np
         >>> import brainbox as bb
-        >>> import alf.io as aio
+        >>> import one.alf.io as alfio
         >>> import ibllib.ephys.spikes as e_spks
         (*Note, if there is no 'alf' directory, make 'alf' directory from 'ks2' output directory):
         >>> e_spks.ks2_to_alf(path_to_ks_out, path_to_alf_out)
         # Get a clusters bunch and a units bunch from a spikes bunch from an alf directory.
-        >>> clstrs_b = aio.load_object(path_to_alf_out, 'clusters')
-        >>> spks_b = aio.load_object(path_to_alf_out, 'spikes')
+        >>> clstrs_b = alfio.load_object(path_to_alf_out, 'clusters')
+        >>> spks_b = alfio.load_object(path_to_alf_out, 'spikes')
         >>> units_b = bb.processing.get_units_bunch(spks, ['times'])
         # Get the timestamps and 20 channels around the max amp channel for unit1, and extract the
         # two sets of waveforms.
@@ -237,8 +237,7 @@ def _download_raw_partial(self, first_chunk=0, last_chunk=0):
                     raise e
                 _logger.warning(f'Failed to download chunk {first_chunk} to {last_chunk}, retrying')
                 time.sleep(1)
-        cbin_local_path = remove_uuid_file(cbin_local_path)
-        cbin_local_path_renamed = cbin_local_path.with_suffix('.stream.cbin')
+        cbin_local_path_renamed = remove_uuid_string(cbin_local_path).with_suffix('.stream.cbin')
         cbin_local_path.replace(cbin_local_path_renamed)
         assert cbin_local_path_renamed.exists()
 

examples/loading_data/loading_trials_data.ipynb

@@ -37,28 +37,68 @@
    "metadata": {},
    "source": [
     "## Relevant Alf objects\n",
-    "* trials"
+    "* trials\n",
+    "* subjectTrials\n",
+    "* subjectTraining"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "source": [
+    "## Loading a session's trials"
+   ],
+   "metadata": {
+    "collapsed": false
+   }
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "outputs": [],
+   "source": [
+    "from one.api import ONE\n",
+    "one = ONE()\n",
+    "eid = '4ecb5d24-f5cc-402c-be28-9d0f7cb14b3a'\n",
+    "trials = one.load_object(eid, 'trials', collection='alf')"
+   ],
+   "metadata": {
+    "collapsed": false,
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   }
+  },
   {
    "cell_type": "markdown",
    "id": "0514237a",
    "metadata": {},
    "source": [
-    "## Loading"
+    "## Loading a subject's trials\n",
+    "This loads all trials data for a given subject (all session trials concatenated) into a DataFrame.\n",
+    "The subjectTraining table contains the training statuses."
    ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
    "id": "544f4e8d",
-   "metadata": {},
+   "metadata": {
+    "ibl_execute": false
+   },
    "outputs": [],
    "source": [
     "from one.api import ONE\n",
     "one = ONE()\n",
-    "eid = '4ecb5d24-f5cc-402c-be28-9d0f7cb14b3a'\n",
-    "trials = one.load_object(eid, 'trials', collection='alf')"
+    "subject = 'SWC_043'\n",
+    "trials = one.load_aggregate('subjects', subject, '_ibl_subjectTrials.table')\n",
+    "\n",
+    "# Load training status and join to trials table\n",
+    "training = one.load_aggregate('subjects', subject, '_ibl_subjectTraining.table')\n",
+    "trials = (trials\n",
+    "          .set_index('session')\n",
+    "          .join(training.set_index('session'))\n",
+    "          .sort_values(by='session_start_time')\n",
+    "          .fillna(method='ffill'))"
    ]
   },
   {
@@ -108,7 +148,7 @@
    "source": [
     "from brainbox.behavior.training import compute_performance\n",
     "\n",
-    "# compute performance \n",
+    "# compute performance\n",
     "performance, contrasts, n_contrasts = compute_performance(trials)\n",
     "\n",
     "# compute performance expressed as probability of choosing right\n",
@@ -214,7 +254,11 @@
   {
    "cell_type": "markdown",
    "id": "5738f9fb",
-   "metadata": {},
+   "metadata": {
+    "pycharm": {
+     "name": "#%% md\n"
+    }
+   },
    "source": [
     "## Other relevant examples\n",
     "* COMING SOON"
@@ -243,4 +287,4 @@
  },
  "nbformat": 4,
  "nbformat_minor": 5
-}
+}

requirements.txt

@@ -10,8 +10,7 @@ numba>=0.56
 numpy>=1.18
 nptdms
 opencv-python-headless
-# pandas>=0.24.2
-pandas<2.0
+pandas
 phylib>=2.4
 pyarrow
 pynrrd>=0.4.0
@@ -25,6 +24,6 @@ tqdm>=4.32.1
 ibl-neuropixel>=0.4.0
 iblutil>=1.5.0
 labcams  # widefield extractor
-ONE-api>=1.19.1
+ONE-api>=2.0
 slidingRP>=1.0.0  # steinmetz lab refractory period metrics
 wfield>=0.3.6  # widefield extractor