diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile
index b87fb76..c63a49d 100644
--- a/.devcontainer/Dockerfile
+++ b/.devcontainer/Dockerfile
@@ -25,11 +25,14 @@ RUN \
# dircolors -b >> /home/vscode/.bashrc && \ # somehow fix colors
apt-get clean
COPY ./requirements.txt /tmp/
+COPY ./requirements_dev.txt /tmp/
RUN \
# workflow dependencies
apt-get install gcc ffmpeg libsm6 libxext6 -y && \
pip install --no-cache-dir -r /tmp/requirements.txt && \
+ pip install --no-cache-dir -r /tmp/requirements_dev.txt && \
rm /tmp/requirements.txt && \
+ rm /tmp/requirements_dev.txt && \
apt-get clean
ENV DJ_HOST fakeservices.datajoint.io
diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
index 183006c..0768b84 100644
--- a/.devcontainer/devcontainer.json
+++ b/.devcontainer/devcontainer.json
@@ -1,6 +1,6 @@
// For format details, see https://aka.ms/devcontainer.json.
{
- "name": "Remote Demo",
+ "name": "Demo with remote data",
"dockerComposeFile": "docker-compose.yaml",
"service": "app",
"workspaceFolder": "/workspaces/${localWorkspaceFolderBasename}",
diff --git a/.devcontainer/local/devcontainer.json b/.devcontainer/local/devcontainer.json
index 8b02b9f..68692fe 100644
--- a/.devcontainer/local/devcontainer.json
+++ b/.devcontainer/local/devcontainer.json
@@ -1,6 +1,6 @@
// For format details, see https://aka.ms/devcontainer.json.
{
- "name": "Local Demo",
+ "name": "Demo with local data",
"dockerComposeFile": "docker-compose.yaml",
"service": "app",
"workspaceFolder": "/workspaces/${localWorkspaceFolderBasename}",
diff --git a/.gitignore b/.gitignore
index 8d5e3eb..954c942 100644
--- a/.gitignore
+++ b/.gitignore
@@ -109,4 +109,5 @@ docker-compose.yml
# temporary figures
temp_ephys_figures/
-example_data
\ No newline at end of file
+example_data
+*.code-workspace
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 4e8653f..52bdae1 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -3,6 +3,12 @@
Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and
[Keep a Changelog](https://keepachangelog.com/en/1.0.0/) convention.
+## [0.3.2] - 2023-04-14
+
++ Add - `quality_metrics.ipynb` for visualizing quality metrics.
++ Add - Documentation for attributes in `ephys.QualityMetrics.Waveform`.
++ Update - pytest for `ephys.QualityMetrics.populate`.
+
## [0.3.1] - 2023-04-12
+ Add - pytest for new `QCmetric` tables.
@@ -87,6 +93,7 @@ Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and
+ Add - Version
+[0.3.2]: https://github.com/datajoint/workflow-array-ephys/releases/tag/0.3.2
[0.3.1]: https://github.com/datajoint/workflow-array-ephys/releases/tag/0.3.1
[0.3.0]: https://github.com/datajoint/workflow-array-ephys/releases/tag/0.3.0
[0.2.6]: https://github.com/datajoint/workflow-array-ephys/releases/tag/0.2.6
diff --git a/README.md b/README.md
index 6ee63cf..9c1c72a 100644
--- a/README.md
+++ b/README.md
@@ -26,17 +26,18 @@ The easiest way to learn about DataJoint Elements is to use the tutorial noteboo
Here are some options that provide a great experience:
-- **Cloud-based IDE**: (*recommended*)
+- Cloud-based Development Environment: (*recommended*)
- Launch using [GitHub Codespaces](https://github.com/features/codespaces) using the `+` option which will `Create codespace on main` in the codebase repository on your fork with default options. For more control, see the `...` where you may create `New with options...`.
- Build time for a codespace is **~7m**. This is done infrequently and cached for convenience.
- Start time for a codespace is **~30s**. This will pull the built codespace from cache when you need it.
- *Tip*: Each month, GitHub renews a [free-tier](https://docs.github.com/en/billing/managing-billing-for-github-codespaces/about-billing-for-github-codespaces#monthly-included-storage-and-core-hours-for-personal-accounts) quota of compute and storage. Typically we run into the storage limits before anything else since Codespaces consume storage while stopped. It is best to delete Codespaces when not actively in use and recreate when needed. We'll soon be creating prebuilds to avoid larger build times. Once any portion of your quota is reached, you will need to wait for it to be reset at the end of your cycle or add billing info to your GitHub account to handle overages.
- *Tip*: GitHub auto names the codespace but you can rename the codespace so that it is easier to identify later.
-- **Local IDE**:
- - Ensure you have [Git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git)
- - Ensure you have [Docker](https://docs.docker.com/get-docker/)
- - Ensure you have [VSCode](https://code.visualstudio.com/)
- - Install the [Dev Containers extension](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers)
+- Local Development Environment:
+ - Note: On Windows, running the tutorial notebook with the example data in a Dev Container is not currently possible due to a s3fs mounting issue. Please use the `Cloud-based Development Environment` option above.
+ - Install [Git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git)
+ - Install [Docker](https://docs.docker.com/get-docker/)
+ - Install [VSCode](https://code.visualstudio.com/)
+ - Install the VSCode [Dev Containers extension](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers)
- `git clone` the codebase repository and open it in VSCode
- Use the `Dev Containers extension` to `Reopen in Container` (More info in the `Getting started` included with the extension)
diff --git a/notebooks/demo_prepare.ipynb b/notebooks/demo_prepare.ipynb
index 6f63f1b..957af07 100644
--- a/notebooks/demo_prepare.ipynb
+++ b/notebooks/demo_prepare.ipynb
@@ -140,7 +140,7 @@
" insertion_number=1,\n",
" paramset_idx=1,\n",
" task_mode='load', # load or trigger\n",
- " clustering_output_dir=\"subject5/session1/probe_1/ks2.1_01\"\n",
+ " clustering_output_dir=\"subject5/session1/probe_1/kilosort2-5_1\"\n",
" )\n",
")\n",
"\n",
@@ -204,13 +204,6 @@
"\n",
"# drop_databases(databases=['analysis', 'trial', 'event', 'ephys_report', 'ephys', 'probe', 'session', 'subject', 'project', 'lab'])\n"
]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": []
}
],
"metadata": {
@@ -229,7 +222,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.9.2"
+ "version": "3.9.16"
},
"orig_nbformat": 4,
"vscode": {
diff --git a/notebooks/quality_metrics.ipynb b/notebooks/quality_metrics.ipynb
new file mode 100644
index 0000000..9a1ce44
--- /dev/null
+++ b/notebooks/quality_metrics.ipynb
@@ -0,0 +1,1046 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Quality Metrics\n",
+ "\n",
+ "Visualize the spike sorting quality metrics that are generated from the [Kilosort](https://github.com/MouseLand/Kilosort) results with the [ecephys_spike_sorting](https://github.com/datajoint/ecephys_spike_sorting) package (i.e. `metrics.csv`) and stored in the DataJoint pipeline (i.e. `element-array-ephys`).\n",
+ "\n",
+ "If you are new to using this DataJoint pipeline for analyzing electrophysiology recordings from Neuropixels probes, please see the [tutorial](./tutorial.ipynb) notebook for an in-depth explanation to set up and run the workflow.\n",
+ "\n",
+ "This notebook can run in a [GitHub Codespace](https://github.com/datajoint/workflow-array-ephys#interactive-tutorial), and requires the example data to be populated into the database using the [demo_prepare](./demo_prepare.ipynb) notebook."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "[2023-04-21 00:44:04,859][WARNING]: lab.Project and related tables will be removed in a future version of Element Lab. Please use the project schema.\n",
+ "[2023-04-21 00:44:04,862][INFO]: Connecting root@fakeservices.datajoint.io:3306\n",
+ "[2023-04-21 00:44:04,868][INFO]: Connected root@fakeservices.datajoint.io:3306\n"
+ ]
+ }
+ ],
+ "source": [
+ "import matplotlib.pyplot as plt\n",
+ "import numpy as np\n",
+ "from workflow_array_ephys.pipeline import ephys"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Populate the `QualityMetrics` table"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "ephys.QualityMetrics.populate()"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Unit quality metrics\n",
+ "\n",
+ "| Metric | Description |\n",
+ "| --- | --- |\n",
+ "| Firing rate (Hz) | Total number of spikes per second. |\n",
+ "| Signal-to-noise ratio | Ratio of the maximum amplitude of the mean spike waveform to the standard deviation of the background noise on a given channel. |\n",
+ "| Presence ratio | Proportion of time during a session that a unit is spiking, ranging from 0 to 0.99. |\n",
+ "| Interspike interval (ISI) violation | Rate of inter-spike-interval (ISI) refractory period violations. |\n",
+ "| Number violation | Total number of ISI violations. |\n",
+ "| Amplitude cut-off | False negative rate of a unit measured by the degree to which its distribution of spike amplitudes is truncated, indicating the fraction of missing spikes. An amplitude cutoff of 0.1 indicates approximately 10% missing spikes. |\n",
+ "| Isolation distance | A metric that uses the principal components (PCs) of a unit's waveforms, which are projected into a lower-dimensional PC space after spike sorting. This quantifies how well-isolated the unit is from other potential clusters. |\n",
+ "| L-ratio | A metric to quantify the distribution of spike distances from a cluster. A low L-ratio indicates that there is a relatively low number of non-member spikes around the target cluster. |\n",
+ "| D-prime | A metric calculated from waveform principal components using linear discriminant analysis. This measures the separability of one unit's PC cluster from all the others, with a higher d-prime value indicating better isolation of the unit. |\n",
+ "| Nearest-neighbors hit rate | The proportion of its nearest neighbors that belong to the same given cluster based on its first principal components. |\n",
+ "| Nearest-neighbors miss rate | The proportion of its nearest neighbors that do not belong to the same given cluster based on its first principal components. |\n",
+ "| Silhouette score | The ratio between cohesiveness of a cluster (distance between member spikes) and its separation from other clusters (distance to non-member spikes). |\n",
+ "| Max drift | The maximum shift in spike location, calculated as the center of mass of the energy of the first principal component score. |\n",
+ "| Cumulative drift | The cumulative change in spike position throughout a recording. |\n",
+ "\n",
+ "For further details of the quality metrics, please see:\n",
+ "- [Allen Institute Documentation](https://allensdk.readthedocs.io/en/latest/_static/examples/nb/ecephys_quality_metrics.html)\n",
+ "\n",
+ "- [Buccino et al., eLife 2020](https://elifesciences.org/articles/61834)\n",
+ "\n",
+ "We'll grab an example key for demonstration."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "\n",
+ " \n",
+ " \n",
+ " \n",
+ " Cluster metrics for a particular unit \n",
+ " \n",
+ "
\n",
+ " \n",
+ " subject5 \n",
+ "2023-04-21 00:40:32 \n",
+ "1 \n",
+ "1 \n",
+ "1 \n",
+ "0 \n",
+ "6.71766 \n",
+ "0.392532 \n",
+ "0.98 \n",
+ "1.0 \n",
+ "105 \n",
+ "0.126571 \n",
+ "nan \n",
+ "0.0 \n",
+ "nan \n",
+ "nan \n",
+ "nan \n",
+ "0.0256577 \n",
+ "0.0 \n",
+ "0.0 \n",
+ "1.13881 subject5 \n",
+ "2023-04-21 00:40:32 \n",
+ "1 \n",
+ "1 \n",
+ "1 \n",
+ "1 \n",
+ "4.64183 \n",
+ "0.391947 \n",
+ "0.99 \n",
+ "1.0 \n",
+ "24 \n",
+ "0.5 \n",
+ "nan \n",
+ "0.0 \n",
+ "nan \n",
+ "nan \n",
+ "nan \n",
+ "0.126882 \n",
+ "7.77 \n",
+ "20.13 \n",
+ "1.08565 subject5 \n",
+ "2023-04-21 00:40:32 \n",
+ "1 \n",
+ "1 \n",
+ "1 \n",
+ "2 \n",
+ "0.171264 \n",
+ "1.91799 \n",
+ "0.39 \n",
+ "0.0 \n",
+ "0 \n",
+ "0.5 \n",
+ "41.5366 \n",
+ "0.01055 \n",
+ "2.00497 \n",
+ "0.109195 \n",
+ "0.00941423 \n",
+ "0.0382387 \n",
+ "17.01 \n",
+ "0.0 \n",
+ "0.0 subject5 \n",
+ "2023-04-21 00:40:32 \n",
+ "1 \n",
+ "1 \n",
+ "1 \n",
+ "3 \n",
+ "8.46869 \n",
+ "0.454273 \n",
+ "0.98 \n",
+ "1.0 \n",
+ "53 \n",
+ "0.5 \n",
+ "48.976 \n",
+ "0.369185 \n",
+ "2.74545 \n",
+ "0.987333 \n",
+ "1.0 \n",
+ "0.0382387 \n",
+ "3.9 \n",
+ "9.13 \n",
+ "0.670886 subject5 \n",
+ "2023-04-21 00:40:32 \n",
+ "1 \n",
+ "1 \n",
+ "1 \n",
+ "4 \n",
+ "0.413395 \n",
+ "2.56485 \n",
+ "0.73 \n",
+ "0.0 \n",
+ "0 \n",
+ "0.5 \n",
+ "360.148 \n",
+ "0.00934174 \n",
+ "4.40108 \n",
+ "0.966667 \n",
+ "0.135897 \n",
+ "0.0571513 \n",
+ "6.35 \n",
+ "13.91 \n",
+ "0.0 subject5 \n",
+ "2023-04-21 00:40:32 \n",
+ "1 \n",
+ "1 \n",
+ "1 \n",
+ "5 \n",
+ "0.218509 \n",
+ "2.4666 \n",
+ "0.48 \n",
+ "0.0 \n",
+ "0 \n",
+ "0.5 \n",
+ "162.35 \n",
+ "3.64917e-05 \n",
+ "6.75158 \n",
+ "0.981982 \n",
+ "0.0102881 \n",
+ "0.10466 \n",
+ "6.24 \n",
+ "9.81 \n",
+ "0.0 subject5 \n",
+ "2023-04-21 00:40:32 \n",
+ "1 \n",
+ "1 \n",
+ "1 \n",
+ "6 \n",
+ "0.100396 \n",
+ "3.87529 \n",
+ "0.31 \n",
+ "0.0 \n",
+ "0 \n",
+ "0.5 \n",
+ "7.98228e+16 \n",
+ "nan \n",
+ "4.6297 \n",
+ "0.843137 \n",
+ "0.026749 \n",
+ "0.0571513 \n",
+ "0.0 \n",
+ "0.0 \n",
+ "0.0 subject5 \n",
+ "2023-04-21 00:40:32 \n",
+ "1 \n",
+ "1 \n",
+ "1 \n",
+ "7 \n",
+ "0.064962 \n",
+ "1.85867 \n",
+ "0.21 \n",
+ "0.0 \n",
+ "0 \n",
+ "0.279006 \n",
+ "4.61058e+16 \n",
+ "nan \n",
+ "4.39722 \n",
+ "0.727273 \n",
+ "0.0147849 \n",
+ "nan \n",
+ "nan \n",
+ "0.0 \n",
+ "1.0 subject5 \n",
+ "2023-04-21 00:40:32 \n",
+ "1 \n",
+ "1 \n",
+ "1 \n",
+ "8 \n",
+ "15.1657 \n",
+ "0.445549 \n",
+ "0.99 \n",
+ "1.0 \n",
+ "412 \n",
+ "0.5 \n",
+ "nan \n",
+ "0.0 \n",
+ "nan \n",
+ "nan \n",
+ "nan \n",
+ "0.131529 \n",
+ "0.0 \n",
+ "0.0 \n",
+ "1.11235 subject5 \n",
+ "2023-04-21 00:40:32 \n",
+ "1 \n",
+ "1 \n",
+ "1 \n",
+ "9 \n",
+ "1.27267 \n",
+ "0.255138 \n",
+ "0.88 \n",
+ "1.0 \n",
+ "2 \n",
+ "0.158831 \n",
+ "185.314 \n",
+ "0.0121516 \n",
+ "5.55476 \n",
+ "0.996906 \n",
+ "0.0454545 \n",
+ "0.152277 \n",
+ "7.8 \n",
+ "17.6 \n",
+ "0.201681 subject5 \n",
+ "2023-04-21 00:40:32 \n",
+ "1 \n",
+ "1 \n",
+ "1 \n",
+ "10 \n",
+ "0.0856318 \n",
+ "1.71077 \n",
+ "0.25 \n",
+ "0.0 \n",
+ "0 \n",
+ "0.5 \n",
+ "1.32036e+16 \n",
+ "nan \n",
+ "5.22485 \n",
+ "0.977012 \n",
+ "0.000674764 \n",
+ "0.0705441 \n",
+ "nan \n",
+ "0.0 \n",
+ "1.0 subject5 \n",
+ "2023-04-21 00:40:32 \n",
+ "1 \n",
+ "1 \n",
+ "1 \n",
+ "11 \n",
+ "0.0767733 \n",
+ "1.25363 \n",
+ "0.22 \n",
+ "0.0 \n",
+ "0 \n",
+ "0.5 \n",
+ "8456290000000000.0 \n",
+ "nan \n",
+ "6.3283 \n",
+ "0.858974 \n",
+ "0.000118455 \n",
+ "nan \n",
+ "nan \n",
+ "0.0 \n",
+ "1.0 \n",
+ "
\n",
+ "
...
\n",
+ "
Total: 227
"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "fig, axes = plt.subplots(4, 4, figsize=(12, 9))\n",
+ "axes = axes.flatten()\n",
+ "plt.suptitle(f\"Cluster Quality Metrics for {key}\", y=.99, fontsize=12)\n",
+ "\n",
+ "# Firing Rates\n",
+ "data = np.log10(query.fetch(\"firing_rate\"))\n",
+ "bins = np.linspace(-3,2,100)\n",
+ "plot_metric(axes[0], data, bins, title=\"Firing Rate (Hz) (log$_{10}$)\")\n",
+ "axes[0].set_ylabel(\"Count\")\n",
+ "\n",
+ "# Signal-to-Noise Ratio\n",
+ "data = query.fetch(\"snr\")\n",
+ "bins = np.linspace(0, 10, 100)\n",
+ "plot_metric(axes[1], data, bins, title=\"Signal-to-Noise Ratio\")\n",
+ "\n",
+ "# Presence Ratio\n",
+ "data = query.fetch(\"presence_ratio\")\n",
+ "bins = np.linspace(0, 1, 100)\n",
+ "plot_metric(axes[2], data, bins, title=\"Presence Ratio\")\n",
+ "\n",
+ "# ISI Violation\n",
+ "data = query.fetch(\"isi_violation\")\n",
+ "bins = np.linspace(0, 1, 100)\n",
+ "plot_metric(axes[3], data, bins, title=\"ISI Violation\")\n",
+ "\n",
+ "# Number Violation\n",
+ "data = query.fetch(\"number_violation\")\n",
+ "bins = np.linspace(0, 1000, 100)\n",
+ "plot_metric(axes[4], data, bins, title=\"Number Violation\")\n",
+ "axes[4].set_ylabel(\"Count\")\n",
+ "\n",
+ "# Amplitude Cutoff\n",
+ "data = query.fetch(\"amplitude_cutoff\")\n",
+ "bins = np.linspace(0, 0.5, 100)\n",
+ "plot_metric(axes[5], data, bins, title=\"Amplitude Cutoff\")\n",
+ "\n",
+ "# Isolation Distance\n",
+ "data = query.fetch(\"isolation_distance\")\n",
+ "bins = np.linspace(0, 170, 50)\n",
+ "plot_metric(axes[6], data, bins, title=\"Isolation Distance\")\n",
+ "\n",
+ "# L-Ratio\n",
+ "data = query.fetch(\"l_ratio\")\n",
+ "bins = np.linspace(0, 1, 100)\n",
+ "plot_metric(axes[7], data, bins, title=\"L-Ratio\")\n",
+ "\n",
+ "# d-Prime\n",
+ "data = query.fetch(\"d_prime\")\n",
+ "bins = np.linspace(0, 15, 50)\n",
+ "plot_metric(axes[8], data, bins, title=\"d-Prime\")\n",
+ "axes[8].set_ylabel(\"Count\")\n",
+ "\n",
+ "# Nearest-Neighbors Hit Rate\n",
+ "data = query.fetch(\"nn_hit_rate\")\n",
+ "bins = np.linspace(0, 1, 100)\n",
+ "plot_metric(axes[9], data, bins, title=\"Nearest-Neighbors Hit Rate\")\n",
+ "\n",
+ "# Nearest-Neighbors Miss Rate\n",
+ "data = query.fetch(\"nn_miss_rate\")\n",
+ "bins = np.linspace(0, 1, 100)\n",
+ "plot_metric(axes[10], data, bins, title=\"Nearest-Neighbors Miss Rate\")\n",
+ "\n",
+ "# Silhouette Score\n",
+ "data = query.fetch(\"silhouette_score\")\n",
+ "bins = np.linspace(0, 1, 100)\n",
+ "plot_metric(axes[11], data, bins, title=\"Silhouette Score\")\n",
+ "\n",
+ "# Max Drift\n",
+ "data = query.fetch(\"max_drift\")\n",
+ "bins = np.linspace(0, 100, 100)\n",
+ "plot_metric(axes[12], data, bins, title=\"Max Drift\")\n",
+ "axes[12].set_ylabel(\"Count\")\n",
+ "\n",
+ "# Cumulative Drift\n",
+ "data = query.fetch(\"cumulative_drift\")\n",
+ "bins = np.linspace(0, 100, 100)\n",
+ "plot_metric(axes[13], data, bins, title=\"Cumulative Drift\")\n",
+ "\n",
+ "[ax.remove() for ax in axes[14:]]\n",
+ "plt.tight_layout()"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Waveform quality metrics\n",
+ "\n",
+ "| Metric | Description |\n",
+ "| -- | -- |\n",
+ "| Amplitude (μV) | Absolute difference between the waveform peak and trough. |\n",
+ "| Duration (ms) | Time interval between the waveform peak and trough. |\n",
+ "| Peak-to-Trough (PT) Ratio | Absolute amplitude of the peak divided by the absolute amplitude of the trough relative to 0. |\n",
+ "| Repolarization Slope | Slope of the fitted regression line to the first 30μs from trough to peak. |\n",
+ "| Recovery Slope | Slope of the fitted regression line to the first 30μs from peak to tail. |\n",
+ "| Spread (μm) | Spatial extent of channels where the waveform amplitude exceeds 12% of the peak amplitude. |\n",
+ "| Velocity Above (s/m) | Inverse velocity of waveform propagation from the soma toward the top of the probe. |\n",
+ "| Velocity Below (s/m) | Inverse velocity of waveform propagation from the soma toward the bottom of the probe. |"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 7,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "\n",
+ " \n",
+ " \n",
+ " \n",
+ " Waveform metrics for a particular unit \n",
+ " \n",
+ "
\n",
+ " \n",
+ " subject5 \n",
+ "2023-04-21 00:40:32 \n",
+ "1 \n",
+ "1 \n",
+ "1 \n",
+ "0 \n",
+ "120.402 \n",
+ "1.08509 \n",
+ "nan \n",
+ "0.74622 \n",
+ "0.0388668 \n",
+ "-0.3638 \n",
+ "280.0 \n",
+ "0.0735822 \n",
+ "-0.0114461 subject5 \n",
+ "2023-04-21 00:40:32 \n",
+ "1 \n",
+ "1 \n",
+ "1 \n",
+ "1 \n",
+ "151.819 \n",
+ "1.05762 \n",
+ "nan \n",
+ "0.657574 \n",
+ "0.0700459 \n",
+ "-0.0806154 \n",
+ "300.0 \n",
+ "-0.0576308 \n",
+ "-0.137353 subject5 \n",
+ "2023-04-21 00:40:32 \n",
+ "1 \n",
+ "1 \n",
+ "1 \n",
+ "2 \n",
+ "142.161 \n",
+ "0.59062 \n",
+ "nan \n",
+ "0.711684 \n",
+ "0.175466 \n",
+ "-0.119525 \n",
+ "300.0 \n",
+ "0.081758 \n",
+ "-0.0572306 subject5 \n",
+ "2023-04-21 00:40:32 \n",
+ "1 \n",
+ "1 \n",
+ "1 \n",
+ "3 \n",
+ "144.028 \n",
+ "1.03015 \n",
+ "nan \n",
+ "0.741737 \n",
+ "0.0801046 \n",
+ "-0.0811708 \n",
+ "300.0 \n",
+ "-0.138989 \n",
+ "0.0711516 subject5 \n",
+ "2023-04-21 00:40:32 \n",
+ "1 \n",
+ "1 \n",
+ "1 \n",
+ "4 \n",
+ "87.3382 \n",
+ "0.563149 \n",
+ "nan \n",
+ "0.869518 \n",
+ "0.189142 \n",
+ "-0.0258353 \n",
+ "300.0 \n",
+ "0.595198 \n",
+ "-0.582626 subject5 \n",
+ "2023-04-21 00:40:32 \n",
+ "1 \n",
+ "1 \n",
+ "1 \n",
+ "5 \n",
+ "80.6693 \n",
+ "0.508208 \n",
+ "nan \n",
+ "0.864687 \n",
+ "0.24975 \n",
+ "-0.0206796 \n",
+ "300.0 \n",
+ "-1.13317 \n",
+ "1.08081 subject5 \n",
+ "2023-04-21 00:40:32 \n",
+ "1 \n",
+ "1 \n",
+ "1 \n",
+ "6 \n",
+ "86.7877 \n",
+ "0.576884 \n",
+ "nan \n",
+ "0.834892 \n",
+ "0.256677 \n",
+ "-0.0614186 \n",
+ "300.0 \n",
+ "2.34645 \n",
+ "0.0654064 subject5 \n",
+ "2023-04-21 00:40:32 \n",
+ "1 \n",
+ "1 \n",
+ "1 \n",
+ "7 \n",
+ "80.8594 \n",
+ "0.453266 \n",
+ "nan \n",
+ "0.840421 \n",
+ "0.212582 \n",
+ "-0.0467612 \n",
+ "300.0 \n",
+ "2.08319 \n",
+ "0.228922 subject5 \n",
+ "2023-04-21 00:40:32 \n",
+ "1 \n",
+ "1 \n",
+ "1 \n",
+ "8 \n",
+ "100.313 \n",
+ "1.05762 \n",
+ "nan \n",
+ "0.800589 \n",
+ "0.0222054 \n",
+ "-0.518803 \n",
+ "300.0 \n",
+ "0.173843 \n",
+ "-0.0654064 subject5 \n",
+ "2023-04-21 00:40:32 \n",
+ "1 \n",
+ "1 \n",
+ "1 \n",
+ "9 \n",
+ "63.2377 \n",
+ "0.576884 \n",
+ "nan \n",
+ "0.909614 \n",
+ "0.152643 \n",
+ "-0.0177585 \n",
+ "280.0 \n",
+ "1.0138 \n",
+ "0.147164 subject5 \n",
+ "2023-04-21 00:40:32 \n",
+ "1 \n",
+ "1 \n",
+ "1 \n",
+ "10 \n",
+ "124.865 \n",
+ "0.480737 \n",
+ "nan \n",
+ "0.733803 \n",
+ "0.26627 \n",
+ "-0.0227592 \n",
+ "300.0 \n",
+ "0.869905 \n",
+ "0.220747 subject5 \n",
+ "2023-04-21 00:40:32 \n",
+ "1 \n",
+ "1 \n",
+ "1 \n",
+ "11 \n",
+ "56.4303 \n",
+ "0.329648 \n",
+ "nan \n",
+ "0.885421 \n",
+ "0.202103 \n",
+ "-0.0599684 \n",
+ "300.0 \n",
+ "3.85898 \n",
+ "-2.00307 \n",
+ "
\n",
+ "
...
\n",
+ "
Total: 227
"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "fig, axes = plt.subplots(2, 4, figsize=(12, 5))\n",
+ "axes = axes.flatten()\n",
+ "plt.suptitle(f\"Waveform Quality Metrics for {key}\", y=.99, fontsize=12)\n",
+ "\n",
+ "# Amplitude\n",
+ "data = np.log10(query.fetch(\"amplitude\"))\n",
+ "bins = np.linspace(0, 3, 100)\n",
+ "plot_metric(axes[0], data, bins, title=\"Amplitude (μV) (log$_{10}$)\")\n",
+ "axes[0].set_ylabel(\"Count\")\n",
+ "\n",
+ "# Duration\n",
+ "data = query.fetch(\"duration\")\n",
+ "bins = np.linspace(0, 3, 100)\n",
+ "plot_metric(axes[1], data, bins, title=\"Duration (ms)\")\n",
+ "\n",
+ "# Peak-to-Trough Ratio\n",
+ "data = query.fetch(\"pt_ratio\")\n",
+ "bins = np.linspace(0, 1, 100)\n",
+ "plot_metric(axes[2], data, bins, title=\"Peak-to-Trough Ratio\")\n",
+ "\n",
+ "# Repolarization Slope\n",
+ "data = query.fetch(\"repolarization_slope\")\n",
+ "bins = np.linspace(-0.1, 2, 100)\n",
+ "plot_metric(axes[3], data, bins, title=\"Repolarization Slope\")\n",
+ "\n",
+ "# Recovery Slope\n",
+ "data = query.fetch(\"recovery_slope\")\n",
+ "bins = np.linspace(-0.5, 0.5, 100)\n",
+ "plot_metric(axes[4], data, bins, title=\"Recovery Slope\")\n",
+ "axes[4].set_ylabel(\"Count\")\n",
+ "\n",
+ "# Spread\n",
+ "data = np.log10(query.fetch(\"spread\"))\n",
+ "bins = np.linspace(0, 3, 100)\n",
+ "plot_metric(axes[5], data, bins, title=\"Spread (μm) (log$_{10}$)\")\n",
+ "\n",
+ "# Velocity Above\n",
+ "data = query.fetch(\"velocity_above\")\n",
+ "bins = np.linspace(-5, 15, 100)\n",
+ "plot_metric(axes[6], data, bins, title=\"Velocity Above (s/m)\")\n",
+ "\n",
+ "# Velocity Below\n",
+ "data = query.fetch(\"velocity_below\")\n",
+ "bins = np.linspace(-10, 10, 100)\n",
+ "plot_metric(axes[7], data, bins, title=\"Velocity Below (s/m)\")\n",
+ "plt.tight_layout()"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "python3p10",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.9.16"
+ },
+ "orig_nbformat": 4,
+ "vscode": {
+ "interpreter": {
+ "hash": "ff52d424e56dd643d8b2ec122f40a2e279e94970100b4e6430cb9025a65ba4cf"
+ }
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/requirements.txt b/requirements.txt
index b80b763..d1a7ae3 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,6 +1,6 @@
datajoint>=0.13.0
element-animal>=0.1.5
-element-array-ephys>=0.2.4
+element-array-ephys>=0.2.7
element-electrode-localization>=0.1.2
element-event>=0.1.2
element-interface>=0.5.0
diff --git a/tests/conftest.py b/tests/conftest.py
index 0d95ea5..a1bea99 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -336,8 +336,8 @@ def ingest_data(setup, pipeline, test_data):
def testdata_paths():
"""Paths for test data 'subjectX/sessionY/probeZ/etc'"""
return {
- "npx3A-p1-ks": "subject5/session1/probe_1/ks2.1_01",
- "npx3A-p2-ks": "subject5/session1/probe_2/ks2.1_01",
+ "npx3A-p1-ks": "subject5/session1/probe_1/kilosort2-5_1",
+ "npx3A-p2-ks": "subject5/session1/probe_2/kilosort2-5_1",
"oe_npx3B-ks": "subject4/experiment1/recording1/continuous/"
+ "Neuropix-PXI-100.0/ks",
"sglx_npx3A-p1": "subject5/session1/probe_1",
diff --git a/tests/test_populate.py b/tests/test_populate.py
index 40dea92..931dae7 100644
--- a/tests/test_populate.py
+++ b/tests/test_populate.py
@@ -314,7 +314,6 @@ def test_build_electrode_layouts(pipeline):
probe = pipeline["probe"]
for probe_type, config in probe_configs.items():
-
test_df = pd.DataFrame(probe.build_electrode_layouts(probe_type, **config))
test_arr = np.array(test_df.drop(columns=["probe_type"]), dtype=np.int16)
@@ -329,6 +328,55 @@ def test_build_electrode_layouts(pipeline):
), f"probe type '{probe_type}' electrode layout does not match"
+def test_quality_metrics_populate(pipeline):
+ """
+ Populate the ephys.QualityMetrics table and compare values with the `metrics.csv` file.
+ Run the `demo_prepare.ipynb` notebook, prior to running this test.
+ """
+ ephys = pipeline["ephys"]
+ key = {"subject": "subject5", "insertion_number": 1}
+ ephys.QualityMetrics.populate(key)
+
+ rename_dict = {
+ "isi_viol": "isi_violation",
+ "num_viol": "number_violation",
+ "contam_rate": "contamination_rate",
+ }
+
+ cluster_df = (ephys.QualityMetrics.Cluster & key).fetch(
+ format="frame", order_by="unit"
+ )
+ waveform_df = (ephys.QualityMetrics.Waveform & key).fetch(
+ format="frame", order_by="unit"
+ )
+ test_df = pd.concat([cluster_df, waveform_df], axis=1).reset_index()
+
+ metrics_df = pd.read_csv(
+ "/workspaces/workflow-array-ephys/example_data/processed/subject5/session1/probe_1/kilosort2-5_1/metrics.csv"
+ )
+ metrics_df.rename(columns=rename_dict, inplace=True)
+ metrics_df.columns = metrics_df.columns.str.lower()
+
+ for col_name in metrics_df:
+ if (
+ "cluster_id" in col_name
+ or "epoch_name" in col_name
+ or "peak_channel" in col_name
+ ):
+ continue
+
+ try:
+ assert np.allclose(
+ metrics_df[col_name].values.astype(float),
+ test_df[col_name].values.astype(float),
+ rtol=1e-03,
+ atol=1e-03,
+ equal_nan=True,
+ ), f"values in '{col_name}' do not match!"
+ except KeyError as e:
+ raise KeyError(f"Attribute {e} does not exist in ephys.QualityMetrics")
+
+
# ---- HELPER FUNCTIONS ----
diff --git a/workflow_array_ephys/version.py b/workflow_array_ephys/version.py
index 6a2f607..5b67f9e 100644
--- a/workflow_array_ephys/version.py
+++ b/workflow_array_ephys/version.py
@@ -2,4 +2,4 @@
Package metadata
Update the Docker image tag in `docker-compose.yaml` to match
"""
-__version__ = "0.3.1"
+__version__ = "0.3.2"