diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 0000000..a50f210 --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,47 @@ +FROM python:3.9-slim@sha256:5f0192a4f58a6ce99f732fe05e3b3d00f12ae62e183886bca3ebe3d202686c7f + +ENV PATH /usr/local/bin:$PATH +ENV PYTHON_VERSION 3.9.17 + +RUN \ + adduser --system --disabled-password --shell /bin/bash vscode && \ + # install docker + apt-get update && \ + apt-get install ca-certificates curl gnupg lsb-release -y && \ + mkdir -m 0755 -p /etc/apt/keyrings && \ + curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg && \ + echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null && \ + apt-get update && \ + apt-get install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin -y && \ + usermod -aG docker vscode && \ + apt-get clean + +RUN \ + # dev setup + apt update && \ + apt-get install sudo git bash-completion graphviz default-mysql-client s3fs procps -y && \ + usermod -aG sudo vscode && \ + echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers && \ + pip install --no-cache-dir --upgrade black pip nbconvert && \ + echo '. /etc/bash_completion' >> /home/vscode/.bashrc && \ + echo 'export PS1="\[\e[32;1m\]\u\[\e[m\]@\[\e[34;1m\]\H\[\e[m\]:\[\e[33;1m\]\w\[\e[m\]$ "' >> /home/vscode/.bashrc && \ + apt-get clean + +COPY ./ /tmp/element-optogenetics/ + +RUN \ + # pipeline dependencies + apt-get install gcc g++ ffmpeg libsm6 libxext6 -y && \ + pip install --no-cache-dir -e /tmp/element-optogenetics[elements,tests] && \ + # clean up + rm -rf /tmp/element-optogenetics && \ + apt-get clean + +ENV DJ_HOST fakeservices.datajoint.io +ENV DJ_USER root +ENV DJ_PASS simple + +ENV DATABASE_PREFIX neuro_ + +USER vscode +CMD bash -c "sudo rm /var/run/docker.pid; sudo dockerd" \ No newline at end of file diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000..ec9b835 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,30 @@ +{ + "name": "Environment + Data", + "dockerComposeFile": "docker-compose.yaml", + "service": "app", + "workspaceFolder": "/workspaces/${localWorkspaceFolderBasename}", + "remoteEnv": { + "LOCAL_WORKSPACE_FOLDER": "${localWorkspaceFolder}" + }, + "onCreateCommand": "pip install -e .", + "postStartCommand": "docker volume prune -f", + "hostRequirements": { + "cpus": 4, + "memory": "8gb", + "storage": "32gb" + }, + "forwardPorts": [ + 3306 + ], + "customizations": { + "settings": { + "python.pythonPath": "/usr/local/bin/python" + }, + "vscode": { + "extensions": [ + "ms-python.python@2023.8.0", + "ms-toolsai.jupyter@2023.3.1201040234" + ] + } + } +} \ No newline at end of file diff --git a/.devcontainer/docker-compose.yaml b/.devcontainer/docker-compose.yaml new file mode 100644 index 0000000..af1047e --- /dev/null +++ b/.devcontainer/docker-compose.yaml @@ -0,0 +1,23 @@ +version: "3" +services: + app: + cpus: 4 + mem_limit: 8g + build: + context: .. + dockerfile: ./.devcontainer/Dockerfile + # image: datajoint/element_optogenetics:latest + extra_hosts: + - fakeservices.datajoint.io:127.0.0.1 + devices: + - /dev/fuse + cap_add: + - SYS_ADMIN + security_opt: + - apparmor:unconfined + volumes: + - ..:/workspaces/element-optogenetics:cached + - docker_data:/var/lib/docker # persist docker images + privileged: true # only because of dind +volumes: + docker_data: diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 0000000..2eedb0e --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,19 @@ +name: Release +on: + workflow_dispatch: +jobs: + make_github_release: + uses: datajoint/.github/.github/workflows/make_github_release.yaml@main + mkdocs_release: + uses: datajoint/.github/.github/workflows/mkdocs_release.yaml@main + permissions: + contents: write + devcontainer-build: + uses: datajoint/.github/.github/workflows/devcontainer-build.yaml@main + devcontainer-publish: + needs: + - devcontainer-build + uses: datajoint/.github/.github/workflows/devcontainer-publish.yaml@main + secrets: + DOCKERHUB_USERNAME: ${{secrets.DOCKERHUB_USERNAME}} + DOCKERHUB_TOKEN: ${{secrets.DOCKERHUB_TOKEN_FOR_ELEMENTS}} \ No newline at end of file diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml new file mode 100644 index 0000000..f29dafe --- /dev/null +++ b/.github/workflows/test.yaml @@ -0,0 +1,34 @@ +name: Test +on: + push: + pull_request: + workflow_dispatch: +jobs: + devcontainer-build: + uses: datajoint/.github/.github/workflows/devcontainer-build.yaml@main + tests: + runs-on: ubuntu-latest + strategy: + matrix: + py_ver: ["3.9", "3.10"] + mysql_ver: ["8.0", "5.7"] + include: + - py_ver: "3.8" + mysql_ver: "5.7" + - py_ver: "3.7" + mysql_ver: "5.7" + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{matrix.py_ver}} + uses: actions/setup-python@v4 + with: + python-version: ${{matrix.py_ver}} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install flake8 "black[jupyter]" + - name: Run style tests + run: | + python_version=${{matrix.py_ver}} + black element_optogenetics --check --verbose --target-version py${python_version//.} + diff --git a/.github/workflows/u24_element_before_release.yaml b/.github/workflows/u24_element_before_release.yaml deleted file mode 100644 index 692cf82..0000000 --- a/.github/workflows/u24_element_before_release.yaml +++ /dev/null @@ -1,17 +0,0 @@ -name: u24_element_before_release -on: - pull_request: - push: - branches: - - '**' - tags-ignore: - - '**' - workflow_dispatch: -jobs: - call_context_check: - uses: dj-sciops/djsciops-cicd/.github/workflows/context_check.yaml@main - call_u24_elements_build_alpine: - uses: dj-sciops/djsciops-cicd/.github/workflows/u24_element_build.yaml@main - with: - py_ver: 3.9 - image: djbase diff --git a/.github/workflows/u24_element_release_call.yaml b/.github/workflows/u24_element_release_call.yaml deleted file mode 100644 index 4324cca..0000000 --- a/.github/workflows/u24_element_release_call.yaml +++ /dev/null @@ -1,28 +0,0 @@ -name: u24_element_release_call -on: - workflow_run: - workflows: ["u24_element_tag_to_release"] - types: - - completed -jobs: - call_context_check: - uses: dj-sciops/djsciops-cicd/.github/workflows/context_check.yaml@main - test_call_u24_elements_release_alpine: - if: >- - github.event.workflow_run.conclusion == 'success' && ( contains(github.event.workflow_run.head_branch, 'test') || (github.event.workflow_run.event == 'pull_request')) - uses: dj-sciops/djsciops-cicd/.github/workflows/u24_element_release.yaml@main - with: - py_ver: 3.9 - twine_repo: testpypi - secrets: - TWINE_USERNAME: ${{secrets.TWINE_TEST_USERNAME}} - TWINE_PASSWORD: ${{secrets.TWINE_TEST_PASSWORD}} - call_u24_elements_release_alpine: - if: >- - github.event.workflow_run.conclusion == 'success' && github.repository_owner == 'datajoint' && !contains(github.event.workflow_run.head_branch, 'test') - uses: dj-sciops/djsciops-cicd/.github/workflows/u24_element_release.yaml@main - with: - py_ver: 3.9 - secrets: - TWINE_USERNAME: ${{secrets.TWINE_USERNAME}} - TWINE_PASSWORD: ${{secrets.TWINE_PASSWORD}} diff --git a/.github/workflows/u24_element_tag_to_release.yaml b/.github/workflows/u24_element_tag_to_release.yaml deleted file mode 100644 index 57334e9..0000000 --- a/.github/workflows/u24_element_tag_to_release.yaml +++ /dev/null @@ -1,14 +0,0 @@ -name: u24_element_tag_to_release -on: - push: - tags: - - '*.*.*' - - 'test*.*.*' -jobs: - call_context_check: - uses: dj-sciops/djsciops-cicd/.github/workflows/context_check.yaml@main - call_u24_elements_build_alpine: - uses: dj-sciops/djsciops-cicd/.github/workflows/u24_element_build.yaml@main - with: - py_ver: 3.9 - image: djbase diff --git a/.gitignore b/.gitignore index 9e92e15..3f5c0d8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,15 @@ +# User data +.DS_Store + # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] *$py.class -# Distribution, packaging, PyInstaller +# C extensions +*.so + +# Distribution / packaging .Python env/ build/ @@ -21,11 +27,17 @@ wheels/ *.egg-info/ .installed.cfg *.egg +.idea/ + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. *.manifest *.spec + +# Installer logs pip-log.txt -pip-delete*.txt -.idea/ +pip-delete-this-directory.txt # Unit test / coverage reports htmlcov/ @@ -37,52 +49,82 @@ nosetests.xml coverage.xml *.cover .hypothesis/ -.pytest_cache/ -# C extension, Translations -*.so +# Translations *.mo *.pot -# editors: vscode, emacs, Mac -.vscode -**/*~ -**/#*# -**/.#* -.DS_Store - -# Django, Flask, Scrapy, Sphinx, mkdocs: -# PyBuilder, Jupyter, SageMath, celery beat +# Django stuff: *.log local_settings.py + +# Flask stuff: instance/ .webassets-cache + +# Scrapy stuff: .scrapy scratchpaper.* + +# Sphinx documentation docs/_build/ -/site + +# PyBuilder target/ + +# Jupyter Notebook .ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file celerybeat-schedule + +# SageMath parsed files *.sage.py -# dotenv, virtualenv, pyenv, mypy +# dotenv ./.env +.env + +# virtualenv .venv venv/ ENV/ -.python-version -.mypy_cache/ -# Spyder/Rope project settings +# Spyder project settings .spyderproject .spyproject + +# Rope project settings .ropeproject -# datajoint, notes, nwb export -dj_local_c*.json +# mkdocs documentation +docs/site +docs/src/tutorials/*ipynb + +# mypy +.mypy_cache/ + +# datajoint +dj_local_conf.json +dj_local_conf_old.json + +# emacs +**/*~ +**/#*# +**/.#* + + +# include +!docs/docker-compose.yaml + +# vscode settings +*.code-workspace + +# exports/notes temp* -temp/* -*nwb -/docs/site -/docs/src/tutorials/*ipynb + +# Codespaces +example_data/ \ No newline at end of file diff --git a/.markdownlint.yaml b/.markdownlint.yaml new file mode 100644 index 0000000..ac52a8a --- /dev/null +++ b/.markdownlint.yaml @@ -0,0 +1,18 @@ +# Markdown Linter configuration for docs +# https://github.com/DavidAnson/markdownlint +# https://github.com/DavidAnson/markdownlint/blob/main/doc/Rules.md +MD009: false # permit trailing spaces +MD007: false # List indenting - permit 4 spaces +MD013: + line_length: "88" # Line length limits + tables: false # disable for tables + headings: false # disable for headings +MD030: false # Number of spaces after a list +MD033: # HTML elements allowed + allowed_elements: + - "br" + - "figure" + - "figcaption" +MD034: false # Permit bare URLs +MD031: false # Spacing w/code blocks. Conflicts with `??? Note` and code tab styling +MD046: false # Spacing w/code blocks. Conflicts with `??? Note` and code tab styling diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2c685f6..0d513df 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -44,7 +44,7 @@ repos: # these are errors that will be ignored by flake8 # https://www.flake8rules.com/rules/{code}.html - - "--ignore=E203,E501,W503,W605" + - "--ignore=E203,E501,W503,W605,E402" # E203 - Colons should not have any space before them. # Needed for list indexing # E501 - Line lengths are recommended to be no greater than 79 characters. @@ -54,3 +54,5 @@ repos: # W605 - a backslash-character pair that is not a valid escape sequence now # generates a DeprecationWarning. This will eventually become a SyntaxError. # Needed because we use \d as an escape sequence + # E402 - Place module level import at the top. + # Needed to prevent circular import error diff --git a/CHANGELOG.md b/CHANGELOG.md index 37a00c8..ee444b4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,14 @@ Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) convention. +## [0.2.0] - 2024-01-17 + ++ Add - devcontainer configuration ++ Add - tutorial notebook for Codespaces ++ Update - `setup.py` ++ Update - GitHub Actions do not release to PyPI ++ Update - drawio diagrams + ## [0.1.3] - 2023-05-12 + Fix - Docs @@ -20,6 +28,7 @@ Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and + Add - Table structure and basic docs (changelog, contribution guidelines, etc.) +[0.2.0]: https://github.com/datajoint/element-optogenetics/releases/tag/0.2.0 [0.1.3]: https://github.com/datajoint/element-optogenetics/releases/tag/0.1.3 [0.1.2]: https://github.com/datajoint/element-optogenetics/releases/tag/0.1.2 [0.1.1]: https://github.com/datajoint/element-optogenetics/releases/tag/0.1.1 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e04d170..2bd0f49 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,5 +1,5 @@ # Contribution Guidelines This project follows the -[DataJoint Contribution Guidelines](https://datajoint.com/docs/community/contribute/). +[DataJoint Contribution Guidelines](https://datajoint.com/docs/about/contribute/). Please reference the link for more full details. diff --git a/LICENSE b/LICENSE index 2f92789..6872305 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2022 DataJoint +Copyright (c) 2024 DataJoint Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.md b/README.md index 8297015..9807c49 100644 --- a/README.md +++ b/README.md @@ -4,28 +4,76 @@ DataJoint Element for managing data from optogenetics experiments. DataJoint Ele collectively standardize and automate data collection and analysis for neuroscience experiments. Each Element is a modular pipeline for data storage and processing with corresponding database tables that can be combined with other Elements to assemble a -fully functional pipeline. +fully functional pipeline. This repository also provides a tutorial +environment and notebooks to learn the pipeline. ## Experiment Flowchart ![flowchart](https://raw.githubusercontent.com/datajoint/element-optogenetics/main/images/flowchart.svg) -## Data Pipeline +## Data Pipeline Diagram ![pipeline](https://raw.githubusercontent.com/datajoint/element-optogenetics/main/images/pipeline.svg) ## Getting Started -+ Install from PyPI ++ Please fork this repository. + ++ Clone the repository to your computer. + +```bash + git clone https://github.com//element-optogenetics.git +``` + ++ Install with `pip`: ```bash - pip install element-optogenetics + pip install -e . ``` - -+ [Interactive tutorial on GitHub Codespaces](https://github.com/datajoint/workflow-optogenetics#interactive-tutorial) + ++ [Interactive tutorial on GitHub + Codespaces](https://github.com/datajoint/element-optogenetics#interactive-tutorial) + [Documentation](https://datajoint.com/docs/elements/element-optogenetics) ## Support -+ If you need help getting started or run into any errors, please contact our team by email at support@datajoint.com. ++ If you need help getting started or run into any errors, please contact our team by + email at support@datajoint.com. + +## Interactive Tutorial + ++ The easiest way to learn about DataJoint Elements is to use the tutorial notebooks within the included interactive environment configured using [Dev Container](https://containers.dev/). + +### Launch Environment + +Here are some options that provide a great experience: + +- (*recommended*) Cloud-based Environment + - Launch using [GitHub Codespaces](https://github.com/features/codespaces) using the `+` option which will `Create codespace on main` in the codebase repository on your fork with default options. For more control, see the `...` where you may create `New with options...`. + - Build time for a codespace is a few minutes. This is done infrequently and cached for convenience. + - Start time for a codespace is less than 1 minute. This will pull the built codespace from cache when you need it. + - *Tip*: Each month, GitHub renews a [free-tier](https://docs.github.com/en/billing/managing-billing-for-github-codespaces/about-billing-for-github-codespaces#monthly-included-storage-and-core-hours-for-personal-accounts) quota of compute and storage. Typically we run into the storage limits before anything else since Codespaces consume storage while stopped. It is best to delete Codespaces when not actively in use and recreate when needed. We'll soon be creating prebuilds to avoid larger build times. Once any portion of your quota is reached, you will need to wait for it to be reset at the end of your cycle or add billing info to your GitHub account to handle overages. + - *Tip*: GitHub auto names the codespace but you can rename the codespace so that it is easier to identify later. + +- Local Environment + > *Note: Access to example data is currently limited to MacOS and Linux due to the s3fs utility. Windows users are recommended to use the above environment.* + - Install [Git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git) + - Install [Docker](https://docs.docker.com/get-docker/) + - Install [VSCode](https://code.visualstudio.com/) + - Install the VSCode [Dev Containers extension](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers) + - `git clone` the codebase repository and open it in VSCode + - Use the `Dev Containers extension` to `Reopen in Container` (More info is in the `Getting started` included with the extension.) + +You will know your environment has finished loading once you either see a terminal open related to `Running postStartCommand` with a final message of `Done` or the `README.md` is opened in `Preview`. + +Once the environment has launched, please run the following command in the terminal: +``` +MYSQL_VER=8.0 docker compose -f docker-compose-db.yaml up --build -d +``` + +### Instructions + +1. We recommend you start by navigating to the `notebooks` directory on the left panel and go through the `tutorial.ipynb` Jupyter notebook. Execute the cells in the notebook to begin your walk through of the tutorial. + +2. Once you are done, see the options available to you in the menu in the bottom-left corner. For example, in Codespace you will have an option to `Stop Current Codespace` but when running Dev Container on your own machine the equivalent option is `Reopen folder locally`. By default, GitHub will also automatically stop the Codespace after 30 minutes of inactivity. Once the Codespace is no longer being used, we recommend deleting the Codespace. diff --git a/docker-compose-db.yaml b/docker-compose-db.yaml new file mode 100644 index 0000000..1d453c8 --- /dev/null +++ b/docker-compose-db.yaml @@ -0,0 +1,15 @@ +# MYSQL_VER=8.0 docker compose -f docker-compose-db.yaml up --build +version: "3" +services: + db: + restart: always + image: datajoint/mysql:${MYSQL_VER} + environment: + - MYSQL_ROOT_PASSWORD=${DJ_PASS} + ports: + - "3306:3306" + healthcheck: + test: [ "CMD", "mysqladmin", "ping", "-h", "localhost" ] + timeout: 15s + retries: 10 + interval: 15s diff --git a/element_optogenetics/optogenetics.py b/element_optogenetics/optogenetics.py index 9b5e555..febc113 100644 --- a/element_optogenetics/optogenetics.py +++ b/element_optogenetics/optogenetics.py @@ -11,7 +11,7 @@ def activate( *, create_schema: bool = True, create_tables: bool = True, - linking_module: str = None + linking_module: str = None, ): """Activate this schema. diff --git a/element_optogenetics/version.py b/element_optogenetics/version.py index 29d8a5d..bc510a2 100644 --- a/element_optogenetics/version.py +++ b/element_optogenetics/version.py @@ -1,2 +1,2 @@ """Package metadata""" -__version__ = "0.1.3" +__version__ = "0.2.0" diff --git a/images/flowchart.drawio b/images/flowchart.drawio index dc4b997..49a5ff8 100644 --- a/images/flowchart.drawio +++ b/images/flowchart.drawio @@ -1,63 +1,63 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/images/flowchart.svg b/images/flowchart.svg index 9a9692e..97322b5 100644 --- a/images/flowchart.svg +++ b/images/flowchart.svg @@ -1,57 +1,4 @@ -
Synchronize data modalities & exploratory analysis
Synchronize data...
Visualize Stimulus
Waveform and
Neurophysiology
Recording
Visualize Stimulus...

 Export & publish

 
 
Export & publish...
Create project &
stimulation protocol

 
Create project &...
Enter metadata
into pipeline
Enter metadata...
Collect session 
recordings
Collect session...
Enter metadata of
 fiber location
Enter metadata of...Viewer does not support full SVG 1.1 \ No newline at end of file + + + +
Synchronize data modalities & exploratory analysis
Synchronize data...
Visualize Stimulus
Waveform and
Neurophysiology
Recording
Visualize Stimulus...

 Export & publish

 
 
Export & publish...
Create project &
stimulation protocol

 
Create project &...
Enter metadata
into pipeline
Enter metadata...
Collect session 
recordings
Collect session...
Enter metadata of
 fiber location
Enter metadata of...Text is not SVG - cannot display \ No newline at end of file diff --git a/notebooks/tutorial.ipynb b/notebooks/tutorial.ipynb new file mode 100644 index 0000000..d832dc6 --- /dev/null +++ b/notebooks/tutorial.ipynb @@ -0,0 +1,2731 @@ +{ + "cells": [ + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# DataJoint Elements for Optogenetics\n", + "\n", + "#### Open-source data pipeline for processing and analyzing optogenetics datasets.\n", + "\n", + "Welcome to the tutorial for the DataJoint Element for optogenetics. This\n", + "tutorial aims to provide a comprehensive understanding of the open-source data pipeline\n", + "created using `element-optogenetics`.\n", + "\n", + "This package is designed to seamlessly ingest and track optogenetics data. By the end of this\n", + "tutorial you will have a clear grasp on setting up and integrating `element-optogenetics`\n", + "into your specific research projects and lab. \n", + "\n", + "![flowchart](../images/flowchart.svg)\n", + "\n", + "### Prerequisites\n", + "\n", + "Please see the [datajoint tutorials GitHub\n", + "repository](https://github.com/datajoint/datajoint-tutorials/tree/main) before\n", + "proceeding. \n", + "\n", + "#### **Tutorial Overview**\n", + "\n", + "+ Setup\n", + "+ *Activate* the DataJoint pipeline.\n", + "+ *Insert* subject, session, surgery metadata.\n", + "+ *Insert* optogenetics recording data.\n", + "+ Query and view data\n", + "\n", + "### **Setup**\n", + "\n", + "This tutorial examines DataJoint tables that track optogenetics data. The goal is to store, track\n", + "and manage all metadata associated with optogenetics experiments, including surgical\n", + "implanation data. \n", + "\n", + "The results of this Element can be combined with **other modalities** to create\n", + "a complete, customizable data pipeline for your specific lab or study. For instance, you\n", + "can combine `element-optogenetics` with `element-array-ephys` and\n", + "`element-event` to characterize the neural activity during specific optogenetic stimulus\n", + "events.\n", + "\n", + "Let's start this tutorial by importing the packages necessary to run the notebook." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import datajoint as dj" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "If the tutorial is run in Codespaces, a private, local database server is created and\n", + "made available for you. This is where we will insert and store our processed results.\n", + "Let's connect to the database server." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[2024-01-17 12:35:14,558][INFO]: Connecting root@fakeservices.datajoint.io:3306\n", + "[2024-01-17 12:35:14,571][INFO]: Connected root@fakeservices.datajoint.io:3306\n" + ] + }, + { + "data": { + "text/plain": [ + "DataJoint connection (connected) root@fakeservices.datajoint.io:3306" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "dj.conn()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### **Activate the DataJoint Pipeline**\n", + "\n", + "This tutorial activates the `optogenetics.py` module from `element-optogenetics`, along\n", + "with upstream dependencies from `element-animal` and `element-session`. Please refer to the\n", + "[`tutorial_pipeline.py`](./tutorial_pipeline.py) for the source code." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[2024-01-17 12:35:14,756][WARNING]: lab.Project and related tables will be removed in a future version of Element Lab. Please use the project schema.\n" + ] + } + ], + "source": [ + "from tutorial_pipeline import lab, subject, surgery, session, optogenetics, Device" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can represent the tables in the `optogenetics` schemas as well as some of the\n", + "upstream dependencies to `session` and `subject` schemas as a diagram." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "image/svg+xml": [ + "\n", + "\n", + "%3\n", + "\n", + "\n", + "\n", + "optogenetics.OptoStimParams\n", + "\n", + "\n", + "optogenetics.OptoStimParams\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "optogenetics.OptoProtocol\n", + "\n", + "\n", + "optogenetics.OptoProtocol\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "optogenetics.OptoStimParams->optogenetics.OptoProtocol\n", + "\n", + "\n", + "\n", + "\n", + "Device\n", + "\n", + "\n", + "Device\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "Device->optogenetics.OptoProtocol\n", + "\n", + "\n", + "\n", + "\n", + "optogenetics.OptoWaveform\n", + "\n", + "\n", + "optogenetics.OptoWaveform\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "optogenetics.OptoWaveform->optogenetics.OptoStimParams\n", + "\n", + "\n", + "\n", + "\n", + "optogenetics.OptoWaveform.Sine\n", + "\n", + "\n", + "optogenetics.OptoWaveform.Sine\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "optogenetics.OptoWaveform->optogenetics.OptoWaveform.Sine\n", + "\n", + "\n", + "\n", + "\n", + "optogenetics.OptoWaveform.Ramp\n", + "\n", + "\n", + "optogenetics.OptoWaveform.Ramp\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "optogenetics.OptoWaveform->optogenetics.OptoWaveform.Ramp\n", + "\n", + "\n", + "\n", + "\n", + "optogenetics.OptoWaveform.Square\n", + "\n", + "\n", + "optogenetics.OptoWaveform.Square\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "optogenetics.OptoWaveform->optogenetics.OptoWaveform.Square\n", + "\n", + "\n", + "\n", + "\n", + "optogenetics.OptoEvent\n", + "\n", + "\n", + "optogenetics.OptoEvent\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "surgery.Implantation\n", + "\n", + "\n", + "surgery.Implantation\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "surgery.Implantation->optogenetics.OptoProtocol\n", + "\n", + "\n", + "\n", + "\n", + "session.Session\n", + "\n", + "\n", + "session.Session\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "session.Session->optogenetics.OptoProtocol\n", + "\n", + "\n", + "\n", + "\n", + "optogenetics.OptoWaveformType\n", + "\n", + "\n", + "optogenetics.OptoWaveformType\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "optogenetics.OptoWaveformType->optogenetics.OptoWaveform\n", + "\n", + "\n", + "\n", + "\n", + "subject.Subject\n", + "\n", + "\n", + "subject.Subject\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "subject.Subject->surgery.Implantation\n", + "\n", + "\n", + "\n", + "\n", + "subject.Subject->session.Session\n", + "\n", + "\n", + "\n", + "\n", + "optogenetics.OptoProtocol->optogenetics.OptoEvent\n", + "\n", + "\n", + "\n", + "" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "(\n", + " dj.Diagram(subject.Subject)\n", + " + dj.Diagram(surgery.Implantation)\n", + " + dj.Diagram(session.Session)\n", + " + dj.Diagram(Device)\n", + " + dj.Diagram(optogenetics)\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "As evident from the diagram, this data pipeline encompasses tables associated with\n", + "optogenetic stimulation parameters, protocol, and events. A few tables, such as `subject.Subject` or `session.Session`,\n", + "while important for a complete pipeline, fall outside the scope of the `element-optogenetics`\n", + "tutorial, and will therefore, not be explored extensively here. The primary focus of\n", + "this tutorial will be on the `optogenetics` schemas.\n", + "\n", + "### **Insert subject, surgery, and session metadata**\n", + "\n", + "Let's start with the first table in the schema diagram (i.e. `subject.Subject` table).\n", + "\n", + "To know what data to insert into the table, we can view its dependencies and attributes using the `.describe()` and `.heading` methods." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + " \n", + " \n", + " \n", + "
\n", + " \n", + " \n", + " \n", + "
\n", + "

subject

\n", + " \n", + "
\n", + "

subject_nickname

\n", + " \n", + "
\n", + "

sex

\n", + " \n", + "
\n", + "

subject_birth_date

\n", + " \n", + "
\n", + "

subject_description

\n", + " \n", + "
\n", + " \n", + "

Total: 0

\n", + " " + ], + "text/plain": [ + "*subject subject_nickna sex subject_birth_ subject_descri\n", + "+---------+ +------------+ +-----+ +------------+ +------------+\n", + "\n", + " (Total: 0)" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "subject.Subject()" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "subject : varchar(8) \n", + "---\n", + "subject_nickname=\"\" : varchar(64) \n", + "sex : enum('M','F','U') \n", + "subject_birth_date : date \n", + "subject_description=\"\" : varchar(1024) \n", + "\n" + ] + } + ], + "source": [ + "print(subject.Subject.describe())" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "# \n", + "subject : varchar(8) # \n", + "---\n", + "subject_nickname=\"\" : varchar(64) # \n", + "sex : enum('M','F','U') # \n", + "subject_birth_date : date # \n", + "subject_description=\"\" : varchar(1024) # " + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "subject.Subject.heading" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The cells above show all attributes of the subject table.\n", + "We will insert data into the\n", + "`subject.Subject` table. " + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + " \n", + " \n", + " \n", + "
\n", + " \n", + " \n", + " \n", + "\n", + "\n", + "\n", + "\n", + "
\n", + "

subject

\n", + " \n", + "
\n", + "

subject_nickname

\n", + " \n", + "
\n", + "

sex

\n", + " \n", + "
\n", + "

subject_birth_date

\n", + " \n", + "
\n", + "

subject_description

\n", + " \n", + "
subject1F2020-01-01Optogenetic pilot subject
\n", + " \n", + "

Total: 1

\n", + " " + ], + "text/plain": [ + "*subject subject_nickna sex subject_birth_ subject_descri\n", + "+----------+ +------------+ +-----+ +------------+ +------------+\n", + "subject1 F 2020-01-01 Optogenetic pi\n", + " (Total: 1)" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "subject.Subject.insert1(\n", + " dict(\n", + " subject=\"subject1\",\n", + " sex=\"F\",\n", + " subject_birth_date=\"2020-01-01\",\n", + " subject_description=\"Optogenetic pilot subject\",\n", + " )\n", + ")\n", + "subject.Subject()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's repeat the steps above for the `Session` table and see how the output varies between\n", + "`.describe` and `.heading`." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-> subject.Subject\n", + "session_id : int \n", + "---\n", + "session_datetime=null : datetime \n", + "\n" + ] + } + ], + "source": [ + "print(session.Session.describe())" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "# \n", + "subject : varchar(8) # \n", + "session_id : int # \n", + "---\n", + "session_datetime=null : datetime # " + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "session.Session.heading" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Notice that `describe`, displays the table's structure and highlights its dependencies, such as its reliance on the `Subject` table. These dependencies represent foreign key references, linking data across tables.\n", + "\n", + "On the other hand, `heading` provides an exhaustive list of the table's attributes. This\n", + "list includes both the attributes declared in this table and any inherited from upstream\n", + "tables.\n", + "\n", + "With this understanding, let's move on to insert a session associated with our subject.\n", + "\n", + "We will insert into the `session.Session` table by passing a dictionary to the `insert1` method." + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "session_key = dict(subject=\"subject1\", session_id=0, session_datetime=\"2023-01-01 00:00:00\")" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + " \n", + " \n", + " \n", + "
\n", + " \n", + " \n", + " \n", + "\n", + "\n", + "
\n", + "

subject

\n", + " \n", + "
\n", + "

session_id

\n", + " \n", + "
\n", + "

session_datetime

\n", + " \n", + "
subject102023-01-01 00:00:00
\n", + " \n", + "

Total: 1

\n", + " " + ], + "text/plain": [ + "*subject *session_id session_dateti\n", + "+----------+ +------------+ +------------+\n", + "subject1 0 2023-01-01 00:\n", + " (Total: 1)" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "session.Session.insert1(session_key)\n", + "session.Session()" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "As the Diagram indicates, the `OptoProtocol` table requires an entry in both `Session`\n", + "and `Device` tables. Let's insert into the `Device` table." + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + " \n", + " \n", + " \n", + "
\n", + " \n", + " \n", + " \n", + "\n", + "\n", + "\n", + "\n", + "
\n", + "

device

\n", + " \n", + "
\n", + "

modality

\n", + " \n", + "
\n", + "

description

\n", + " \n", + "
OPTG_4OptogeneticsDoric Pulse Sequence Generator
OPTG_8Optogenetics8 channel pulse sequence device
\n", + " \n", + "

Total: 2

\n", + " " + ], + "text/plain": [ + "*device modality description \n", + "+--------+ +------------+ +------------+\n", + "OPTG_4 Optogenetics Doric Pulse Se\n", + "OPTG_8 Optogenetics 8 channel puls\n", + " (Total: 2)" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "Device.insert1(\n", + " dict(\n", + " device=\"OPTG_8\",\n", + " modality=\"Optogenetics\",\n", + " description=\"8 channel pulse sequence device\",\n", + " )\n", + ")\n", + "Device()" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-> subject.Subject\n", + "implant_date : datetime # surgery date\n", + "-> surgery.ImplantationType\n", + "-> surgery.BrainRegion.proj(target_region=\"region_acronym\")\n", + "-> surgery.Hemisphere.proj(target_hemisphere=\"hemisphere\")\n", + "---\n", + "-> lab.User.proj(surgeon=\"user\")\n", + "implant_comment=\"\" : varchar(1024) # Comments about the implant\n", + "\n" + ] + } + ], + "source": [ + "print(surgery.Implantation.describe())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The `surgery.Implantation` table's attribute includes the `User` table. Let's insert\n", + "into the `User` table." + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + " \n", + " \n", + " Table for storing user information.\n", + "
\n", + " \n", + " \n", + " \n", + "\n", + "\n", + "\n", + "
\n", + "

user

\n", + " username, short identifier\n", + "
\n", + "

user_email

\n", + " \n", + "
\n", + "

user_cellphone

\n", + " \n", + "
\n", + "

user_fullname

\n", + " Full name used to uniquely identify an individual\n", + "
User1
\n", + " \n", + "

Total: 1

\n", + " " + ], + "text/plain": [ + "*user user_email user_cellphone user_fullname \n", + "+-------+ +------------+ +------------+ +------------+\n", + "User1 \n", + " (Total: 1)" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "lab.User.insert1(dict(user=\"User1\"))\n", + "lab.User()" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The `Implantation` table's attributes includes the `CoordinateReference` and `Hemisphere` tables. Let's view the contents of these lookup tables, which have default contents." + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + " \n", + " \n", + " \n", + "
\n", + " \n", + " \n", + " \n", + "
\n", + "

reference

\n", + " \n", + "
bregma
dura
lambda
sagittal_suture
sinus
skull_surface
\n", + " \n", + "

Total: 6

\n", + " " + ], + "text/plain": [ + "*reference \n", + "+------------+\n", + "bregma \n", + "dura \n", + "lambda \n", + "sagittal_sutur\n", + "sinus \n", + "skull_surface \n", + " (Total: 6)" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "surgery.CoordinateReference()" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + " \n", + " \n", + " \n", + "
\n", + " \n", + " \n", + " \n", + "
\n", + "

hemisphere

\n", + " Brain region hemisphere\n", + "
left
middle
right
\n", + " \n", + "

Total: 3

\n", + " " + ], + "text/plain": [ + "*hemisphere \n", + "+------------+\n", + "left \n", + "middle \n", + "right \n", + " (Total: 3)" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "surgery.Hemisphere()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Insert a new entry for the location of the optogenetics probe." + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + " \n", + " \n", + " \n", + "
\n", + " \n", + " \n", + " \n", + "\n", + "
\n", + "

region_acronym

\n", + " Brain region shorthand\n", + "
\n", + "

region_name

\n", + " Brain region full name\n", + "
dHPDorsal Hippocampus
\n", + " \n", + "

Total: 1

\n", + " " + ], + "text/plain": [ + "*region_acrony region_name \n", + "+------------+ +------------+\n", + "dHP Dorsal Hippoca\n", + " (Total: 1)" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "surgery.BrainRegion.insert1(\n", + " dict(region_acronym=\"dHP\", region_name=\"Dorsal Hippocampus\")\n", + ")\n", + "surgery.BrainRegion()" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [], + "source": [ + "surgery.Implantation.insert1(\n", + " dict(\n", + " subject=\"subject1\",\n", + " implant_date=\"2022-04-01 12:13:14\",\n", + " implant_type=\"opto\",\n", + " target_region=\"dHP\",\n", + " target_hemisphere=\"left\",\n", + " surgeon=\"User1\",\n", + " )\n", + ")\n", + "surgery.Implantation.Coordinate.insert1(\n", + " dict(\n", + " subject=\"subject1\",\n", + " implant_date=\"2022-04-01 12:13:14\",\n", + " implant_type=\"opto\",\n", + " target_region=\"dHP\",\n", + " target_hemisphere=\"left\",\n", + " ap=\"-7.9\", # [mm] anterior-posterior distance\n", + " ap_ref=\"bregma\",\n", + " ml=\"-1.8\", # [mm] medial axis distance\n", + " ml_ref=\"bregma\",\n", + " dv=\"5\", # [mm] dorso-ventral axis distance\n", + " dv_ref=\"skull_surface\",\n", + " theta=\"11.5\", # [0, 180] degree rotation about ml-axis relative to z\n", + " phi=\"0\", # [0, 360] degree rotation about dv-axis relative to x\n", + " beta=None, # [-180, 180] degree rotation about shank relative to anterior\n", + " )\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Confirm the inserted information:" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + " \n", + " \n", + " \n", + "
\n", + " \n", + " \n", + " \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
\n", + "

subject

\n", + " \n", + "
\n", + "

implant_date

\n", + " surgery date\n", + "
\n", + "

implant_type

\n", + " Short name for type of implanted device\n", + "
\n", + "

target_region

\n", + " Brain region shorthand\n", + "
\n", + "

target_hemisphere

\n", + " Brain region hemisphere\n", + "
\n", + "

surgeon

\n", + " username, short identifier\n", + "
\n", + "

implant_comment

\n", + " Comments about the implant\n", + "
subject12022-04-01 12:13:14optodHPleftUser1
\n", + " \n", + "

Total: 1

\n", + " " + ], + "text/plain": [ + "*subject *implant_date *implant_type *target_region *target_hemisp surgeon implant_commen\n", + "+----------+ +------------+ +------------+ +------------+ +------------+ +---------+ +------------+\n", + "subject1 2022-04-01 12: opto dHP left User1 \n", + " (Total: 1)" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "surgery.Implantation()" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + " \n", + " \n", + " \n", + "
\n", + " \n", + " \n", + " \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
\n", + "

subject

\n", + " \n", + "
\n", + "

implant_date

\n", + " surgery date\n", + "
\n", + "

implant_type

\n", + " Short name for type of implanted device\n", + "
\n", + "

target_region

\n", + " Brain region shorthand\n", + "
\n", + "

target_hemisphere

\n", + " Brain region hemisphere\n", + "
\n", + "

ap

\n", + " (mm) anterior-posterior; ref is 0\n", + "
\n", + "

ap_ref

\n", + " \n", + "
\n", + "

ml

\n", + " (mm) medial axis; ref is 0\n", + "
\n", + "

ml_ref

\n", + " \n", + "
\n", + "

dv

\n", + " (mm) dorso-ventral axis; ventral negative\n", + "
\n", + "

dv_ref

\n", + " \n", + "
\n", + "

theta

\n", + " (deg) rot about ml-axis [0, 180] wrt z\n", + "
\n", + "

phi

\n", + " (deg) rot about dv-axis [0, 360] wrt x\n", + "
\n", + "

beta

\n", + " (deg) rot about shank [-180, 180] wrt anterior\n", + "
subject12022-04-01 12:13:14optodHPleft-7.9bregma-1.8bregma5.0skull_surface11.50.0nan
\n", + " \n", + "

Total: 1

\n", + " " + ], + "text/plain": [ + "*subject *implant_date *implant_type *target_region *target_hemisp ap ap_ref ml ml_ref dv dv_ref theta phi beta \n", + "+----------+ +------------+ +------------+ +------------+ +------------+ +------+ +--------+ +------+ +--------+ +-----+ +------------+ +-------+ +-----+ +------+\n", + "subject1 2022-04-01 12: opto dHP left -7.9 bregma -1.8 bregma 5.0 skull_surface 11.5 0.0 nan \n", + " (Total: 1)" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "surgery.Implantation.Coordinate()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We'll add information to describe the stimulus, including waveform shape and stimulation parameters." + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [], + "source": [ + "optogenetics.OptoWaveform.insert1(\n", + " dict(\n", + " waveform_name=\"square_10\",\n", + " waveform_type=\"square\",\n", + " waveform_description=\"Square waveform: 10%/90% on/off cycle\",\n", + " )\n", + ")\n", + "\n", + "# Square is one part table of OptoWaveform.\n", + "# For sine and ramp waveforms, see the corresponding tables.\n", + "optogenetics.OptoWaveform.Square.insert1(\n", + " dict(waveform_name=\"square_10\", on_proportion=0.10, off_proportion=0.90)\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + " \n", + " \n", + " Defines a single optical stimulus that repeats.\n", + "
\n", + " \n", + " \n", + " \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
\n", + "

opto_params_id

\n", + " \n", + "
\n", + "

waveform_name

\n", + " \n", + "
\n", + "

wavelength

\n", + " (nm) wavelength of optical stimulation light\n", + "
\n", + "

power

\n", + " (mW) total power from light source\n", + "
\n", + "

light_intensity

\n", + " (mW/mm2) power for given area\n", + "
\n", + "

frequency

\n", + " (Hz) frequency of the waveform\n", + "
\n", + "

duration

\n", + " (ms) duration of each optical stimulus\n", + "
1square_10470None10.201.0241.0
\n", + " \n", + "

Total: 1

\n", + " " + ], + "text/plain": [ + "*opto_params_i waveform_name wavelength power light_intensit frequency duration \n", + "+------------+ +------------+ +------------+ +-------+ +------------+ +-----------+ +----------+\n", + "1 square_10 470 None 10.20 1.0 241.0 \n", + " (Total: 1)" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "optogenetics.OptoStimParams.insert1(\n", + " dict(\n", + " opto_params_id=1,\n", + " waveform_name=\"square_10\",\n", + " wavelength=470,\n", + " light_intensity=10.2,\n", + " frequency=1,\n", + " duration=241,\n", + " )\n", + ")\n", + "optogenetics.OptoStimParams()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### **Insert optogenetics recording data**\n", + "Next, we'll describe the session in which these parameters are used in `OptoProtocol`." + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + " \n", + " \n", + " \n", + "
\n", + " \n", + " \n", + " \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
\n", + "

subject

\n", + " \n", + "
\n", + "

session_id

\n", + " \n", + "
\n", + "

protocol_id

\n", + " \n", + "
\n", + "

opto_params_id

\n", + " \n", + "
\n", + "

implant_date

\n", + " surgery date\n", + "
\n", + "

implant_type

\n", + " Short name for type of implanted device\n", + "
\n", + "

target_region

\n", + " Brain region shorthand\n", + "
\n", + "

target_hemisphere

\n", + " Brain region hemisphere\n", + "
\n", + "

device

\n", + " \n", + "
\n", + "

protocol_description

\n", + " description of optogenetics protocol\n", + "
subject10112022-04-01 12:13:14optodHPleftOPTG_4
\n", + " \n", + "

Total: 1

\n", + " " + ], + "text/plain": [ + "*subject *session_id *protocol_id opto_params_id implant_date implant_type target_region target_hemisph device protocol_descr\n", + "+----------+ +------------+ +------------+ +------------+ +------------+ +------------+ +------------+ +------------+ +--------+ +------------+\n", + "subject1 0 1 1 2022-04-01 12: opto dHP left OPTG_4 \n", + " (Total: 1)" + ] + }, + "execution_count": 24, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "optogenetics.OptoProtocol.insert1(\n", + " dict(\n", + " subject=\"subject1\",\n", + " session_id=\"0\",\n", + " protocol_id=\"1\",\n", + " opto_params_id=\"1\",\n", + " implant_date=\"2022-04-01 12:13:14\",\n", + " implant_type=\"opto\",\n", + " target_region=\"dHP\",\n", + " target_hemisphere=\"left\",\n", + " device=\"OPTG_4\",\n", + " )\n", + ")\n", + "optogenetics.OptoProtocol()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can describe the timing of these stimulations in `OptoEvent`." + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + " \n", + " \n", + " \n", + "
\n", + " \n", + " \n", + " \n", + "\n", + "\n", + "\n", + "\n", + "
\n", + "

subject

\n", + " \n", + "
\n", + "

session_id

\n", + " \n", + "
\n", + "

protocol_id

\n", + " \n", + "
\n", + "

stim_start_time

\n", + " (s) stimulus start time relative to session start\n", + "
\n", + "

stim_end_time

\n", + " (s) stimulus end time relative session start\n", + "
subject101241.0482.0
\n", + " \n", + "

Total: 1

\n", + " " + ], + "text/plain": [ + "*subject *session_id *protocol_id *stim_start_ti stim_end_time \n", + "+----------+ +------------+ +------------+ +------------+ +------------+\n", + "subject1 0 1 241.0 482.0 \n", + " (Total: 1)" + ] + }, + "execution_count": 25, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "optogenetics.OptoEvent.insert1(\n", + " dict(\n", + " subject=\"subject1\",\n", + " session_id=0,\n", + " protocol_id=1,\n", + " stim_start_time=241,\n", + " stim_end_time=482,\n", + " )\n", + ")\n", + "optogenetics.OptoEvent()" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can insert a second set of timing information for the stimulation." + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + " \n", + " \n", + " \n", + "
\n", + " \n", + " \n", + " \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
\n", + "

subject

\n", + " \n", + "
\n", + "

session_id

\n", + " \n", + "
\n", + "

protocol_id

\n", + " \n", + "
\n", + "

stim_start_time

\n", + " (s) stimulus start time relative to session start\n", + "
\n", + "

stim_end_time

\n", + " (s) stimulus end time relative session start\n", + "
subject101241.0482.0
subject101543.0797.0
\n", + " \n", + "

Total: 2

\n", + " " + ], + "text/plain": [ + "*subject *session_id *protocol_id *stim_start_ti stim_end_time \n", + "+----------+ +------------+ +------------+ +------------+ +------------+\n", + "subject1 0 1 241.0 482.0 \n", + "subject1 0 1 543.0 797.0 \n", + " (Total: 2)" + ] + }, + "execution_count": 26, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "optogenetics.OptoEvent.insert1(\n", + " dict(\n", + " subject=\"subject1\",\n", + " session_id=0,\n", + " protocol_id=1,\n", + " stim_start_time=543,\n", + " stim_end_time=797,\n", + " )\n", + ")\n", + "optogenetics.OptoEvent()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### **Query and view data**\n", + "\n", + "Queries allow you to view the contents of the database. The simplest query is the\n", + "instance of the table class." + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + " \n", + " \n", + " \n", + "
\n", + " \n", + " \n", + " \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
\n", + "

subject

\n", + " \n", + "
\n", + "

session_id

\n", + " \n", + "
\n", + "

protocol_id

\n", + " \n", + "
\n", + "

stim_start_time

\n", + " (s) stimulus start time relative to session start\n", + "
\n", + "

stim_end_time

\n", + " (s) stimulus end time relative session start\n", + "
subject101241.0482.0
subject101543.0797.0
\n", + " \n", + "

Total: 2

\n", + " " + ], + "text/plain": [ + "*subject *session_id *protocol_id *stim_start_ti stim_end_time \n", + "+----------+ +------------+ +------------+ +------------+ +------------+\n", + "subject1 0 1 241.0 482.0 \n", + "subject1 0 1 543.0 797.0 \n", + " (Total: 2)" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "optogenetics.OptoEvent()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "With the `&` operator, we will restrict the contents of the `OptoEvent` table to those entries with a `stim_start_time` of 543." + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + " \n", + " \n", + " \n", + "
\n", + " \n", + " \n", + " \n", + "\n", + "\n", + "\n", + "\n", + "
\n", + "

subject

\n", + " \n", + "
\n", + "

session_id

\n", + " \n", + "
\n", + "

protocol_id

\n", + " \n", + "
\n", + "

stim_start_time

\n", + " (s) stimulus start time relative to session start\n", + "
\n", + "

stim_end_time

\n", + " (s) stimulus end time relative session start\n", + "
subject101543.0797.0
\n", + " \n", + "

Total: 1

\n", + " " + ], + "text/plain": [ + "*subject *session_id *protocol_id *stim_start_ti stim_end_time \n", + "+----------+ +------------+ +------------+ +------------+ +------------+\n", + "subject1 0 1 543.0 797.0 \n", + " (Total: 1)" + ] + }, + "execution_count": 28, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "optogenetics.OptoEvent & \"stim_start_time=543\"" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "DataJoint queries can be a highly flexible tool with several [operators](https://datajoint.com/docs/core/concepts/query-lang/operators/). The next operator we will explore is `join` which combines matching information from tables." + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + " \n", + " \n", + " \n", + "
\n", + " \n", + " \n", + " \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
\n", + "

subject

\n", + " \n", + "
\n", + "

session_id

\n", + " \n", + "
\n", + "

protocol_id

\n", + " \n", + "
\n", + "

opto_params_id

\n", + " \n", + "
\n", + "

implant_date

\n", + " surgery date\n", + "
\n", + "

implant_type

\n", + " Short name for type of implanted device\n", + "
\n", + "

target_region

\n", + " Brain region shorthand\n", + "
\n", + "

target_hemisphere

\n", + " Brain region hemisphere\n", + "
\n", + "

device

\n", + " \n", + "
\n", + "

protocol_description

\n", + " description of optogenetics protocol\n", + "
\n", + "

waveform_name

\n", + " \n", + "
\n", + "

wavelength

\n", + " (nm) wavelength of optical stimulation light\n", + "
\n", + "

power

\n", + " (mW) total power from light source\n", + "
\n", + "

light_intensity

\n", + " (mW/mm2) power for given area\n", + "
\n", + "

frequency

\n", + " (Hz) frequency of the waveform\n", + "
\n", + "

duration

\n", + " (ms) duration of each optical stimulus\n", + "
subject10112022-04-01 12:13:14optodHPleftOPTG_4square_10470None10.201.0241.0
\n", + " \n", + "

Total: 1

\n", + " " + ], + "text/plain": [ + "*subject *session_id *protocol_id *opto_params_i implant_date implant_type target_region target_hemisph device protocol_descr waveform_name wavelength power light_intensit frequency duration \n", + "+----------+ +------------+ +------------+ +------------+ +------------+ +------------+ +------------+ +------------+ +--------+ +------------+ +------------+ +------------+ +-------+ +------------+ +-----------+ +----------+\n", + "subject1 0 1 1 2022-04-01 12: opto dHP left OPTG_4 square_10 470 None 10.20 1.0 241.0 \n", + " (Total: 1)" + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "optogenetics.OptoProtocol * optogenetics.OptoStimParams" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The `fetch` and `fetch1` methods download the data from the query object into the workspace.\n", + "\n", + "Below we will run `fetch()` without any arguments to return all attributes of all entries in the table." + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[{'subject': 'subject1',\n", + " 'session_id': 0,\n", + " 'protocol_id': 1,\n", + " 'stim_start_time': 241.0,\n", + " 'stim_end_time': 482.0},\n", + " {'subject': 'subject1',\n", + " 'session_id': 0,\n", + " 'protocol_id': 1,\n", + " 'stim_start_time': 543.0,\n", + " 'stim_end_time': 797.0}]" + ] + }, + "execution_count": 30, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "optogenetics.OptoEvent.fetch(as_dict=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Next, we will fetch the entry with a `stim_start_time` of 543 with the `fetch1` method, which returns a dictionary containing all attributes of one entry in the table." + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'subject': 'subject1',\n", + " 'session_id': 0,\n", + " 'protocol_id': 1,\n", + " 'stim_start_time': 543.0,\n", + " 'stim_end_time': 797.0}" + ] + }, + "execution_count": 31, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "(optogenetics.OptoEvent & \"stim_start_time=543\").fetch1()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "Following this tutorial, we have: \n", + "+ Covered the essential functionality of `element-optogenetics`.\n", + "+ Learned how to manually insert data into tables.\n", + "+ Queried and viewed the data. \n", + "\n", + "#### Documentation and DataJoint Tutorials\n", + "\n", + "+ [Detailed documentation on\n", + " `element-optogenetics`.](https://datajoint.com/docs/elements/element-optogenetics/)\n", + "+ [General `datajoint-python`\n", + " tutorials.](https://github.com/datajoint/datajoint-tutorials) covering fundamentals,\n", + " such as table tiers, query operations, fetch operations, automated computations with the\n", + " make function, and more.\n", + "+ [Documentation for\n", + " `datajoint-python`.](https://datajoint.com/docs/core/datajoint-python/)\n", + "\n", + "##### Run this tutorial on your own data\n", + "\n", + "To run this tutorial notebook on your own data, please use the following steps:\n", + "+ Download the [mysql-docker image for\n", + " DataJoint](https://github.com/datajoint/mysql-docker) and run the container according\n", + " to the instructions provide in the repository.\n", + "+ Create a fork of this repository to your GitHub account.\n", + "+ Clone the repository and open the files using your IDE.\n", + "+ Add a code cell immediately after the first code cell in the notebook - we will setup\n", + " the local connection using this cell. In this cell, type in the following code. \n", + "\n", + "```python\n", + "import datajoint as dj\n", + "dj.config[\"database.host\"] = \"localhost\"\n", + "dj.config[\"database.user\"] = \"\"\n", + "dj.config[\"database.password\"] = \"\"\n", + "dj.config[\"custom\"] = {\"imaging_root_data_dir\": \"path/to/your/data/dir\",\n", + "\"database_prefix\": \"\"}\n", + "dj.config.save_local()\n", + "dj.conn()\n", + "```\n", + "\n", + "+ Run the code block above and proceed with the rest of the notebook." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "python3p10", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.17" + }, + "orig_nbformat": 4, + "vscode": { + "interpreter": { + "hash": "ff52d424e56dd643d8b2ec122f40a2e279e94970100b4e6430cb9025a65ba4cf" + } + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/notebooks/tutorial_pipeline.py b/notebooks/tutorial_pipeline.py new file mode 100644 index 0000000..f18c250 --- /dev/null +++ b/notebooks/tutorial_pipeline.py @@ -0,0 +1,55 @@ +import os +import datajoint as dj +from element_animal import subject, surgery +from element_animal.subject import Subject # Dependency for session schema +from element_animal.surgery import Implantation # Dependency for optogenetics schema +from element_lab import lab +from element_lab.lab import Lab, Project, Protocol, Source, User +from element_optogenetics import optogenetics +from element_session import session_with_id as session +from element_session.session_with_id import Session + + +if "custom" not in dj.config: + dj.config["custom"] = {} + +# overwrite dj.config['custom'] values with environment variables if available + +dj.config["custom"]["database.prefix"] = os.getenv( + "DATABASE_PREFIX", dj.config["custom"].get("database.prefix", "") +) + +db_prefix = dj.config["custom"].get("database.prefix", "") + + +# Activate schemas +lab.activate(db_prefix + "lab") +subject.activate(db_prefix + "subject", linking_module=__name__) +surgery.activate(db_prefix + "surgery", linking_module=__name__) + +Experimenter = User +session.activate(db_prefix + "session", linking_module=__name__) + + +@lab.schema +class Device(dj.Lookup): + """Table for managing lab devices. + + Attributes: + device ( varchar(32) ): Device short name. + modality ( varchar(64) ): Modality for which this device is used. + description ( varchar(256), optional ): Description of device. + """ + + definition = """ + device : varchar(32) + --- + modality : varchar(64) + description='' : varchar(256) + """ + contents = [ + ["OPTG_4", "Optogenetics", "Doric Pulse Sequence Generator"], + ] + + +optogenetics.activate(db_prefix + "optogenetics", linking_module=__name__) diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index fc7a8e6..0000000 --- a/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -datajoint>=0.13.0 diff --git a/requirements_dev.txt b/requirements_dev.txt deleted file mode 100644 index 416634f..0000000 --- a/requirements_dev.txt +++ /dev/null @@ -1 +0,0 @@ -pre-commit diff --git a/setup.py b/setup.py index 6798877..4d5c096 100644 --- a/setup.py +++ b/setup.py @@ -1,16 +1,13 @@ from os import path - from setuptools import find_packages, setup -pkg_name = next(p for p in find_packages() if "." not in p) + +pkg_name = "element_optogenetics" here = path.abspath(path.dirname(__file__)) with open(path.join(here, "README.md"), "r") as f: long_description = f.read() -with open(path.join(here, "requirements.txt")) as f: - requirements = f.read().splitlines() - with open(path.join(here, pkg_name, "version.py")) as f: exec(f.read()) @@ -27,5 +24,16 @@ keywords="neuroscience optogenetics science datajoint", packages=find_packages(exclude=["contrib", "docs", "tests*"]), scripts=[], - install_requires=requirements, + install_requires=[ + "datajoint>=0.13.0", + ], + extras_require={ + "elements": [ + "element-animal @ git+https://github.com/datajoint/element-animal.git", + "element-event @ git+https://github.com/datajoint/element-event.git", + "element-lab @ git+https://github.com/datajoint/element-lab.git", + "element-session @ git+https://github.com/datajoint/element-session.git", + ], + "tests": ["pre-commit", "pytest", "pytest-cov"], + }, )