diff --git a/CHANGELOG.md b/CHANGELOG.md index 0276d7ae..da226b77 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,11 @@ Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) convention. +## [0.2.1] - 2022-07-22 ++ Add - Mention CodeBook data directory in notebooks ++ Update - Remove directory assertion in notebooks for CodeBook deployment ++ Update - Move all export functions from `pipeline` to `export` script + ## [0.2.0] - 2022-07-08 + Add - Adopt black formatting into code base diff --git a/notebooks/00-data-download-optional.ipynb b/notebooks/00-data-download-optional.ipynb index a2a91e43..d2a6a4c0 100644 --- a/notebooks/00-data-download-optional.ipynb +++ b/notebooks/00-data-download-optional.ipynb @@ -6,7 +6,9 @@ "source": [ "# Download example data\n", "\n", - "This workflow will need Ephys data collected from either SpikeGLX or OpenEphys and the output from kilosort2. We provided an example dataset to be downloaded to run through the pipeline. This notebook walks you through the process to download the dataset." + "This workflow will need Ephys data collected from either SpikeGLX or OpenEphys and the output from kilosort2. We provided an example dataset to be downloaded to run through the pipeline. This notebook walks you through the process to download the dataset.\n", + "\n", + "[CodeBook](codebook.datajoint.io) users can skip this step." ] }, { @@ -173,7 +175,7 @@ "formats": "ipynb,py" }, "kernelspec": { - "display_name": "Python 3.10.4 64-bit ('python3p10')", + "display_name": "Python 3.9.12 ('ele')", "language": "python", "name": "python3" }, @@ -187,11 +189,11 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.4" + "version": "3.9.12" }, "vscode": { "interpreter": { - "hash": "ff52d424e56dd643d8b2ec122f40a2e279e94970100b4e6430cb9025a65ba4cf" + "hash": "61456c693db5d9aa6731701ec9a9b08ab88a172bee0780139a3679beb166da16" } } }, diff --git a/notebooks/01-configure.ipynb b/notebooks/01-configure.ipynb index c741816e..d87310e4 100644 --- a/notebooks/01-configure.ipynb +++ b/notebooks/01-configure.ipynb @@ -35,11 +35,7 @@ "outputs": [], "source": [ "# change to the upper level folder\n", - "if os.path.basename(os.getcwd()) == \"notebooks\":\n", - " os.chdir(\"..\")\n", - "assert os.path.basename(os.getcwd()) == \"workflow-array-ephys\", (\n", - " \"Please move to the \" + \"workflow directory\"\n", - ")\n", + "if os.path.basename(os.getcwd()) == \"notebooks\": os.chdir(\"..\")\n", "import datajoint as dj" ] }, @@ -55,20 +51,11 @@ { "cell_type": "code", "execution_count": null, - "id": "6820ef4c", "metadata": {}, "outputs": [], "source": [ - "import getpass" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "dj.config[\"database.host\"] = \"{YOUR_HOST}\"\n", + "import getpass\n", + "dj.config[\"database.host\"] = \"{YOUR_HOST}\" # CodeBook users should omit this\n", "dj.config[\"database.user\"] = \"{YOUR_USERNAME}\"\n", "dj.config[\"database.password\"] = getpass.getpass() # enter the password securily" ] @@ -106,7 +93,7 @@ "\n", "Giving a prefix to schema could help on the configuration of privilege settings. For example, if we set prefix `neuro_`, every schema created with the current workflow will start with `neuro_`, e.g. `neuro_lab`, `neuro_subject`, `neuro_ephys` etc.\n", "\n", - "The prefix could be configurated as follows in `dj.config`:" + "The prefix could be configurated in `dj.config` as follows. CodeBook users should keep their username as the prefix for schema for declaration permissions." ] }, { @@ -115,7 +102,8 @@ "metadata": {}, "outputs": [], "source": [ - "dj.config[\"custom\"] = {\"database.prefix\": \"neuro_\"}" + "username_as_prefix = dj.config[\"database.user\"] + \"_\"\n", + "dj.config[\"custom\"] = {\"database.prefix\": username_as_prefix}" ] }, { @@ -130,10 +118,10 @@ "\n", "The root path typically **do not** contain information of subjects or sessions, all data from subjects/sessions should be subdirectories in the root path.\n", "\n", - "In the example dataset downloaded with [these instructions](00-data-download-optional.ipynb), `/tmp/test_data` will be the root\n", + "- In the example dataset downloaded with [these instructions](00-data-download-optional.ipynb), `/tmp/test_data` will be the root. \n", + "- For CodeBook users, the root is `/home/inbox/`\n", "\n", "```\n", - "/tmp/test_data/\n", "- subject6\n", " - session1\n", " - towersTask_g0_imec0\n", @@ -151,7 +139,7 @@ "# If there is only one root path.\n", "dj.config[\"custom\"][\"ephys_root_data_dir\"] = \"/tmp/test_data\"\n", "# If there are multiple possible root paths:\n", - "dj.config[\"custom\"][\"ephys_root_data_dir\"] = [\"/tmp/test_data1\", \"/tmp/test_data2\"]" + "dj.config[\"custom\"][\"ephys_root_data_dir\"] = [\"/tmp/test_data\", \"/home/inbox/\"]" ] }, { @@ -164,20 +152,13 @@ ] }, { - "cell_type": "code", - "execution_count": null, + "cell_type": "markdown", "id": "f7733450", "metadata": {}, - "outputs": [], "source": [ - "[markdown]\n", - "# + In the database, every path for the ephys raw data is **relative to root path(s)**. The benefit is that the absolute path could be configured for each machine, and when data transfer happens, we just need to change the root directory in the config file.\n", - "#\n", - "# + The workflow supports **multiple root directories**. If there are multiple possible root directories, specify the `ephys_root_data_dir` as a list.\n", - "#\n", - "# + The root path(s) are **specific to each machine**, as the name of mounted drive could be different for different operating systems or machines.\n", - "#\n", - "# + In the context of the workflow, all the paths saved into the database or saved in the config file need to be in the **POSIX standards** (Unix/Linux), with `/`. The path conversion for machines of any operating system is taken care of inside the elements." + "+ In the database, every path for the ephys raw data is **relative to root path(s)** to allow for the absolute path to be configured for **each machine**. When transferring data, we just need to change the root directory in the config file.\n", + "\n", + "+ DataJoint Elements use `pathlib.Path()` to maintain path information in **POSIX standards** (Unix/Linux), with `/`. The path conversion for machines of any operating system is taken care of inside the elements." ] }, { @@ -269,7 +250,7 @@ "formats": "ipynb,py_scripts//py" }, "kernelspec": { - "display_name": "Python 3.10.4 64-bit ('python3p10')", + "display_name": "Python 3.9.12 ('ele')", "language": "python", "name": "python3" }, @@ -283,11 +264,11 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.4" + "version": "3.9.12" }, "vscode": { "interpreter": { - "hash": "ff52d424e56dd643d8b2ec122f40a2e279e94970100b4e6430cb9025a65ba4cf" + "hash": "61456c693db5d9aa6731701ec9a9b08ab88a172bee0780139a3679beb166da16" } } }, diff --git a/notebooks/02-workflow-structure-optional.ipynb b/notebooks/02-workflow-structure-optional.ipynb index f3ea02ef..16b63998 100644 --- a/notebooks/02-workflow-structure-optional.ipynb +++ b/notebooks/02-workflow-structure-optional.ipynb @@ -11,10 +11,10 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "This notebook gives a brief overview of the workflow structure and introduces some useful DataJoint tools to facilitate the exploration.\n", - "+ DataJoint needs to be pre-configured before running this notebook, if you haven't set up the configuration, refer to notebook [01-configure](01-configure.ipynb).\n", - "+ If you are familar with DataJoint and the workflow structure, proceed to the next notebook [03-process](03-process.ipynb) directly to run the workflow.\n", - "+ For a more thorough introduction of DataJoint functions, please visit our [general tutorial site](https://codebook.datajoint.io)" + "This notebook gives an overview of the workflow structure and introduces useful DataJoint tools for exploration.\n", + "+ DataJoint needs to be configured before running this notebook. If you haven't done so, refer to notebook [01-configure](01-configure.ipynb).\n", + "+ If you are familar with DataJoint workflow structures, proceed to the next notebook [03-process](03-process.ipynb) directly to run this workflow.\n", + "+ For a more thorough introduction of DataJoint functions, please visit our [general documentation](https://docs.datajoint.org/python/v0.13/index.html)" ] }, { @@ -24,22 +24,13 @@ "To load the local configuration, we will change the directory to the package root." ] }, - { - "cell_type": "code", - "execution_count": null, - "id": "88a85d2d", - "metadata": {}, - "outputs": [], - "source": [ - "import os" - ] - }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ + "import os\n", "if os.path.basename(os.getcwd()) == \"notebooks\":\n", " os.chdir(\"..\")" ] @@ -48,14 +39,17 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Schemas and tables" + "## Schemas, Diagrams and Tables" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "The current workflow is composed of multiple database schemas, each of them corresponds to a module within `workflow_array_ephys.pipeline`" + "Schemas are conceptually related sets of tables. By importing schemas from `workflow_array_ephys.pipeline`, we'll declare the tables on the server with the prefix in the config (if we have permission to do so). If these tables are already declared, we'll gain access. \n", + "\n", + "- `dj.list_schemas()` lists all schemas a user has access to in the current database\n", + "- `.schema.list_tables()` will provide names for each table in the format used under the hood." ] }, { @@ -76,13 +70,6 @@ "from workflow_array_ephys.pipeline import lab, subject, session, probe, ephys" ] }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "+ Each module contains a schema object that enables interaction with the schema in the database." - ] - }, { "cell_type": "code", "execution_count": 3, @@ -122,14 +109,86 @@ } ], "source": [ - "ephys.schema" + "ephys.schema.list_tables()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "+ The table classes in the module corresponds to a table in the schema in the database." + "`dj.Diagram()` plots tables and dependencies in a schema. To see additional upstream or downstream connections, add `- N` or `+ N`.\n", + "\n", + "- `probe`: Neuropixels-based probe information\n", + "- `ephys`: Electrophysiology data" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "title": "`dj.Diagram()`: plot tables and dependencies" + }, + "outputs": [ + { + "data": { + "image/svg+xml": "\n\n%3\n\n\nephys.AcquisitionSoftware\n\n\nephys.AcquisitionSoftware\n\n\n\n\nephys.EphysRecording\n\n\nephys.EphysRecording\n\n\n\n\nephys.AcquisitionSoftware->ephys.EphysRecording\n\n\n\nephys.EphysRecording.EphysFile\n\n\nephys.EphysRecording.EphysFile\n\n\n\n\nephys.EphysRecording->ephys.EphysRecording.EphysFile\n\n\n\nephys.LFP\n\n\nephys.LFP\n\n\n\n\nephys.EphysRecording->ephys.LFP\n\n\n\nephys.ClusteringTask\n\n\nephys.ClusteringTask\n\n\n\n\nephys.EphysRecording->ephys.ClusteringTask\n\n\n\nephys.ClusterQualityLabel\n\n\nephys.ClusterQualityLabel\n\n\n\n\nephys.CuratedClustering.Unit\n\n\nephys.CuratedClustering.Unit\n\n\n\n\nephys.ClusterQualityLabel->ephys.CuratedClustering.Unit\n\n\n\nephys.ClusteringParamSet\n\n\nephys.ClusteringParamSet\n\n\n\n\nephys.ClusteringParamSet->ephys.ClusteringTask\n\n\n\nephys.WaveformSet.Waveform\n\n\nephys.WaveformSet.Waveform\n\n\n\n\nephys.CuratedClustering.Unit->ephys.WaveformSet.Waveform\n\n\n\nephys.WaveformSet.PeakWaveform\n\n\nephys.WaveformSet.PeakWaveform\n\n\n\n\nephys.CuratedClustering.Unit->ephys.WaveformSet.PeakWaveform\n\n\n\nephys.ProbeInsertion\n\n\nephys.ProbeInsertion\n\n\n\n\nephys.ProbeInsertion->ephys.EphysRecording\n\n\n\nephys.InsertionLocation\n\n\nephys.InsertionLocation\n\n\n\n\nephys.ProbeInsertion->ephys.InsertionLocation\n\n\n\nephys.Curation\n\n\nephys.Curation\n\n\n\n\nephys.CuratedClustering\n\n\nephys.CuratedClustering\n\n\n\n\nephys.Curation->ephys.CuratedClustering\n\n\n\nephys.LFP.Electrode\n\n\nephys.LFP.Electrode\n\n\n\n\nephys.LFP->ephys.LFP.Electrode\n\n\n\nephys.WaveformSet\n\n\nephys.WaveformSet\n\n\n\n\nephys.WaveformSet->ephys.WaveformSet.Waveform\n\n\n\nephys.WaveformSet->ephys.WaveformSet.PeakWaveform\n\n\n\nephys.Clustering\n\n\nephys.Clustering\n\n\n\n\nephys.ClusteringTask->ephys.Clustering\n\n\n\nephys.CuratedClustering->ephys.CuratedClustering.Unit\n\n\n\nephys.CuratedClustering->ephys.WaveformSet\n\n\n\nephys.Clustering->ephys.Curation\n\n\n\nephys.ClusteringMethod\n\n\nephys.ClusteringMethod\n\n\n\n\nephys.ClusteringMethod->ephys.ClusteringParamSet\n\n\n\n", + "text/plain": [ + "" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# plot diagram for all tables in a schema\n", + "dj.Diagram(probe)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "dj.Diagram(ephys) + dj.Diagram(session) - 1" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Table Types\n", + "\n", + "- **Manual table**: green box, manually inserted table, expect new entries daily, e.g. Subject, ProbeInsertion. \n", + "- **Lookup table**: gray box, pre inserted table, commonly used for general facts or parameters. e.g. Strain, ClusteringMethod, ClusteringParamSet. \n", + "- **Imported table**: blue oval, auto-processing table, the processing depends on the importing of external files. e.g. process of Clustering requires output files from kilosort2. \n", + "- **Computed table**: red circle, auto-processing table, the processing does not depend on files external to the database, commonly used for \n", + "- **Part table**: plain text, as an appendix to the master table, all the part entries of a given master entry represent a intact set of the master entry. e.g. Unit of a CuratedClustering.\n", + "\n", + "### Table Links\n", + "\n", + "- **One-to-one primary**: thick solid line, share the exact same primary key, meaning the child table inherits all the primary key fields from the parent table as its own primary key. \n", + "- **One-to-many primary**: thin solid line, inherit the primary key from the parent table, but have additional field(s) as part of the primary key as well\n", + "- **Secondary dependency**: dashed line, the child table inherits the primary key fields from parent table as its own secondary attribute." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Common Table Functions" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "- `()` show table contents\n", + "- `heading` shows attribute definitions\n", + "- `describe()` show table definition with foreign key references" ] }, { @@ -237,1764 +296,22 @@ "ephys.EphysRecording()" ] }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "+ By importing the modules for the first time, the schemas and tables will be created inside the database.\n", - "+ Once created, importing modules will not create schemas and tables again, but the existing schemas/tables can be accessed and manipulated by the modules." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## DataJoint tools to explore schemas and tables" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "lines_to_next_cell": 0, - "title": "The schemas and tables will not be re-created when importing modules if they have existed." - }, - "source": [ - "+ `dj.list_schemas()`: list all schemas a user has access to in the current database" - ] - }, { "cell_type": "code", "execution_count": null, - "metadata": { - "title": "`dj.list_schemas()`: list all schemas a user could access." - }, - "outputs": [], - "source": [ - "dj.list_schemas()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "+ `dj.Diagram()`: plot tables and dependencies. " - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": { - "title": "`dj.Diagram()`: plot tables and dependencies" - }, - "outputs": [ - { - "data": { - "image/svg+xml": [ - "\n", - "\n", - "%3\n", - "\n", - "\n", - "ephys.AcquisitionSoftware\n", - "\n", - "\n", - "ephys.AcquisitionSoftware\n", - "\n", - "\n", - "\n", - "\n", - "ephys.EphysRecording\n", - "\n", - "\n", - "ephys.EphysRecording\n", - "\n", - "\n", - "\n", - "\n", - "ephys.AcquisitionSoftware->ephys.EphysRecording\n", - "\n", - "\n", - "\n", - "ephys.EphysRecording.EphysFile\n", - "\n", - "\n", - "ephys.EphysRecording.EphysFile\n", - "\n", - "\n", - "\n", - "\n", - "ephys.EphysRecording->ephys.EphysRecording.EphysFile\n", - "\n", - "\n", - "\n", - "ephys.LFP\n", - "\n", - "\n", - "ephys.LFP\n", - "\n", - "\n", - "\n", - "\n", - "ephys.EphysRecording->ephys.LFP\n", - "\n", - "\n", - "\n", - "ephys.ClusteringTask\n", - "\n", - "\n", - "ephys.ClusteringTask\n", - "\n", - "\n", - "\n", - "\n", - "ephys.EphysRecording->ephys.ClusteringTask\n", - "\n", - "\n", - "\n", - "ephys.ClusterQualityLabel\n", - "\n", - "\n", - "ephys.ClusterQualityLabel\n", - "\n", - "\n", - "\n", - "\n", - "ephys.CuratedClustering.Unit\n", - "\n", - "\n", - "ephys.CuratedClustering.Unit\n", - "\n", - "\n", - "\n", - "\n", - "ephys.ClusterQualityLabel->ephys.CuratedClustering.Unit\n", - "\n", - "\n", - "\n", - "ephys.ClusteringParamSet\n", - "\n", - "\n", - "ephys.ClusteringParamSet\n", - "\n", - "\n", - "\n", - "\n", - "ephys.ClusteringParamSet->ephys.ClusteringTask\n", - "\n", - "\n", - "\n", - "ephys.WaveformSet.Waveform\n", - "\n", - "\n", - "ephys.WaveformSet.Waveform\n", - "\n", - "\n", - "\n", - "\n", - "ephys.CuratedClustering.Unit->ephys.WaveformSet.Waveform\n", - "\n", - "\n", - "\n", - "ephys.WaveformSet.PeakWaveform\n", - "\n", - "\n", - "ephys.WaveformSet.PeakWaveform\n", - "\n", - "\n", - "\n", - "\n", - "ephys.CuratedClustering.Unit->ephys.WaveformSet.PeakWaveform\n", - "\n", - "\n", - "\n", - "ephys.ProbeInsertion\n", - "\n", - "\n", - "ephys.ProbeInsertion\n", - "\n", - "\n", - "\n", - "\n", - "ephys.ProbeInsertion->ephys.EphysRecording\n", - "\n", - "\n", - "\n", - "ephys.InsertionLocation\n", - "\n", - "\n", - "ephys.InsertionLocation\n", - "\n", - "\n", - "\n", - "\n", - "ephys.ProbeInsertion->ephys.InsertionLocation\n", - "\n", - "\n", - "\n", - "ephys.Curation\n", - "\n", - "\n", - "ephys.Curation\n", - "\n", - "\n", - "\n", - "\n", - "ephys.CuratedClustering\n", - "\n", - "\n", - "ephys.CuratedClustering\n", - "\n", - "\n", - "\n", - "\n", - "ephys.Curation->ephys.CuratedClustering\n", - "\n", - "\n", - "\n", - "ephys.LFP.Electrode\n", - "\n", - "\n", - "ephys.LFP.Electrode\n", - "\n", - "\n", - "\n", - "\n", - "ephys.LFP->ephys.LFP.Electrode\n", - "\n", - "\n", - "\n", - "ephys.WaveformSet\n", - "\n", - "\n", - "ephys.WaveformSet\n", - "\n", - "\n", - "\n", - "\n", - "ephys.WaveformSet->ephys.WaveformSet.Waveform\n", - "\n", - "\n", - "\n", - "ephys.WaveformSet->ephys.WaveformSet.PeakWaveform\n", - "\n", - "\n", - "\n", - "ephys.Clustering\n", - "\n", - "\n", - "ephys.Clustering\n", - "\n", - "\n", - "\n", - "\n", - "ephys.ClusteringTask->ephys.Clustering\n", - "\n", - "\n", - "\n", - "ephys.CuratedClustering->ephys.CuratedClustering.Unit\n", - "\n", - "\n", - "\n", - "ephys.CuratedClustering->ephys.WaveformSet\n", - "\n", - "\n", - "\n", - "ephys.Clustering->ephys.Curation\n", - "\n", - "\n", - "\n", - "ephys.ClusteringMethod\n", - "\n", - "\n", - "ephys.ClusteringMethod\n", - "\n", - "\n", - "\n", - "\n", - "ephys.ClusteringMethod->ephys.ClusteringParamSet\n", - "\n", - "\n", - "\n", - "" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# plot diagram for all tables in a schema\n", - "dj.Diagram(ephys)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "**Table tiers**:\n", - "\n", - "Manual table: green box, manually inserted table, expect new entries daily, e.g. Subject, ProbeInsertion.\n", - "Lookup table: gray box, pre inserted table, commonly used for general facts or parameters. e.g. Strain, ClusteringMethod, ClusteringParamSet.\n", - "Imported table: blue oval, auto-processing table, the processing depends on the importing of external files. e.g. process of Clustering requires output files from kilosort2.\n", - "Computed table: red circle, auto-processing table, the processing does not depend on files external to the database, commonly used for\n", - "Part table: plain text, as an appendix to the master table, all the part entries of a given master entry represent a intact set of the master entry. e.g. Unit of a CuratedClustering.\n", - "\n", - "**Dependencies**:\n", - "\n", - "One-to-one primary: thick solid line, share the exact same primary key, meaning the child table inherits all the primary key fields from the parent table as its own primary key.\n", - "One-to-many primary: thin solid line, inherit the primary key from the parent table, but have additional field(s) as part of the primary key as well\n", - "secondary dependency: dashed line, the child table inherits the primary key fields from parent table as its own secondary attribute." - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": { - "title": "`dj.Diagram()`: plot the diagram of the tables and dependencies. It could be used to plot tables in a schema or selected tables." - }, - "outputs": [ - { - "data": { - "image/svg+xml": [ - "\n", - "\n", - "%3\n", - "\n", - "\n", - "ephys.AcquisitionSoftware\n", - "\n", - "\n", - "ephys.AcquisitionSoftware\n", - "\n", - "\n", - "\n", - "\n", - "ephys.EphysRecording\n", - "\n", - "\n", - "ephys.EphysRecording\n", - "\n", - "\n", - "\n", - "\n", - "ephys.AcquisitionSoftware->ephys.EphysRecording\n", - "\n", - "\n", - "\n", - "ephys.ClusteringParamSet\n", - "\n", - "\n", - "ephys.ClusteringParamSet\n", - "\n", - "\n", - "\n", - "\n", - "ephys.ClusteringTask\n", - "\n", - "\n", - "ephys.ClusteringTask\n", - "\n", - "\n", - "\n", - "\n", - "ephys.ClusteringParamSet->ephys.ClusteringTask\n", - "\n", - "\n", - "\n", - "subject.Line\n", - "\n", - "\n", - "subject.Line\n", - "\n", - "\n", - "\n", - "\n", - "subject.Subject.Line\n", - "\n", - "\n", - "subject.Subject.Line\n", - "\n", - "\n", - "\n", - "\n", - "subject.Line->subject.Subject.Line\n", - "\n", - "\n", - "\n", - "subject.Line.Allele\n", - "\n", - "\n", - "subject.Line.Allele\n", - "\n", - "\n", - "\n", - "\n", - "subject.Line->subject.Line.Allele\n", - "\n", - "\n", - "\n", - "subject.Subject.Protocol\n", - "\n", - "\n", - "subject.Subject.Protocol\n", - "\n", - "\n", - "\n", - "\n", - "subject.SubjectCullMethod\n", - "\n", - "\n", - "subject.SubjectCullMethod\n", - "\n", - "\n", - "\n", - "\n", - "subject.Allele.Source\n", - "\n", - "\n", - "subject.Allele.Source\n", - "\n", - "\n", - "\n", - "\n", - "ephys.Curation\n", - "\n", - "\n", - "ephys.Curation\n", - "\n", - "\n", - "\n", - "\n", - "ephys.CuratedClustering\n", - "\n", - "\n", - "ephys.CuratedClustering\n", - "\n", - "\n", - "\n", - "\n", - "ephys.Curation->ephys.CuratedClustering\n", - "\n", - "\n", - "\n", - "subject.Strain\n", - "\n", - "\n", - "subject.Strain\n", - "\n", - "\n", - "\n", - "\n", - "subject.Subject.Strain\n", - "\n", - "\n", - "subject.Subject.Strain\n", - "\n", - "\n", - "\n", - "\n", - "subject.Strain->subject.Subject.Strain\n", - "\n", - "\n", - "\n", - "subject.Allele\n", - "\n", - "\n", - "subject.Allele\n", - "\n", - "\n", - "\n", - "\n", - "subject.Allele->subject.Allele.Source\n", - "\n", - "\n", - "\n", - "subject.Allele->subject.Line.Allele\n", - "\n", - "\n", - "\n", - "subject.Zygosity\n", - "\n", - "\n", - "subject.Zygosity\n", - "\n", - "\n", - "\n", - "\n", - "subject.Allele->subject.Zygosity\n", - "\n", - "\n", - "\n", - "session.Session\n", - "\n", - "\n", - "session.Session\n", - "\n", - "\n", - "\n", - "\n", - "session.ProjectSession\n", - "\n", - "\n", - "session.ProjectSession\n", - "\n", - "\n", - "\n", - "\n", - "session.Session->session.ProjectSession\n", - "\n", - "\n", - "\n", - "session.SessionDirectory\n", - "\n", - "\n", - "session.SessionDirectory\n", - "\n", - "\n", - "\n", - "\n", - "session.Session->session.SessionDirectory\n", - "\n", - "\n", - "\n", - "ephys.ProbeInsertion\n", - "\n", - "\n", - "ephys.ProbeInsertion\n", - "\n", - "\n", - "\n", - "\n", - "session.Session->ephys.ProbeInsertion\n", - "\n", - "\n", - "\n", - "ephys.ClusteringMethod\n", - "\n", - "\n", - "ephys.ClusteringMethod\n", - "\n", - "\n", - "\n", - "\n", - "ephys.ClusteringMethod->ephys.ClusteringParamSet\n", - "\n", - "\n", - "\n", - "subject.Subject.User\n", - "\n", - "\n", - "subject.Subject.User\n", - "\n", - "\n", - "\n", - "\n", - "ephys.CuratedClustering.Unit\n", - "\n", - "\n", - "ephys.CuratedClustering.Unit\n", - "\n", - "\n", - "\n", - "\n", - "ephys.WaveformSet.Waveform\n", - "\n", - "\n", - "ephys.WaveformSet.Waveform\n", - "\n", - "\n", - "\n", - "\n", - "ephys.CuratedClustering.Unit->ephys.WaveformSet.Waveform\n", - "\n", - "\n", - "\n", - "ephys.WaveformSet.PeakWaveform\n", - "\n", - "\n", - "ephys.WaveformSet.PeakWaveform\n", - "\n", - "\n", - "\n", - "\n", - "ephys.CuratedClustering.Unit->ephys.WaveformSet.PeakWaveform\n", - "\n", - "\n", - "\n", - "ephys.LFP.Electrode\n", - "\n", - "\n", - "ephys.LFP.Electrode\n", - "\n", - "\n", - "\n", - "\n", - "ephys.WaveformSet\n", - "\n", - "\n", - "ephys.WaveformSet\n", - "\n", - "\n", - "\n", - "\n", - "ephys.WaveformSet->ephys.WaveformSet.Waveform\n", - "\n", - "\n", - "\n", - "ephys.WaveformSet->ephys.WaveformSet.PeakWaveform\n", - "\n", - "\n", - "\n", - "subject.Subject\n", - "\n", - "\n", - "subject.Subject\n", - "\n", - "\n", - "\n", - "\n", - "subject.Subject->subject.Subject.Protocol\n", - "\n", - "\n", - "\n", - "subject.Subject->subject.SubjectCullMethod\n", - "\n", - "\n", - "\n", - "subject.Subject->subject.Subject.Line\n", - "\n", - "\n", - "\n", - "subject.Subject->session.Session\n", - "\n", - "\n", - "\n", - "subject.Subject->subject.Subject.User\n", - "\n", - "\n", - "\n", - "subject.Subject->subject.Subject.Strain\n", - "\n", - "\n", - "\n", - "subject.Subject.Source\n", - "\n", - "\n", - "subject.Subject.Source\n", - "\n", - "\n", - "\n", - "\n", - "subject.Subject->subject.Subject.Source\n", - "\n", - "\n", - "\n", - "subject.SubjectDeath\n", - "\n", - "\n", - "subject.SubjectDeath\n", - "\n", - "\n", - "\n", - "\n", - "subject.Subject->subject.SubjectDeath\n", - "\n", - "\n", - "\n", - "subject.Subject.Lab\n", - "\n", - "\n", - "subject.Subject.Lab\n", - "\n", - "\n", - "\n", - "\n", - "subject.Subject->subject.Subject.Lab\n", - "\n", - "\n", - "\n", - "subject.Subject->subject.Zygosity\n", - "\n", - "\n", - "\n", - "ephys.EphysRecording.EphysFile\n", - "\n", - "\n", - "ephys.EphysRecording.EphysFile\n", - "\n", - "\n", - "\n", - "\n", - "ephys.EphysRecording->ephys.EphysRecording.EphysFile\n", - "\n", - "\n", - "\n", - "ephys.LFP\n", - "\n", - "\n", - "ephys.LFP\n", - "\n", - "\n", - "\n", - "\n", - "ephys.EphysRecording->ephys.LFP\n", - "\n", - "\n", - "\n", - "ephys.EphysRecording->ephys.ClusteringTask\n", - "\n", - "\n", - "\n", - "ephys.ClusterQualityLabel\n", - "\n", - "\n", - "ephys.ClusterQualityLabel\n", - "\n", - "\n", - "\n", - "\n", - "ephys.ClusterQualityLabel->ephys.CuratedClustering.Unit\n", - "\n", - "\n", - "\n", - "ephys.InsertionLocation\n", - "\n", - "\n", - "ephys.InsertionLocation\n", - "\n", - "\n", - "\n", - "\n", - "ephys.LFP->ephys.LFP.Electrode\n", - "\n", - "\n", - "\n", - "ephys.ProbeInsertion->ephys.EphysRecording\n", - "\n", - "\n", - "\n", - "ephys.ProbeInsertion->ephys.InsertionLocation\n", - "\n", - "\n", - "\n", - "ephys.Clustering\n", - "\n", - "\n", - "ephys.Clustering\n", - "\n", - "\n", - "\n", - "\n", - "ephys.ClusteringTask->ephys.Clustering\n", - "\n", - "\n", - "\n", - "ephys.CuratedClustering->ephys.CuratedClustering.Unit\n", - "\n", - "\n", - "\n", - "ephys.CuratedClustering->ephys.WaveformSet\n", - "\n", - "\n", - "\n", - "ephys.Clustering->ephys.Curation\n", - "\n", - "\n", - "\n", - "" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# plot diagram of tables in multiple schemas\n", - "dj.Diagram(subject) + dj.Diagram(session) + dj.Diagram(ephys)" - ] - }, - { - "cell_type": "code", - "execution_count": 9, "metadata": {}, - "outputs": [ - { - "data": { - "image/svg+xml": [ - "\n", - "\n", - "%3\n", - "\n", - "\n", - "ephys.AcquisitionSoftware\n", - "\n", - "\n", - "ephys.AcquisitionSoftware\n", - "\n", - "\n", - "\n", - "\n", - "ephys.EphysRecording\n", - "\n", - "\n", - "ephys.EphysRecording\n", - "\n", - "\n", - "\n", - "\n", - "ephys.AcquisitionSoftware->ephys.EphysRecording\n", - "\n", - "\n", - "\n", - "ephys.ClusteringParamSet\n", - "\n", - "\n", - "ephys.ClusteringParamSet\n", - "\n", - "\n", - "\n", - "\n", - "ephys.ClusteringTask\n", - "\n", - "\n", - "ephys.ClusteringTask\n", - "\n", - "\n", - "\n", - "\n", - "ephys.ClusteringParamSet->ephys.ClusteringTask\n", - "\n", - "\n", - "\n", - "ephys.Curation\n", - "\n", - "\n", - "ephys.Curation\n", - "\n", - "\n", - "\n", - "\n", - "ephys.CuratedClustering\n", - "\n", - "\n", - "ephys.CuratedClustering\n", - "\n", - "\n", - "\n", - "\n", - "ephys.Curation->ephys.CuratedClustering\n", - "\n", - "\n", - "\n", - "session.Session\n", - "\n", - "\n", - "session.Session\n", - "\n", - "\n", - "\n", - "\n", - "ephys.ProbeInsertion\n", - "\n", - "\n", - "ephys.ProbeInsertion\n", - "\n", - "\n", - "\n", - "\n", - "session.Session->ephys.ProbeInsertion\n", - "\n", - "\n", - "\n", - "ephys.ClusteringMethod\n", - "\n", - "\n", - "ephys.ClusteringMethod\n", - "\n", - "\n", - "\n", - "\n", - "ephys.ClusteringMethod->ephys.ClusteringParamSet\n", - "\n", - "\n", - "\n", - "ephys.CuratedClustering.Unit\n", - "\n", - "\n", - "ephys.CuratedClustering.Unit\n", - "\n", - "\n", - "\n", - "\n", - "ephys.WaveformSet.Waveform\n", - "\n", - "\n", - "ephys.WaveformSet.Waveform\n", - "\n", - "\n", - "\n", - "\n", - "ephys.CuratedClustering.Unit->ephys.WaveformSet.Waveform\n", - "\n", - "\n", - "\n", - "ephys.WaveformSet.PeakWaveform\n", - "\n", - "\n", - "ephys.WaveformSet.PeakWaveform\n", - "\n", - "\n", - "\n", - "\n", - "ephys.CuratedClustering.Unit->ephys.WaveformSet.PeakWaveform\n", - "\n", - "\n", - "\n", - "ephys.LFP.Electrode\n", - "\n", - "\n", - "ephys.LFP.Electrode\n", - "\n", - "\n", - "\n", - "\n", - "ephys.WaveformSet\n", - "\n", - "\n", - "ephys.WaveformSet\n", - "\n", - "\n", - "\n", - "\n", - "ephys.WaveformSet->ephys.WaveformSet.Waveform\n", - "\n", - "\n", - "\n", - "ephys.WaveformSet->ephys.WaveformSet.PeakWaveform\n", - "\n", - "\n", - "\n", - "subject.Subject\n", - "\n", - "\n", - "subject.Subject\n", - "\n", - "\n", - "\n", - "\n", - "subject.Subject->session.Session\n", - "\n", - "\n", - "\n", - "ephys.EphysRecording.EphysFile\n", - "\n", - "\n", - "ephys.EphysRecording.EphysFile\n", - "\n", - "\n", - "\n", - "\n", - "ephys.EphysRecording->ephys.EphysRecording.EphysFile\n", - "\n", - "\n", - "\n", - "ephys.LFP\n", - "\n", - "\n", - "ephys.LFP\n", - "\n", - "\n", - "\n", - "\n", - "ephys.EphysRecording->ephys.LFP\n", - "\n", - "\n", - "\n", - "ephys.EphysRecording->ephys.ClusteringTask\n", - "\n", - "\n", - "\n", - "ephys.ClusterQualityLabel\n", - "\n", - "\n", - "ephys.ClusterQualityLabel\n", - "\n", - "\n", - "\n", - "\n", - "ephys.ClusterQualityLabel->ephys.CuratedClustering.Unit\n", - "\n", - "\n", - "\n", - "ephys.InsertionLocation\n", - "\n", - "\n", - "ephys.InsertionLocation\n", - "\n", - "\n", - "\n", - "\n", - "ephys.LFP->ephys.LFP.Electrode\n", - "\n", - "\n", - "\n", - "ephys.ProbeInsertion->ephys.EphysRecording\n", - "\n", - "\n", - "\n", - "ephys.ProbeInsertion->ephys.InsertionLocation\n", - "\n", - "\n", - "\n", - "ephys.Clustering\n", - "\n", - "\n", - "ephys.Clustering\n", - "\n", - "\n", - "\n", - "\n", - "ephys.ClusteringTask->ephys.Clustering\n", - "\n", - "\n", - "\n", - "ephys.CuratedClustering->ephys.CuratedClustering.Unit\n", - "\n", - "\n", - "\n", - "ephys.CuratedClustering->ephys.WaveformSet\n", - "\n", - "\n", - "\n", - "ephys.Clustering->ephys.Curation\n", - "\n", - "\n", - "\n", - "" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# plot diagram of selected tables and schemas\n", - "dj.Diagram(subject.Subject) + dj.Diagram(session.Session) + dj.Diagram(ephys)" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "data": { - "image/svg+xml": [ - "\n", - "\n", - "%3\n", - "\n", - "\n", - "subject.Subject.Protocol\n", - "\n", - "\n", - "subject.Subject.Protocol\n", - "\n", - "\n", - "\n", - "\n", - "subject.SubjectCullMethod\n", - "\n", - "\n", - "subject.SubjectCullMethod\n", - "\n", - "\n", - "\n", - "\n", - "subject.Subject.Source\n", - "\n", - "\n", - "subject.Subject.Source\n", - "\n", - "\n", - "\n", - "\n", - "subject.Subject.Lab\n", - "\n", - "\n", - "subject.Subject.Lab\n", - "\n", - "\n", - "\n", - "\n", - "subject.SubjectDeath\n", - "\n", - "\n", - "subject.SubjectDeath\n", - "\n", - "\n", - "\n", - "\n", - "subject.Zygosity\n", - "\n", - "\n", - "subject.Zygosity\n", - "\n", - "\n", - "\n", - "\n", - "subject.Subject.User\n", - "\n", - "\n", - "subject.Subject.User\n", - "\n", - "\n", - "\n", - "\n", - "subject.Subject.Strain\n", - "\n", - "\n", - "subject.Subject.Strain\n", - "\n", - "\n", - "\n", - "\n", - "subject.Subject.Line\n", - "\n", - "\n", - "subject.Subject.Line\n", - "\n", - "\n", - "\n", - "\n", - "subject.Subject\n", - "\n", - "\n", - "subject.Subject\n", - "\n", - "\n", - "\n", - "\n", - "subject.Subject->subject.Subject.Protocol\n", - "\n", - "\n", - "\n", - "subject.Subject->subject.SubjectCullMethod\n", - "\n", - "\n", - "\n", - "subject.Subject->subject.Subject.Source\n", - "\n", - "\n", - "\n", - "subject.Subject->subject.Subject.Lab\n", - "\n", - "\n", - "\n", - "subject.Subject->subject.SubjectDeath\n", - "\n", - "\n", - "\n", - "subject.Subject->subject.Zygosity\n", - "\n", - "\n", - "\n", - "subject.Subject->subject.Subject.User\n", - "\n", - "\n", - "\n", - "subject.Subject->subject.Subject.Strain\n", - "\n", - "\n", - "\n", - "subject.Subject->subject.Subject.Line\n", - "\n", - "\n", - "\n", - "session.Session\n", - "\n", - "\n", - "session.Session\n", - "\n", - "\n", - "\n", - "\n", - "subject.Subject->session.Session\n", - "\n", - "\n", - "\n", - "" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# plot diagram with 1 additional level of dependency downstream\n", - "dj.Diagram(subject.Subject) + 1" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [ - { - "data": { - "image/svg+xml": [ - "\n", - "\n", - "%3\n", - "\n", - "\n", - "ephys.AcquisitionSoftware\n", - "\n", - "\n", - "ephys.AcquisitionSoftware\n", - "\n", - "\n", - "\n", - "\n", - "ephys.EphysRecording\n", - "\n", - "\n", - "ephys.EphysRecording\n", - "\n", - "\n", - "\n", - "\n", - "ephys.AcquisitionSoftware->ephys.EphysRecording\n", - "\n", - "\n", - "\n", - "ephys.ProbeInsertion\n", - "\n", - "\n", - "ephys.ProbeInsertion\n", - "\n", - "\n", - "\n", - "\n", - "ephys.ProbeInsertion->ephys.EphysRecording\n", - "\n", - "\n", - "\n", - "probe.ElectrodeConfig\n", - "\n", - "\n", - "probe.ElectrodeConfig\n", - "\n", - "\n", - "\n", - "\n", - "probe.ElectrodeConfig->ephys.EphysRecording\n", - "\n", - "\n", - "\n", - "probe.ProbeType\n", - "\n", - "\n", - "probe.ProbeType\n", - "\n", - "\n", - "\n", - "\n", - "probe.ProbeType->probe.ElectrodeConfig\n", - "\n", - "\n", - "\n", - "probe.Probe\n", - "\n", - "\n", - "probe.Probe\n", - "\n", - "\n", - "\n", - "\n", - "probe.ProbeType->probe.Probe\n", - "\n", - "\n", - "\n", - "session.Session\n", - "\n", - "\n", - "session.Session\n", - "\n", - "\n", - "\n", - "\n", - "session.Session->ephys.ProbeInsertion\n", - "\n", - "\n", - "\n", - "probe.Probe->ephys.ProbeInsertion\n", - "\n", - "\n", - "\n", - "" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# plot diagram with 2 additional levels of dependency upstream\n", - "dj.Diagram(ephys.EphysRecording) - 2" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "lines_to_next_cell": 0, - "title": "`heading`:" - }, + "outputs": [], "source": [ - "+ `describe()`: show table definition with foreign key references." + "ephys.Clustering.heading" ] }, { "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "# Ephys recording from a probe insertion for a given session.\n", - "-> ephys.ProbeInsertion\n", - "---\n", - "-> probe.ElectrodeConfig\n", - "-> ephys.AcquisitionSoftware\n", - "sampling_rate : float # (Hz)\n", - "\n" - ] - } - ], - "source": [ - "ephys.EphysRecording.describe()" - ] - }, - { - "cell_type": "markdown", + "execution_count": null, "metadata": {}, + "outputs": [], "source": [ - "+ `heading`: show attribute definitions regardless of foreign key references" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": { - "title": "`heading`: show table attributes regardless of foreign key references." - }, - "outputs": [ - { - "data": { - "text/plain": [ - "# Ephys recording from a probe insertion for a given session.\n", - "subject : varchar(32) # \n", - "session_datetime : datetime(3) # \n", - "insertion_number : tinyint unsigned # \n", - "---\n", - "electrode_config_hash : uuid # \n", - "acq_software : varchar(24) # \n", - "sampling_rate : float # (Hz)" - ] - }, - "execution_count": 13, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ephys.EphysRecording.heading" + "ephys.WaveformSet.describe()" ] }, { @@ -2004,16 +321,13 @@ "title": "probe" }, "source": [ - "# Major DataJoint Elements installed in the current workflow" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "title": "ephys" - }, - "source": [ - "+ [`lab`](https://github.com/datajoint/element-lab): lab management related information, such as Lab, User, Project, Protocol, Source." + "## Other Elements installed with the workflow\n", + "\n", + "- [`lab`](https://github.com/datajoint/element-lab): lab management related information, such as Lab, User, Project, Protocol, Source.\n", + "- [`subject`](https://github.com/datajoint/element-animal): (element-animal) general animal information, User, Genetic background, Death etc.\n", + "- [`session`](https://github.com/datajoint/element-session): General information of experimental sessions.\n", + "\n", + "For more information about these Elements, see [workflow session](https://github.com/datajoint/workflow-session)." ] }, { @@ -2023,161 +337,7 @@ "outputs": [ { "data": { - "image/svg+xml": [ - "\n", - "\n", - "%3\n", - "\n", - "\n", - "`neuro_lab`.`#skull_reference`\n", - "\n", - "`neuro_lab`.`#skull_reference`\n", - "\n", - "\n", - "lab.LabMembership\n", - "\n", - "\n", - "lab.LabMembership\n", - "\n", - "\n", - "\n", - "\n", - "lab.Location\n", - "\n", - "\n", - "lab.Location\n", - "\n", - "\n", - "\n", - "\n", - "lab.Source\n", - "\n", - "\n", - "lab.Source\n", - "\n", - "\n", - "\n", - "\n", - "lab.Protocol\n", - "\n", - "\n", - "lab.Protocol\n", - "\n", - "\n", - "\n", - "\n", - "lab.Project\n", - "\n", - "\n", - "lab.Project\n", - "\n", - "\n", - "\n", - "\n", - "lab.ProjectUser\n", - "\n", - "\n", - "lab.ProjectUser\n", - "\n", - "\n", - "\n", - "\n", - "lab.Project->lab.ProjectUser\n", - "\n", - "\n", - "\n", - "lab.Lab\n", - "\n", - "\n", - "lab.Lab\n", - "\n", - "\n", - "\n", - "\n", - "lab.Lab->lab.LabMembership\n", - "\n", - "\n", - "\n", - "lab.Lab->lab.Location\n", - "\n", - "\n", - "\n", - "lab.User\n", - "\n", - "\n", - "lab.User\n", - "\n", - "\n", - "\n", - "\n", - "lab.User->lab.LabMembership\n", - "\n", - "\n", - "\n", - "lab.User->lab.ProjectUser\n", - "\n", - "\n", - "\n", - "lab.UserRole\n", - "\n", - "\n", - "lab.UserRole\n", - "\n", - "\n", - "\n", - "\n", - "lab.UserRole->lab.LabMembership\n", - "\n", - "\n", - "\n", - "lab.ProtocolType\n", - "\n", - "\n", - "lab.ProtocolType\n", - "\n", - "\n", - "\n", - "\n", - "lab.ProtocolType->lab.Protocol\n", - "\n", - "\n", - "\n", - "" - ], + "image/svg+xml": "\n\n%3\n\n\n`neuro_lab`.`#skull_reference`\n\n`neuro_lab`.`#skull_reference`\n\n\nlab.LabMembership\n\n\nlab.LabMembership\n\n\n\n\nlab.Location\n\n\nlab.Location\n\n\n\n\nlab.Source\n\n\nlab.Source\n\n\n\n\nlab.Protocol\n\n\nlab.Protocol\n\n\n\n\nlab.Project\n\n\nlab.Project\n\n\n\n\nlab.ProjectUser\n\n\nlab.ProjectUser\n\n\n\n\nlab.Project->lab.ProjectUser\n\n\n\nlab.Lab\n\n\nlab.Lab\n\n\n\n\nlab.Lab->lab.LabMembership\n\n\n\nlab.Lab->lab.Location\n\n\n\nlab.User\n\n\nlab.User\n\n\n\n\nlab.User->lab.LabMembership\n\n\n\nlab.User->lab.ProjectUser\n\n\n\nlab.UserRole\n\n\nlab.UserRole\n\n\n\n\nlab.UserRole->lab.LabMembership\n\n\n\nlab.ProtocolType\n\n\nlab.ProtocolType\n\n\n\n\nlab.ProtocolType->lab.Protocol\n\n\n\n", "text/plain": [ "" ] @@ -2191,13 +351,6 @@ "dj.Diagram(lab)" ] }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "+ [`animal`](https://github.com/datajoint/element-animal): general animal information, User, Genetic background, Death etc." - ] - }, { "cell_type": "code", "execution_count": 15, @@ -2205,246 +358,7 @@ "outputs": [ { "data": { - "image/svg+xml": [ - "\n", - "\n", - "%3\n", - "\n", - "\n", - "subject.Line\n", - "\n", - "\n", - "subject.Line\n", - "\n", - "\n", - "\n", - "\n", - "subject.Line.Allele\n", - "\n", - "\n", - "subject.Line.Allele\n", - "\n", - "\n", - "\n", - "\n", - "subject.Line->subject.Line.Allele\n", - "\n", - "\n", - "\n", - "subject.Subject.Line\n", - "\n", - "\n", - "subject.Subject.Line\n", - "\n", - "\n", - "\n", - "\n", - "subject.Line->subject.Subject.Line\n", - "\n", - "\n", - "\n", - "subject.Subject.Protocol\n", - "\n", - "\n", - "subject.Subject.Protocol\n", - "\n", - "\n", - "\n", - "\n", - "subject.SubjectCullMethod\n", - "\n", - "\n", - "subject.SubjectCullMethod\n", - "\n", - "\n", - "\n", - "\n", - "subject.Subject.Source\n", - "\n", - "\n", - "subject.Subject.Source\n", - "\n", - "\n", - "\n", - "\n", - "subject.Allele.Source\n", - "\n", - "\n", - "subject.Allele.Source\n", - "\n", - "\n", - "\n", - "\n", - "subject.Subject.Lab\n", - "\n", - "\n", - "subject.Subject.Lab\n", - "\n", - "\n", - "\n", - "\n", - "subject.SubjectDeath\n", - "\n", - "\n", - "subject.SubjectDeath\n", - "\n", - "\n", - "\n", - "\n", - "subject.Zygosity\n", - "\n", - "\n", - "subject.Zygosity\n", - "\n", - "\n", - "\n", - "\n", - "subject.Strain\n", - "\n", - "\n", - "subject.Strain\n", - "\n", - "\n", - "\n", - "\n", - "subject.Subject.Strain\n", - "\n", - "\n", - "subject.Subject.Strain\n", - "\n", - "\n", - "\n", - "\n", - "subject.Strain->subject.Subject.Strain\n", - "\n", - "\n", - "\n", - "subject.Subject.User\n", - "\n", - "\n", - "subject.Subject.User\n", - "\n", - "\n", - "\n", - "\n", - "subject.Allele\n", - "\n", - "\n", - "subject.Allele\n", - "\n", - "\n", - "\n", - "\n", - "subject.Allele->subject.Allele.Source\n", - "\n", - "\n", - "\n", - "subject.Allele->subject.Zygosity\n", - "\n", - "\n", - "\n", - "subject.Allele->subject.Line.Allele\n", - "\n", - "\n", - "\n", - "subject.Subject\n", - "\n", - "\n", - "subject.Subject\n", - "\n", - "\n", - "\n", - "\n", - "subject.Subject->subject.Subject.Protocol\n", - "\n", - "\n", - "\n", - "subject.Subject->subject.SubjectCullMethod\n", - "\n", - "\n", - "\n", - "subject.Subject->subject.Subject.Source\n", - "\n", - "\n", - "\n", - "subject.Subject->subject.Subject.Lab\n", - "\n", - "\n", - "\n", - "subject.Subject->subject.SubjectDeath\n", - "\n", - "\n", - "\n", - "subject.Subject->subject.Zygosity\n", - "\n", - "\n", - "\n", - "subject.Subject->subject.Subject.User\n", - "\n", - "\n", - "\n", - "subject.Subject->subject.Subject.Strain\n", - "\n", - "\n", - "\n", - "subject.Subject->subject.Subject.Line\n", - "\n", - "\n", - "\n", - "" - ], + "image/svg+xml": "\n\n%3\n\n\nsubject.Line\n\n\nsubject.Line\n\n\n\n\nsubject.Line.Allele\n\n\nsubject.Line.Allele\n\n\n\n\nsubject.Line->subject.Line.Allele\n\n\n\nsubject.Subject.Line\n\n\nsubject.Subject.Line\n\n\n\n\nsubject.Line->subject.Subject.Line\n\n\n\nsubject.Subject.Protocol\n\n\nsubject.Subject.Protocol\n\n\n\n\nsubject.SubjectCullMethod\n\n\nsubject.SubjectCullMethod\n\n\n\n\nsubject.Subject.Source\n\n\nsubject.Subject.Source\n\n\n\n\nsubject.Allele.Source\n\n\nsubject.Allele.Source\n\n\n\n\nsubject.Subject.Lab\n\n\nsubject.Subject.Lab\n\n\n\n\nsubject.SubjectDeath\n\n\nsubject.SubjectDeath\n\n\n\n\nsubject.Zygosity\n\n\nsubject.Zygosity\n\n\n\n\nsubject.Strain\n\n\nsubject.Strain\n\n\n\n\nsubject.Subject.Strain\n\n\nsubject.Subject.Strain\n\n\n\n\nsubject.Strain->subject.Subject.Strain\n\n\n\nsubject.Subject.User\n\n\nsubject.Subject.User\n\n\n\n\nsubject.Allele\n\n\nsubject.Allele\n\n\n\n\nsubject.Allele->subject.Allele.Source\n\n\n\nsubject.Allele->subject.Zygosity\n\n\n\nsubject.Allele->subject.Line.Allele\n\n\n\nsubject.Subject\n\n\nsubject.Subject\n\n\n\n\nsubject.Subject->subject.Subject.Protocol\n\n\n\nsubject.Subject->subject.SubjectCullMethod\n\n\n\nsubject.Subject->subject.Subject.Source\n\n\n\nsubject.Subject->subject.Subject.Lab\n\n\n\nsubject.Subject->subject.SubjectDeath\n\n\n\nsubject.Subject->subject.Zygosity\n\n\n\nsubject.Subject->subject.Subject.User\n\n\n\nsubject.Subject->subject.Subject.Strain\n\n\n\nsubject.Subject->subject.Subject.Line\n\n\n\n", "text/plain": [ "" ] @@ -2483,13 +397,6 @@ "subject.Subject.describe()" ] }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "+ [`session`](https://github.com/datajoint/element-session): General information of experimental sessions." - ] - }, { "cell_type": "code", "execution_count": 17, @@ -2497,53 +404,7 @@ "outputs": [ { "data": { - "image/svg+xml": [ - "\n", - "\n", - "%3\n", - "\n", - "\n", - "session.SessionDirectory\n", - "\n", - "\n", - "session.SessionDirectory\n", - "\n", - "\n", - "\n", - "\n", - "session.ProjectSession\n", - "\n", - "\n", - "session.ProjectSession\n", - "\n", - "\n", - "\n", - "\n", - "session.Session\n", - "\n", - "\n", - "session.Session\n", - "\n", - "\n", - "\n", - "\n", - "session.Session->session.SessionDirectory\n", - "\n", - "\n", - "\n", - "session.Session->session.ProjectSession\n", - "\n", - "\n", - "\n", - "" - ], + "image/svg+xml": "\n\n%3\n\n\nsession.SessionDirectory\n\n\nsession.SessionDirectory\n\n\n\n\nsession.ProjectSession\n\n\nsession.ProjectSession\n\n\n\n\nsession.Session\n\n\nsession.Session\n\n\n\n\nsession.Session->session.SessionDirectory\n\n\n\nsession.Session->session.ProjectSession\n\n\n\n", "text/plain": [ "" ] @@ -2557,464 +418,6 @@ "dj.Diagram(session)" ] }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": { - "title": "[session](https://github.com/datajoint/element-session): experimental session information" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "-> subject.Subject\n", - "session_datetime : datetime(3) \n", - "\n" - ] - } - ], - "source": [ - "session.Session.describe()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "+ [`ephys`](https://github.com/datajoint/element-array-ephys): Neuropixel based probe and ephys information" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": { - "title": "[probe and ephys](https://github.com/datajoint/element-array-ephys): Neuropixel based probe and ephys tables" - }, - "outputs": [ - { - "data": { - "image/svg+xml": [ - "\n", - "\n", - "%3\n", - "\n", - "\n", - "ephys.AcquisitionSoftware\n", - "\n", - "\n", - "ephys.AcquisitionSoftware\n", - "\n", - "\n", - "\n", - "\n", - "ephys.EphysRecording\n", - "\n", - "\n", - "ephys.EphysRecording\n", - "\n", - "\n", - "\n", - "\n", - "ephys.AcquisitionSoftware->ephys.EphysRecording\n", - "\n", - "\n", - "\n", - "ephys.ClusteringParamSet\n", - "\n", - "\n", - "ephys.ClusteringParamSet\n", - "\n", - "\n", - "\n", - "\n", - "ephys.ClusteringTask\n", - "\n", - "\n", - "ephys.ClusteringTask\n", - "\n", - "\n", - "\n", - "\n", - "ephys.ClusteringParamSet->ephys.ClusteringTask\n", - "\n", - "\n", - "\n", - "ephys.Curation\n", - "\n", - "\n", - "ephys.Curation\n", - "\n", - "\n", - "\n", - "\n", - "ephys.CuratedClustering\n", - "\n", - "\n", - "ephys.CuratedClustering\n", - "\n", - "\n", - "\n", - "\n", - "ephys.Curation->ephys.CuratedClustering\n", - "\n", - "\n", - "\n", - "ephys.ClusteringMethod\n", - "\n", - "\n", - "ephys.ClusteringMethod\n", - "\n", - "\n", - "\n", - "\n", - "ephys.ClusteringMethod->ephys.ClusteringParamSet\n", - "\n", - "\n", - "\n", - "ephys.CuratedClustering.Unit\n", - "\n", - "\n", - "ephys.CuratedClustering.Unit\n", - "\n", - "\n", - "\n", - "\n", - "ephys.WaveformSet.Waveform\n", - "\n", - "\n", - "ephys.WaveformSet.Waveform\n", - "\n", - "\n", - "\n", - "\n", - "ephys.CuratedClustering.Unit->ephys.WaveformSet.Waveform\n", - "\n", - "\n", - "\n", - "ephys.WaveformSet.PeakWaveform\n", - "\n", - "\n", - "ephys.WaveformSet.PeakWaveform\n", - "\n", - "\n", - "\n", - "\n", - "ephys.CuratedClustering.Unit->ephys.WaveformSet.PeakWaveform\n", - "\n", - "\n", - "\n", - "ephys.LFP.Electrode\n", - "\n", - "\n", - "ephys.LFP.Electrode\n", - "\n", - "\n", - "\n", - "\n", - "ephys.WaveformSet\n", - "\n", - "\n", - "ephys.WaveformSet\n", - "\n", - "\n", - "\n", - "\n", - "ephys.WaveformSet->ephys.WaveformSet.Waveform\n", - "\n", - "\n", - "\n", - "ephys.WaveformSet->ephys.WaveformSet.PeakWaveform\n", - "\n", - "\n", - "\n", - "probe.Probe\n", - "\n", - "\n", - "probe.Probe\n", - "\n", - "\n", - "\n", - "\n", - "ephys.ProbeInsertion\n", - "\n", - "\n", - "ephys.ProbeInsertion\n", - "\n", - "\n", - "\n", - "\n", - "probe.Probe->ephys.ProbeInsertion\n", - "\n", - "\n", - "\n", - "ephys.EphysRecording.EphysFile\n", - "\n", - "\n", - "ephys.EphysRecording.EphysFile\n", - "\n", - "\n", - "\n", - "\n", - "ephys.EphysRecording->ephys.EphysRecording.EphysFile\n", - "\n", - "\n", - "\n", - "ephys.LFP\n", - "\n", - "\n", - "ephys.LFP\n", - "\n", - "\n", - "\n", - "\n", - "ephys.EphysRecording->ephys.LFP\n", - "\n", - "\n", - "\n", - "ephys.EphysRecording->ephys.ClusteringTask\n", - "\n", - "\n", - "\n", - "ephys.ClusterQualityLabel\n", - "\n", - "\n", - "ephys.ClusterQualityLabel\n", - "\n", - "\n", - "\n", - "\n", - "ephys.ClusterQualityLabel->ephys.CuratedClustering.Unit\n", - "\n", - "\n", - "\n", - "probe.ElectrodeConfig\n", - "\n", - "\n", - "probe.ElectrodeConfig\n", - "\n", - "\n", - "\n", - "\n", - "probe.ElectrodeConfig->ephys.EphysRecording\n", - "\n", - "\n", - "\n", - "probe.ElectrodeConfig.Electrode\n", - "\n", - "\n", - "probe.ElectrodeConfig.Electrode\n", - "\n", - "\n", - "\n", - "\n", - "probe.ElectrodeConfig->probe.ElectrodeConfig.Electrode\n", - "\n", - "\n", - "\n", - "ephys.InsertionLocation\n", - "\n", - "\n", - "ephys.InsertionLocation\n", - "\n", - "\n", - "\n", - "\n", - "ephys.LFP->ephys.LFP.Electrode\n", - "\n", - "\n", - "\n", - "probe.ElectrodeConfig.Electrode->ephys.CuratedClustering.Unit\n", - "\n", - "\n", - "\n", - "probe.ElectrodeConfig.Electrode->ephys.WaveformSet.Waveform\n", - "\n", - "\n", - "\n", - "probe.ElectrodeConfig.Electrode->ephys.LFP.Electrode\n", - "\n", - "\n", - "\n", - "probe.ProbeType\n", - "\n", - "\n", - "probe.ProbeType\n", - "\n", - "\n", - "\n", - "\n", - "probe.ProbeType->probe.Probe\n", - "\n", - "\n", - "\n", - "probe.ProbeType->probe.ElectrodeConfig\n", - "\n", - "\n", - "\n", - "probe.ProbeType.Electrode\n", - "\n", - "\n", - "probe.ProbeType.Electrode\n", - "\n", - "\n", - "\n", - "\n", - "probe.ProbeType->probe.ProbeType.Electrode\n", - "\n", - "\n", - "\n", - "ephys.ProbeInsertion->ephys.EphysRecording\n", - "\n", - "\n", - "\n", - "ephys.ProbeInsertion->ephys.InsertionLocation\n", - "\n", - "\n", - "\n", - "probe.ProbeType.Electrode->probe.ElectrodeConfig.Electrode\n", - "\n", - "\n", - "\n", - "ephys.Clustering\n", - "\n", - "\n", - "ephys.Clustering\n", - "\n", - "\n", - "\n", - "\n", - "ephys.ClusteringTask->ephys.Clustering\n", - "\n", - "\n", - "\n", - "ephys.CuratedClustering->ephys.CuratedClustering.Unit\n", - "\n", - "\n", - "\n", - "ephys.CuratedClustering->ephys.WaveformSet\n", - "\n", - "\n", - "\n", - "ephys.Clustering->ephys.Curation\n", - "\n", - "\n", - "\n", - "" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 19, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "dj.Diagram(probe) + dj.Diagram(ephys)" - ] - }, { "cell_type": "markdown", "metadata": {}, @@ -3033,7 +436,7 @@ "formats": "ipynb,py" }, "kernelspec": { - "display_name": "Python 3.10.4 64-bit ('python3p10')", + "display_name": "Python 3.9.12 ('ele')", "language": "python", "name": "python3" }, @@ -3047,11 +450,11 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.4" + "version": "3.9.12" }, "vscode": { "interpreter": { - "hash": "ff52d424e56dd643d8b2ec122f40a2e279e94970100b4e6430cb9025a65ba4cf" + "hash": "61456c693db5d9aa6731701ec9a9b08ab88a172bee0780139a3679beb166da16" } } }, diff --git a/notebooks/03-process.ipynb b/notebooks/03-process.ipynb index 85d98604..5ba9a73d 100644 --- a/notebooks/03-process.ipynb +++ b/notebooks/03-process.ipynb @@ -25,23 +25,14 @@ "Let's will change the directory to the package root to load configuration and also import relevant schemas." ] }, - { - "cell_type": "code", - "execution_count": null, - "id": "f7e60290", - "metadata": {}, - "outputs": [], - "source": [ - "import os" - ] - }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ - "os.chdir(\"..\")" + "import os\n", + "if os.path.basename(os.getcwd()) == \"notebooks\": os.chdir(\"..\")" ] }, { @@ -84,73 +75,7 @@ "outputs": [ { "data": { - "image/svg+xml": [ - "\n", - "\n", - "%3\n", - "\n", - "\n", - "probe.Probe\n", - "\n", - "\n", - "probe.Probe\n", - "\n", - "\n", - "\n", - "\n", - "ephys.ProbeInsertion\n", - "\n", - "\n", - "ephys.ProbeInsertion\n", - "\n", - "\n", - "\n", - "\n", - "probe.Probe->ephys.ProbeInsertion\n", - "\n", - "\n", - "\n", - "subject.Subject\n", - "\n", - "\n", - "subject.Subject\n", - "\n", - "\n", - "\n", - "\n", - "session.Session\n", - "\n", - "\n", - "session.Session\n", - "\n", - "\n", - "\n", - "\n", - "subject.Subject->session.Session\n", - "\n", - "\n", - "\n", - "session.Session->ephys.ProbeInsertion\n", - "\n", - "\n", - "\n", - "" - ], + "image/svg+xml": "\n\n%3\n\n\nprobe.Probe\n\n\nprobe.Probe\n\n\n\n\nephys.ProbeInsertion\n\n\nephys.ProbeInsertion\n\n\n\n\nprobe.Probe->ephys.ProbeInsertion\n\n\n\nsubject.Subject\n\n\nsubject.Subject\n\n\n\n\nsession.Session\n\n\nsession.Session\n\n\n\n\nsubject.Subject->session.Session\n\n\n\nsession.Session->ephys.ProbeInsertion\n\n\n\n", "text/plain": [ "" ] @@ -923,7 +848,7 @@ "ingest_subjects()\n", "ingest_sessions()\n", "```\n", - "`ingest_sessions` also extracts probe and probe insertion information automatically from the meta file.\n", + "`ingest_sessions` also extracts probe and probe insertion information automatically from the meta files.\n", "\n", "This is the regular routine for daily data processing, illustrated in notebook [04-automate](04-automate[optional].ipynb)." ] @@ -949,101 +874,7 @@ "outputs": [ { "data": { - "image/svg+xml": [ - "\n", - "\n", - "%3\n", - "\n", - "\n", - "ephys.EphysRecording\n", - "\n", - "\n", - "ephys.EphysRecording\n", - "\n", - "\n", - "\n", - "\n", - "ephys.EphysRecording.EphysFile\n", - "\n", - "\n", - "ephys.EphysRecording.EphysFile\n", - "\n", - "\n", - "\n", - "\n", - "ephys.EphysRecording->ephys.EphysRecording.EphysFile\n", - "\n", - "\n", - "\n", - "probe.ElectrodeConfig\n", - "\n", - "\n", - "probe.ElectrodeConfig\n", - "\n", - "\n", - "\n", - "\n", - "probe.ElectrodeConfig->ephys.EphysRecording\n", - "\n", - "\n", - "\n", - "probe.ElectrodeConfig.Electrode\n", - "\n", - "\n", - "probe.ElectrodeConfig.Electrode\n", - "\n", - "\n", - "\n", - "\n", - "probe.ElectrodeConfig->probe.ElectrodeConfig.Electrode\n", - "\n", - "\n", - "\n", - "session.Session\n", - "\n", - "\n", - "session.Session\n", - "\n", - "\n", - "\n", - "\n", - "ephys.ProbeInsertion\n", - "\n", - "\n", - "ephys.ProbeInsertion\n", - "\n", - "\n", - "\n", - "\n", - "session.Session->ephys.ProbeInsertion\n", - "\n", - "\n", - "\n", - "ephys.ProbeInsertion->ephys.EphysRecording\n", - "\n", - "\n", - "\n", - "" - ], + "image/svg+xml": "\n\n%3\n\n\nephys.EphysRecording\n\n\nephys.EphysRecording\n\n\n\n\nephys.EphysRecording.EphysFile\n\n\nephys.EphysRecording.EphysFile\n\n\n\n\nephys.EphysRecording->ephys.EphysRecording.EphysFile\n\n\n\nprobe.ElectrodeConfig\n\n\nprobe.ElectrodeConfig\n\n\n\n\nprobe.ElectrodeConfig->ephys.EphysRecording\n\n\n\nprobe.ElectrodeConfig.Electrode\n\n\nprobe.ElectrodeConfig.Electrode\n\n\n\n\nprobe.ElectrodeConfig->probe.ElectrodeConfig.Electrode\n\n\n\nsession.Session\n\n\nsession.Session\n\n\n\n\nephys.ProbeInsertion\n\n\nephys.ProbeInsertion\n\n\n\n\nsession.Session->ephys.ProbeInsertion\n\n\n\nephys.ProbeInsertion->ephys.EphysRecording\n\n\n\n", "text/plain": [ "" ] @@ -1542,79 +1373,7 @@ "outputs": [ { "data": { - "image/svg+xml": [ - "\n", - "\n", - "%3\n", - "\n", - "\n", - "ephys.ClusteringTask\n", - "\n", - "\n", - "ephys.ClusteringTask\n", - "\n", - "\n", - "\n", - "\n", - "ephys.Clustering\n", - "\n", - "\n", - "ephys.Clustering\n", - "\n", - "\n", - "\n", - "\n", - "ephys.ClusteringTask->ephys.Clustering\n", - "\n", - "\n", - "\n", - "ephys.EphysRecording\n", - "\n", - "\n", - "ephys.EphysRecording\n", - "\n", - "\n", - "\n", - "\n", - "ephys.EphysRecording->ephys.ClusteringTask\n", - "\n", - "\n", - "\n", - "ephys.ClusteringParamSet\n", - "\n", - "\n", - "ephys.ClusteringParamSet\n", - "\n", - "\n", - "\n", - "\n", - "ephys.ClusteringParamSet->ephys.ClusteringTask\n", - "\n", - "\n", - "\n", - "" - ], + "image/svg+xml": "\n\n%3\n\n\nephys.ClusteringTask\n\n\nephys.ClusteringTask\n\n\n\n\nephys.Clustering\n\n\nephys.Clustering\n\n\n\n\nephys.ClusteringTask->ephys.Clustering\n\n\n\nephys.EphysRecording\n\n\nephys.EphysRecording\n\n\n\n\nephys.EphysRecording->ephys.ClusteringTask\n\n\n\nephys.ClusteringParamSet\n\n\nephys.ClusteringParamSet\n\n\n\n\nephys.ClusteringParamSet->ephys.ClusteringTask\n\n\n\n", "text/plain": [ "" ] @@ -2121,97 +1880,7 @@ "outputs": [ { "data": { - "image/svg+xml": [ - "\n", - "\n", - "%3\n", - "\n", - "\n", - "ephys.Clustering\n", - "\n", - "\n", - "ephys.Clustering\n", - "\n", - "\n", - "\n", - "\n", - "ephys.Curation\n", - "\n", - "\n", - "ephys.Curation\n", - "\n", - "\n", - "\n", - "\n", - "ephys.Clustering->ephys.Curation\n", - "\n", - "\n", - "\n", - "ephys.CuratedClustering.Unit\n", - "\n", - "\n", - "ephys.CuratedClustering.Unit\n", - "\n", - "\n", - "\n", - "\n", - "ephys.ClusteringTask\n", - "\n", - "\n", - "ephys.ClusteringTask\n", - "\n", - "\n", - "\n", - "\n", - "ephys.ClusteringTask->ephys.Clustering\n", - "\n", - "\n", - "\n", - "ephys.CuratedClustering\n", - "\n", - "\n", - "ephys.CuratedClustering\n", - "\n", - "\n", - "\n", - "\n", - "ephys.CuratedClustering->ephys.CuratedClustering.Unit\n", - "\n", - "\n", - "\n", - "ephys.Curation->ephys.CuratedClustering\n", - "\n", - "\n", - "\n", - "" - ], + "image/svg+xml": "\n\n%3\n\n\nephys.Clustering\n\n\nephys.Clustering\n\n\n\n\nephys.Curation\n\n\nephys.Curation\n\n\n\n\nephys.Clustering->ephys.Curation\n\n\n\nephys.CuratedClustering.Unit\n\n\nephys.CuratedClustering.Unit\n\n\n\n\nephys.ClusteringTask\n\n\nephys.ClusteringTask\n\n\n\n\nephys.ClusteringTask->ephys.Clustering\n\n\n\nephys.CuratedClustering\n\n\nephys.CuratedClustering\n\n\n\n\nephys.CuratedClustering->ephys.CuratedClustering.Unit\n\n\n\nephys.Curation->ephys.CuratedClustering\n\n\n\n", "text/plain": [ "" ] @@ -2595,60 +2264,7 @@ "outputs": [ { "data": { - "image/svg+xml": [ - "\n", - "\n", - "%3\n", - "\n", - "\n", - "ephys.LFP.Electrode\n", - "\n", - "\n", - "ephys.LFP.Electrode\n", - "\n", - "\n", - "\n", - "\n", - "ephys.EphysRecording\n", - "\n", - "\n", - "ephys.EphysRecording\n", - "\n", - "\n", - "\n", - "\n", - "ephys.LFP\n", - "\n", - "\n", - "ephys.LFP\n", - "\n", - "\n", - "\n", - "\n", - "ephys.EphysRecording->ephys.LFP\n", - "\n", - "\n", - "\n", - "ephys.LFP->ephys.LFP.Electrode\n", - "\n", - "\n", - "\n", - "" - ], + "image/svg+xml": "\n\n%3\n\n\nephys.LFP.Electrode\n\n\nephys.LFP.Electrode\n\n\n\n\nephys.EphysRecording\n\n\nephys.EphysRecording\n\n\n\n\nephys.LFP\n\n\nephys.LFP\n\n\n\n\nephys.EphysRecording->ephys.LFP\n\n\n\nephys.LFP->ephys.LFP.Electrode\n\n\n\n", "text/plain": [ "" ] @@ -2960,99 +2576,7 @@ "outputs": [ { "data": { - "image/svg+xml": [ - "\n", - "\n", - "%3\n", - "\n", - "\n", - "ephys.WaveformSet\n", - "\n", - "\n", - "ephys.WaveformSet\n", - "\n", - "\n", - "\n", - "\n", - "ephys.WaveformSet.PeakWaveform\n", - "\n", - "\n", - "ephys.WaveformSet.PeakWaveform\n", - "\n", - "\n", - "\n", - "\n", - "ephys.WaveformSet->ephys.WaveformSet.PeakWaveform\n", - "\n", - "\n", - "\n", - "ephys.WaveformSet.Waveform\n", - "\n", - "\n", - "ephys.WaveformSet.Waveform\n", - "\n", - "\n", - "\n", - "\n", - "ephys.WaveformSet->ephys.WaveformSet.Waveform\n", - "\n", - "\n", - "\n", - "ephys.CuratedClustering.Unit\n", - "\n", - "\n", - "ephys.CuratedClustering.Unit\n", - "\n", - "\n", - "\n", - "\n", - "ephys.CuratedClustering.Unit->ephys.WaveformSet.PeakWaveform\n", - "\n", - "\n", - "\n", - "ephys.CuratedClustering.Unit->ephys.WaveformSet.Waveform\n", - "\n", - "\n", - "\n", - "ephys.CuratedClustering\n", - "\n", - "\n", - "ephys.CuratedClustering\n", - "\n", - "\n", - "\n", - "\n", - "ephys.CuratedClustering->ephys.WaveformSet\n", - "\n", - "\n", - "\n", - "ephys.CuratedClustering->ephys.CuratedClustering.Unit\n", - "\n", - "\n", - "\n", - "" - ], + "image/svg+xml": "\n\n%3\n\n\nephys.WaveformSet\n\n\nephys.WaveformSet\n\n\n\n\nephys.WaveformSet.PeakWaveform\n\n\nephys.WaveformSet.PeakWaveform\n\n\n\n\nephys.WaveformSet->ephys.WaveformSet.PeakWaveform\n\n\n\nephys.WaveformSet.Waveform\n\n\nephys.WaveformSet.Waveform\n\n\n\n\nephys.WaveformSet->ephys.WaveformSet.Waveform\n\n\n\nephys.CuratedClustering.Unit\n\n\nephys.CuratedClustering.Unit\n\n\n\n\nephys.CuratedClustering.Unit->ephys.WaveformSet.PeakWaveform\n\n\n\nephys.CuratedClustering.Unit->ephys.WaveformSet.Waveform\n\n\n\nephys.CuratedClustering\n\n\nephys.CuratedClustering\n\n\n\n\nephys.CuratedClustering->ephys.WaveformSet\n\n\n\nephys.CuratedClustering->ephys.CuratedClustering.Unit\n\n\n\n", "text/plain": [ "" ] @@ -3561,7 +3085,7 @@ "formats": "ipynb,py_scripts//py" }, "kernelspec": { - "display_name": "Python 3.10.4 64-bit ('python3p10')", + "display_name": "Python 3.9.12 ('ele')", "language": "python", "name": "python3" }, @@ -3575,11 +3099,11 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.4" + "version": "3.9.12" }, "vscode": { "interpreter": { - "hash": "ff52d424e56dd643d8b2ec122f40a2e279e94970100b4e6430cb9025a65ba4cf" + "hash": "61456c693db5d9aa6731701ec9a9b08ab88a172bee0780139a3679beb166da16" } } }, diff --git a/notebooks/04-automate-optional.ipynb b/notebooks/04-automate-optional.ipynb index e062ef11..9a885c96 100644 --- a/notebooks/04-automate-optional.ipynb +++ b/notebooks/04-automate-optional.ipynb @@ -27,16 +27,8 @@ } ], "source": [ - "import os" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "os.chdir(\"..\")" + "import os\n", + "if os.path.basename(os.getcwd()) == \"notebooks\": os.chdir(\"..\")" ] }, { @@ -384,7 +376,7 @@ "formats": "ipynb,py_scripts//py" }, "kernelspec": { - "display_name": "Python 3.10.4 64-bit ('python3p10')", + "display_name": "Python 3.9.12 ('ele')", "language": "python", "name": "python3" }, @@ -398,11 +390,11 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.4" + "version": "3.9.12" }, "vscode": { "interpreter": { - "hash": "ff52d424e56dd643d8b2ec122f40a2e279e94970100b4e6430cb9025a65ba4cf" + "hash": "61456c693db5d9aa6731701ec9a9b08ab88a172bee0780139a3679beb166da16" } } }, diff --git a/notebooks/05-explore.ipynb b/notebooks/05-explore.ipynb index 7eaed580..4cbf4424 100644 --- a/notebooks/05-explore.ipynb +++ b/notebooks/05-explore.ipynb @@ -9,22 +9,14 @@ "This notebook will describe the steps for interacting with the data ingested into `workflow-array-ephys`." ] }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "import os" - ] - }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "os.chdir(\"..\")" + "import os\n", + "if os.path.basename(os.getcwd()) == \"notebooks\": os.chdir(\"..\")" ] }, { @@ -1838,7 +1830,7 @@ "formats": "ipynb,py_scripts//py" }, "kernelspec": { - "display_name": "Python 3.10.4 64-bit ('python3p10')", + "display_name": "Python 3.9.12 ('ele')", "language": "python", "name": "python3" }, @@ -1852,11 +1844,11 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.4" + "version": "3.9.12" }, "vscode": { "interpreter": { - "hash": "ff52d424e56dd643d8b2ec122f40a2e279e94970100b4e6430cb9025a65ba4cf" + "hash": "61456c693db5d9aa6731701ec9a9b08ab88a172bee0780139a3679beb166da16" } } }, diff --git a/notebooks/06-drop-optional.ipynb b/notebooks/06-drop-optional.ipynb index 05cb36d2..7db90e60 100644 --- a/notebooks/06-drop-optional.ipynb +++ b/notebooks/06-drop-optional.ipynb @@ -25,16 +25,8 @@ "metadata": {}, "outputs": [], "source": [ - "import os" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "os.chdir(\"..\")" + "import os\n", + "if os.path.basename(os.getcwd()) == \"notebooks\": os.chdir(\"..\")" ] }, { @@ -73,7 +65,7 @@ "formats": "ipynb,py_scripts//py" }, "kernelspec": { - "display_name": "Python 3.10.4 64-bit ('python3p10')", + "display_name": "Python 3.9.12 ('ele')", "language": "python", "name": "python3" }, @@ -87,11 +79,11 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.4" + "version": "3.9.12" }, "vscode": { "interpreter": { - "hash": "ff52d424e56dd643d8b2ec122f40a2e279e94970100b4e6430cb9025a65ba4cf" + "hash": "61456c693db5d9aa6731701ec9a9b08ab88a172bee0780139a3679beb166da16" } } }, diff --git a/notebooks/07-downstream-analysis.ipynb b/notebooks/07-downstream-analysis.ipynb index 1c6f3bef..9025e1a1 100644 --- a/notebooks/07-downstream-analysis.ipynb +++ b/notebooks/07-downstream-analysis.ipynb @@ -35,22 +35,8 @@ "metadata": {}, "outputs": [], "source": [ - "import os" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "a0ec4e38", - "metadata": {}, - "outputs": [], - "source": [ - "# change to the upper level folder to detect dj_local_conf.json\n", - "if os.path.basename(os.getcwd()) == \"notebooks\":\n", - " os.chdir(\"..\")\n", - "assert os.path.basename(os.getcwd()) == \"workflow-array-ephys\", (\n", - " \"Please move to the \" + \"workflow directory\"\n", - ")" + "import os\n", + "if os.path.basename(os.getcwd()) == \"notebooks\": os.chdir(\"..\")" ] }, { @@ -61,16 +47,7 @@ "outputs": [], "source": [ "# We'll be working with long tables, so we'll make visualization easier with a limit\n", - "import datajoint as dj" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "4fcb0cda", - "metadata": {}, - "outputs": [], - "source": [ + "import datajoint as dj\n", "dj.config[\"display.limit\"] = 10" ] }, @@ -1755,7 +1732,7 @@ "formats": "ipynb,py_scripts//py" }, "kernelspec": { - "display_name": "Python 3.10.4 64-bit ('python3p10')", + "display_name": "Python 3.9.12 ('ele')", "language": "python", "name": "python3" }, @@ -1769,11 +1746,11 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.4" + "version": "3.9.12" }, "vscode": { "interpreter": { - "hash": "ff52d424e56dd643d8b2ec122f40a2e279e94970100b4e6430cb9025a65ba4cf" + "hash": "61456c693db5d9aa6731701ec9a9b08ab88a172bee0780139a3679beb166da16" } } }, diff --git a/notebooks/08-electrode-localization.ipynb b/notebooks/08-electrode-localization.ipynb index 1457a02b..3931a9b1 100644 --- a/notebooks/08-electrode-localization.ipynb +++ b/notebooks/08-electrode-localization.ipynb @@ -14,15 +14,6 @@ "Change into the parent directory to find the `dj_local_conf.json` file." ] }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "import os" - ] - }, { "cell_type": "code", "execution_count": null, @@ -30,11 +21,9 @@ "outputs": [], "source": [ "# change to the upper level folder to detect dj_local_conf.json\n", + "import os\n", "if os.path.basename(os.getcwd()) == \"notebooks\":\n", - " os.chdir(\"..\")\n", - "assert os.path.basename(os.getcwd()) == \"workflow-array-ephys\", (\n", - " \"Please move to the \" + \"workflow directory\"\n", - ")" + " os.chdir(\"..\")" ] }, { @@ -44,15 +33,7 @@ "outputs": [], "source": [ "# We'll be working with long tables, so we'll make visualization easier with a limit\n", - "import datajoint as dj" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ + "import datajoint as dj\n", "dj.config[\"display.limit\"] = 10" ] }, @@ -108,135 +89,7 @@ "outputs": [ { "data": { - "image/svg+xml": [ - "\n", - "\n", - "\n", - "\n", - "\n", - "3\n", - "\n", - "3\n", - "\n", - "\n", - "\n", - "ccf.ParentBrainRegion\n", - "\n", - "\n", - "ccf.ParentBrainRegion\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "3->ccf.ParentBrainRegion\n", - "\n", - "\n", - "\n", - "\n", - "ccf.BrainRegionAnnotation.BrainRegion\n", - "\n", - "\n", - "ccf.BrainRegionAnnotation.BrainRegion\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "ccf.BrainRegionAnnotation.BrainRegion->3\n", - "\n", - "\n", - "\n", - "\n", - "ccf.BrainRegionAnnotation.BrainRegion->ccf.ParentBrainRegion\n", - "\n", - "\n", - "\n", - "\n", - "ccf.BrainRegionAnnotation.Voxel\n", - "\n", - "\n", - "ccf.BrainRegionAnnotation.Voxel\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "ccf.BrainRegionAnnotation.BrainRegion->ccf.BrainRegionAnnotation.Voxel\n", - "\n", - "\n", - "\n", - "\n", - "ccf.CCF.Voxel\n", - "\n", - "\n", - "ccf.CCF.Voxel\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "ccf.CCF.Voxel->ccf.BrainRegionAnnotation.Voxel\n", - "\n", - "\n", - "\n", - "\n", - "ccf.CCF\n", - "\n", - "\n", - "ccf.CCF\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "ccf.CCF->ccf.CCF.Voxel\n", - "\n", - "\n", - "\n", - "\n", - "ccf.BrainRegionAnnotation\n", - "\n", - "\n", - "ccf.BrainRegionAnnotation\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "ccf.CCF->ccf.BrainRegionAnnotation\n", - "\n", - "\n", - "\n", - "\n", - "ccf.BrainRegionAnnotation->ccf.BrainRegionAnnotation.BrainRegion\n", - "\n", - "\n", - "\n", - "" - ], + "image/svg+xml": "\n\n\n\n\n3\n\n3\n\n\n\nccf.ParentBrainRegion\n\n\nccf.ParentBrainRegion\n\n\n\n\n\n3->ccf.ParentBrainRegion\n\n\n\n\nccf.BrainRegionAnnotation.BrainRegion\n\n\nccf.BrainRegionAnnotation.BrainRegion\n\n\n\n\n\nccf.BrainRegionAnnotation.BrainRegion->3\n\n\n\n\nccf.BrainRegionAnnotation.BrainRegion->ccf.ParentBrainRegion\n\n\n\n\nccf.BrainRegionAnnotation.Voxel\n\n\nccf.BrainRegionAnnotation.Voxel\n\n\n\n\n\nccf.BrainRegionAnnotation.BrainRegion->ccf.BrainRegionAnnotation.Voxel\n\n\n\n\nccf.CCF.Voxel\n\n\nccf.CCF.Voxel\n\n\n\n\n\nccf.CCF.Voxel->ccf.BrainRegionAnnotation.Voxel\n\n\n\n\nccf.CCF\n\n\nccf.CCF\n\n\n\n\n\nccf.CCF->ccf.CCF.Voxel\n\n\n\n\nccf.BrainRegionAnnotation\n\n\nccf.BrainRegionAnnotation\n\n\n\n\n\nccf.CCF->ccf.BrainRegionAnnotation\n\n\n\n\nccf.BrainRegionAnnotation->ccf.BrainRegionAnnotation.BrainRegion\n\n\n\n", "text/plain": [ "" ] @@ -647,214 +500,7 @@ "outputs": [ { "data": { - "image/svg+xml": [ - "\n", - "\n", - "\n", - "\n", - "\n", - "35\n", - "\n", - "35\n", - "\n", - "\n", - "\n", - "ccf.ParentBrainRegion\n", - "\n", - "\n", - "ccf.ParentBrainRegion\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "35->ccf.ParentBrainRegion\n", - "\n", - "\n", - "\n", - "\n", - "ccf.BrainRegionAnnotation.BrainRegion\n", - "\n", - "\n", - "ccf.BrainRegionAnnotation.BrainRegion\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "ccf.BrainRegionAnnotation.BrainRegion->35\n", - "\n", - "\n", - "\n", - "\n", - "ccf.BrainRegionAnnotation.BrainRegion->ccf.ParentBrainRegion\n", - "\n", - "\n", - "\n", - "\n", - "ccf.BrainRegionAnnotation.Voxel\n", - "\n", - "\n", - "ccf.BrainRegionAnnotation.Voxel\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "ccf.BrainRegionAnnotation.BrainRegion->ccf.BrainRegionAnnotation.Voxel\n", - "\n", - "\n", - "\n", - "\n", - "eloc.probe.ProbeType.Electrode\n", - "\n", - "\n", - "eloc.probe.ProbeType.Electrode\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "eloc.ElectrodePosition.Electrode\n", - "\n", - "\n", - "eloc.ElectrodePosition.Electrode\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "eloc.probe.ProbeType.Electrode->eloc.ElectrodePosition.Electrode\n", - "\n", - "\n", - "\n", - "\n", - "ccf.CCF.Voxel\n", - "\n", - "\n", - "ccf.CCF.Voxel\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "ccf.CCF.Voxel->eloc.ElectrodePosition.Electrode\n", - "\n", - "\n", - "\n", - "\n", - "ccf.CCF.Voxel->ccf.BrainRegionAnnotation.Voxel\n", - "\n", - "\n", - "\n", - "\n", - "eloc.ProbeInsertion\n", - "\n", - "\n", - "eloc.ProbeInsertion\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "eloc.ElectrodePosition\n", - "\n", - "\n", - "eloc.ElectrodePosition\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "eloc.ProbeInsertion->eloc.ElectrodePosition\n", - "\n", - "\n", - "\n", - "\n", - "ccf.CCF\n", - "\n", - "\n", - "ccf.CCF\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "ccf.CCF->ccf.CCF.Voxel\n", - "\n", - "\n", - "\n", - "\n", - "ccf.BrainRegionAnnotation\n", - "\n", - "\n", - "ccf.BrainRegionAnnotation\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "ccf.CCF->ccf.BrainRegionAnnotation\n", - "\n", - "\n", - "\n", - "\n", - "ccf.CCF->eloc.ElectrodePosition\n", - "\n", - "\n", - "\n", - "\n", - "ccf.BrainRegionAnnotation->ccf.BrainRegionAnnotation.BrainRegion\n", - "\n", - "\n", - "\n", - "\n", - "eloc.ElectrodePosition->eloc.ElectrodePosition.Electrode\n", - "\n", - "\n", - "\n", - "" - ], + "image/svg+xml": "\n\n\n\n\n35\n\n35\n\n\n\nccf.ParentBrainRegion\n\n\nccf.ParentBrainRegion\n\n\n\n\n\n35->ccf.ParentBrainRegion\n\n\n\n\nccf.BrainRegionAnnotation.BrainRegion\n\n\nccf.BrainRegionAnnotation.BrainRegion\n\n\n\n\n\nccf.BrainRegionAnnotation.BrainRegion->35\n\n\n\n\nccf.BrainRegionAnnotation.BrainRegion->ccf.ParentBrainRegion\n\n\n\n\nccf.BrainRegionAnnotation.Voxel\n\n\nccf.BrainRegionAnnotation.Voxel\n\n\n\n\n\nccf.BrainRegionAnnotation.BrainRegion->ccf.BrainRegionAnnotation.Voxel\n\n\n\n\neloc.probe.ProbeType.Electrode\n\n\neloc.probe.ProbeType.Electrode\n\n\n\n\n\neloc.ElectrodePosition.Electrode\n\n\neloc.ElectrodePosition.Electrode\n\n\n\n\n\neloc.probe.ProbeType.Electrode->eloc.ElectrodePosition.Electrode\n\n\n\n\nccf.CCF.Voxel\n\n\nccf.CCF.Voxel\n\n\n\n\n\nccf.CCF.Voxel->eloc.ElectrodePosition.Electrode\n\n\n\n\nccf.CCF.Voxel->ccf.BrainRegionAnnotation.Voxel\n\n\n\n\neloc.ProbeInsertion\n\n\neloc.ProbeInsertion\n\n\n\n\n\neloc.ElectrodePosition\n\n\neloc.ElectrodePosition\n\n\n\n\n\neloc.ProbeInsertion->eloc.ElectrodePosition\n\n\n\n\nccf.CCF\n\n\nccf.CCF\n\n\n\n\n\nccf.CCF->ccf.CCF.Voxel\n\n\n\n\nccf.BrainRegionAnnotation\n\n\nccf.BrainRegionAnnotation\n\n\n\n\n\nccf.CCF->ccf.BrainRegionAnnotation\n\n\n\n\nccf.CCF->eloc.ElectrodePosition\n\n\n\n\nccf.BrainRegionAnnotation->ccf.BrainRegionAnnotation.BrainRegion\n\n\n\n\neloc.ElectrodePosition->eloc.ElectrodePosition.Electrode\n\n\n\n", "text/plain": [ "" ] @@ -1309,7 +955,7 @@ "formats": "ipynb,py_scripts//py" }, "kernelspec": { - "display_name": "Python 3.10.4 64-bit ('python3p10')", + "display_name": "Python 3.9.12 ('ele')", "language": "python", "name": "python3" }, @@ -1323,11 +969,11 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.4" + "version": "3.9.12" }, "vscode": { "interpreter": { - "hash": "ff52d424e56dd643d8b2ec122f40a2e279e94970100b4e6430cb9025a65ba4cf" + "hash": "61456c693db5d9aa6731701ec9a9b08ab88a172bee0780139a3679beb166da16" } } }, diff --git a/notebooks/09-NWB-export.ipynb b/notebooks/09-NWB-export.ipynb index 261b993f..1d388743 100644 --- a/notebooks/09-NWB-export.ipynb +++ b/notebooks/09-NWB-export.ipynb @@ -20,16 +20,6 @@ "First, let's change directories to find the `dj_local_conf` file." ] }, - { - "cell_type": "code", - "execution_count": 1, - "id": "921a4a03", - "metadata": {}, - "outputs": [], - "source": [ - "import os" - ] - }, { "cell_type": "code", "execution_count": null, @@ -37,10 +27,9 @@ "metadata": {}, "outputs": [], "source": [ + "import os\n", "# change to the upper level folder to detect dj_local_conf.json\n", - "if os.path.basename(os.getcwd())=='notebooks': os.chdir('..')\n", - "assert os.path.basename(os.getcwd())=='workflow-array-ephys', (\"Please move to the \"\n", - " + \"workflow directory\")" + "if os.path.basename(os.getcwd())=='notebooks': os.chdir('..')" ] }, { @@ -51,16 +40,7 @@ "outputs": [], "source": [ "# We'll be working with long tables, so we'll make visualization easier with a limit\n", - "import datajoint as dj" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "129d4f3e", - "metadata": {}, - "outputs": [], - "source": [ + "import datajoint as dj\n", "dj.config['display.limit']=10" ] }, diff --git a/notebooks/py_scripts/00-data-download-optional.py b/notebooks/py_scripts/00-data-download-optional.py index 6c0e9313..26fab78c 100644 --- a/notebooks/py_scripts/00-data-download-optional.py +++ b/notebooks/py_scripts/00-data-download-optional.py @@ -6,9 +6,9 @@ # extension: .py # format_name: light # format_version: '1.5' -# jupytext_version: 1.14.0 +# jupytext_version: 1.13.7 # kernelspec: -# display_name: Python 3.10.4 64-bit ('python3p10') +# display_name: Python 3.9.12 ('ele') # language: python # name: python3 # --- @@ -16,6 +16,8 @@ # # Download example data # # This workflow will need Ephys data collected from either SpikeGLX or OpenEphys and the output from kilosort2. We provided an example dataset to be downloaded to run through the pipeline. This notebook walks you through the process to download the dataset. +# +# [CodeBook](codebook.datajoint.io) users can skip this step. # ## Install djarchive-client @@ -49,7 +51,7 @@ client.download( "workflow-array-ephys-benchmark", target_directory="/tmp/test_data", - revision="v1", + revision='v1', ) # ## Directory organization diff --git a/notebooks/py_scripts/01-configure.py b/notebooks/py_scripts/01-configure.py index 8fe31247..f47febd8 100644 --- a/notebooks/py_scripts/01-configure.py +++ b/notebooks/py_scripts/01-configure.py @@ -1,14 +1,13 @@ # --- # jupyter: # jupytext: -# formats: ipynb,py_scripts//py # text_representation: # extension: .py # format_name: light # format_version: '1.5' -# jupytext_version: 1.14.0 +# jupytext_version: 1.13.7 # kernelspec: -# display_name: Python 3.10.4 64-bit ('python3p10') +# display_name: Python 3.9.12 ('ele') # language: python # name: python3 # --- @@ -28,11 +27,7 @@ import os # change to the upper level folder -if os.path.basename(os.getcwd()) == "notebooks": - os.chdir("..") -assert os.path.basename(os.getcwd()) == "workflow-array-ephys", ( - "Please move to the " + "workflow directory" -) +if os.path.basename(os.getcwd()) == "notebooks": os.chdir("..") import datajoint as dj # ## Setup - Credentials @@ -40,8 +35,7 @@ # Now let's set up the host, user and password in the `dj.config` global variable import getpass - -dj.config["database.host"] = "{YOUR_HOST}" +dj.config["database.host"] = "{YOUR_HOST}" # CodeBook users should omit this dj.config["database.user"] = "{YOUR_USERNAME}" dj.config["database.password"] = getpass.getpass() # enter the password securily @@ -57,9 +51,10 @@ # # Giving a prefix to schema could help on the configuration of privilege settings. For example, if we set prefix `neuro_`, every schema created with the current workflow will start with `neuro_`, e.g. `neuro_lab`, `neuro_subject`, `neuro_ephys` etc. # -# The prefix could be configurated as follows in `dj.config`: +# The prefix could be configurated in `dj.config` as follows. CodeBook users should keep their username as the prefix for schema for declaration permissions. -dj.config["custom"] = {"database.prefix": "neuro_"} +username_as_prefix = dj.config["database.user"] + "_" +dj.config["custom"] = {"database.prefix": username_as_prefix} # ### Root directories for raw/processed data # @@ -69,10 +64,10 @@ # # The root path typically **do not** contain information of subjects or sessions, all data from subjects/sessions should be subdirectories in the root path. # -# In the example dataset downloaded with [these instructions](00-data-download-optional.ipynb), `/tmp/test_data` will be the root +# - In the example dataset downloaded with [these instructions](00-data-download-optional.ipynb), `/tmp/test_data` will be the root. +# - For CodeBook users, the root is `/home/inbox/` # # ``` -# /tmp/test_data/ # - subject6 # - session1 # - towersTask_g0_imec0 @@ -83,18 +78,13 @@ # If there is only one root path. dj.config["custom"]["ephys_root_data_dir"] = "/tmp/test_data" # If there are multiple possible root paths: -dj.config["custom"]["ephys_root_data_dir"] = ["/tmp/test_data1", "/tmp/test_data2"] +dj.config["custom"]["ephys_root_data_dir"] = ["/tmp/test_data", "/home/inbox/"] dj.config -[markdown] -# # + In the database, every path for the ephys raw data is **relative to root path(s)**. The benefit is that the absolute path could be configured for each machine, and when data transfer happens, we just need to change the root directory in the config file. -# -# # + The workflow supports **multiple root directories**. If there are multiple possible root directories, specify the `ephys_root_data_dir` as a list. -# -# # + The root path(s) are **specific to each machine**, as the name of mounted drive could be different for different operating systems or machines. +# + In the database, every path for the ephys raw data is **relative to root path(s)** to allow for the absolute path to be configured for **each machine**. When transferring data, we just need to change the root directory in the config file. # -# # + In the context of the workflow, all the paths saved into the database or saved in the config file need to be in the **POSIX standards** (Unix/Linux), with `/`. The path conversion for machines of any operating system is taken care of inside the elements. +# + DataJoint Elements use `pathlib.Path()` to maintain path information in **POSIX standards** (Unix/Linux), with `/`. The path conversion for machines of any operating system is taken care of inside the elements. # ### Ephys Mode # diff --git a/notebooks/py_scripts/02-workflow-structure-optional.py b/notebooks/py_scripts/02-workflow-structure-optional.py index fb00d71c..cac2f022 100644 --- a/notebooks/py_scripts/02-workflow-structure-optional.py +++ b/notebooks/py_scripts/02-workflow-structure-optional.py @@ -7,130 +7,103 @@ # extension: .py # format_name: light # format_version: '1.5' -# jupytext_version: 1.14.0 +# jupytext_version: 1.13.7 # kernelspec: -# display_name: Python 3.10.4 64-bit ('python3p10') +# display_name: Python 3.9.12 ('ele') # language: python # name: python3 # --- # # Introduction to the workflow structure -# This notebook gives a brief overview of the workflow structure and introduces some useful DataJoint tools to facilitate the exploration. -# + DataJoint needs to be pre-configured before running this notebook, if you haven't set up the configuration, refer to notebook [01-configure](01-configure.ipynb). -# + If you are familar with DataJoint and the workflow structure, proceed to the next notebook [03-process](03-process.ipynb) directly to run the workflow. -# + For a more thorough introduction of DataJoint functions, please visit our [general tutorial site](https://codebook.datajoint.io) +# This notebook gives an overview of the workflow structure and introduces useful DataJoint tools for exploration. +# + DataJoint needs to be configured before running this notebook. If you haven't done so, refer to notebook [01-configure](01-configure.ipynb). +# + If you are familar with DataJoint workflow structures, proceed to the next notebook [03-process](03-process.ipynb) directly to run this workflow. +# + For a more thorough introduction of DataJoint functions, please visit our [general documentation](https://docs.datajoint.org/python/v0.13/index.html) # To load the local configuration, we will change the directory to the package root. import os - if os.path.basename(os.getcwd()) == "notebooks": os.chdir("..") -# ## Schemas and tables +# ## Schemas, Diagrams and Tables -# The current workflow is composed of multiple database schemas, each of them corresponds to a module within `workflow_array_ephys.pipeline` +# Schemas are conceptually related sets of tables. By importing schemas from `workflow_array_ephys.pipeline`, we'll declare the tables on the server with the prefix in the config (if we have permission to do so). If these tables are already declared, we'll gain access. +# +# - `dj.list_schemas()` lists all schemas a user has access to in the current database +# - `.schema.list_tables()` will provide names for each table in the format used under the hood. import datajoint as dj from workflow_array_ephys.pipeline import lab, subject, session, probe, ephys -# + Each module contains a schema object that enables interaction with the schema in the database. - probe.schema # + Each module imported above corresponds to one schema inside the database. For example, `ephys` corresponds to `neuro_ephys` schema in the database. -ephys.schema - -# + The table classes in the module corresponds to a table in the schema in the database. - -# + Each datajoint table class inside the module corresponds to a table inside the schema. For example, the class `ephys.EphysRecording` correponds to the table `_ephys_recording` in the schema `neuro_ephys` in the database. -# preview table columns and contents in a table -ephys.EphysRecording() - -# + By importing the modules for the first time, the schemas and tables will be created inside the database. -# + Once created, importing modules will not create schemas and tables again, but the existing schemas/tables can be accessed and manipulated by the modules. - -# ## DataJoint tools to explore schemas and tables - -# + The schemas and tables will not be re-created when importing modules if they have existed. [markdown] -# # + `dj.list_schemas()`: list all schemas a user has access to in the current database -# + `dj.list_schemas()`: list all schemas a user could access. -dj.list_schemas() +ephys.schema.list_tables() +# - -# + `dj.Diagram()`: plot tables and dependencies. +# `dj.Diagram()` plots tables and dependencies in a schema. To see additional upstream or downstream connections, add `- N` or `+ N`. +# +# - `probe`: Neuropixels-based probe information +# - `ephys`: Electrophysiology data # + `dj.Diagram()`: plot tables and dependencies # plot diagram for all tables in a schema -dj.Diagram(ephys) +dj.Diagram(probe) # - -# **Table tiers**: +dj.Diagram(ephys) + dj.Diagram(session) - 1 + +# ### Table Types # -# Manual table: green box, manually inserted table, expect new entries daily, e.g. Subject, ProbeInsertion. -# Lookup table: gray box, pre inserted table, commonly used for general facts or parameters. e.g. Strain, ClusteringMethod, ClusteringParamSet. -# Imported table: blue oval, auto-processing table, the processing depends on the importing of external files. e.g. process of Clustering requires output files from kilosort2. -# Computed table: red circle, auto-processing table, the processing does not depend on files external to the database, commonly used for -# Part table: plain text, as an appendix to the master table, all the part entries of a given master entry represent a intact set of the master entry. e.g. Unit of a CuratedClustering. +# - **Manual table**: green box, manually inserted table, expect new entries daily, e.g. Subject, ProbeInsertion. +# - **Lookup table**: gray box, pre inserted table, commonly used for general facts or parameters. e.g. Strain, ClusteringMethod, ClusteringParamSet. +# - **Imported table**: blue oval, auto-processing table, the processing depends on the importing of external files. e.g. process of Clustering requires output files from kilosort2. +# - **Computed table**: red circle, auto-processing table, the processing does not depend on files external to the database, commonly used for +# - **Part table**: plain text, as an appendix to the master table, all the part entries of a given master entry represent a intact set of the master entry. e.g. Unit of a CuratedClustering. # -# **Dependencies**: +# ### Table Links # -# One-to-one primary: thick solid line, share the exact same primary key, meaning the child table inherits all the primary key fields from the parent table as its own primary key. -# One-to-many primary: thin solid line, inherit the primary key from the parent table, but have additional field(s) as part of the primary key as well -# secondary dependency: dashed line, the child table inherits the primary key fields from parent table as its own secondary attribute. - -# + `dj.Diagram()`: plot the diagram of the tables and dependencies. It could be used to plot tables in a schema or selected tables. -# plot diagram of tables in multiple schemas -dj.Diagram(subject) + dj.Diagram(session) + dj.Diagram(ephys) -# - +# - **One-to-one primary**: thick solid line, share the exact same primary key, meaning the child table inherits all the primary key fields from the parent table as its own primary key. +# - **One-to-many primary**: thin solid line, inherit the primary key from the parent table, but have additional field(s) as part of the primary key as well +# - **Secondary dependency**: dashed line, the child table inherits the primary key fields from parent table as its own secondary attribute. -# plot diagram of selected tables and schemas -dj.Diagram(subject.Subject) + dj.Diagram(session.Session) + dj.Diagram(ephys) +# ## Common Table Functions -# plot diagram with 1 additional level of dependency downstream -dj.Diagram(subject.Subject) + 1 - -# plot diagram with 2 additional levels of dependency upstream -dj.Diagram(ephys.EphysRecording) - 2 +# +# - `
()` show table contents +# - `heading` shows attribute definitions +# - `describe()` show table definition with foreign key references -# + `heading`: [markdown] -# # + `describe()`: show table definition with foreign key references. +# + Each datajoint table class inside the module corresponds to a table inside the schema. For example, the class `ephys.EphysRecording` correponds to the table `_ephys_recording` in the schema `neuro_ephys` in the database. +# preview table columns and contents in a table +ephys.EphysRecording() # - -ephys.EphysRecording.describe() -# + `heading`: show attribute definitions regardless of foreign key references +ephys.Clustering.heading -# + `heading`: show table attributes regardless of foreign key references. -ephys.EphysRecording.heading +ephys.WaveformSet.describe() # + probe [markdown] -# # Major DataJoint Elements installed in the current workflow -# + ephys [markdown] -# # + [`lab`](https://github.com/datajoint/element-lab): lab management related information, such as Lab, User, Project, Protocol, Source. +# ## Other Elements installed with the workflow +# +# - [`lab`](https://github.com/datajoint/element-lab): lab management related information, such as Lab, User, Project, Protocol, Source. +# - [`subject`](https://github.com/datajoint/element-animal): (element-animal) general animal information, User, Genetic background, Death etc. +# - [`session`](https://github.com/datajoint/element-session): General information of experimental sessions. +# +# For more information about these Elements, see [workflow session](https://github.com/datajoint/workflow-session). # - - dj.Diagram(lab) -# + [`animal`](https://github.com/datajoint/element-animal): general animal information, User, Genetic background, Death etc. - dj.Diagram(subject) # + [subject](https://github.com/datajoint/element-animal): contains the basic information of subject, including Strain, Line, Subject, Zygosity, and SubjectDeath information. subject.Subject.describe() - -# + [`session`](https://github.com/datajoint/element-session): General information of experimental sessions. +# - dj.Diagram(session) -# + [session](https://github.com/datajoint/element-session): experimental session information -session.Session.describe() - -# + [`ephys`](https://github.com/datajoint/element-array-ephys): Neuropixel based probe and ephys information - -# + [probe and ephys](https://github.com/datajoint/element-array-ephys): Neuropixel based probe and ephys tables -dj.Diagram(probe) + dj.Diagram(ephys) -# - - # ## Summary and next step # # + This notebook introduced the overall structures of the schemas and tables in the workflow and relevant tools to explore the schema structure and table definitions. diff --git a/notebooks/py_scripts/03-process.py b/notebooks/py_scripts/03-process.py index f9eaf1b9..577d2ab1 100644 --- a/notebooks/py_scripts/03-process.py +++ b/notebooks/py_scripts/03-process.py @@ -2,14 +2,13 @@ # --- # jupyter: # jupytext: -# formats: ipynb,py_scripts//py # text_representation: # extension: .py # format_name: light # format_version: '1.5' -# jupytext_version: 1.14.0 +# jupytext_version: 1.13.7 # kernelspec: -# display_name: Python 3.10.4 64-bit ('python3p10') +# display_name: Python 3.9.12 ('ele') # language: python # name: python3 # --- @@ -25,8 +24,7 @@ # Let's will change the directory to the package root to load configuration and also import relevant schemas. import os - -os.chdir("..") +if os.path.basename(os.getcwd()) == "notebooks": os.chdir("..") import datajoint as dj from workflow_array_ephys.pipeline import lab, subject, session, probe, ephys @@ -129,7 +127,7 @@ # ingest_subjects() # ingest_sessions() # ``` -# `ingest_sessions` also extracts probe and probe insertion information automatically from the meta file. +# `ingest_sessions` also extracts probe and probe insertion information automatically from the meta files. # # This is the regular routine for daily data processing, illustrated in notebook [04-automate](04-automate[optional].ipynb). diff --git a/notebooks/py_scripts/04-automate-optional.py b/notebooks/py_scripts/04-automate-optional.py index 9ffdb40a..fdf85fee 100644 --- a/notebooks/py_scripts/04-automate-optional.py +++ b/notebooks/py_scripts/04-automate-optional.py @@ -1,14 +1,13 @@ # --- # jupyter: # jupytext: -# formats: ipynb,py_scripts//py # text_representation: # extension: .py # format_name: light # format_version: '1.5' -# jupytext_version: 1.14.0 +# jupytext_version: 1.13.7 # kernelspec: -# display_name: Python 3.10.4 64-bit ('python3p10') +# display_name: Python 3.9.12 ('ele') # language: python # name: python3 # --- @@ -20,8 +19,7 @@ # - import os - -os.chdir("..") +if os.path.basename(os.getcwd()) == "notebooks": os.chdir("..") import numpy as np from workflow_array_ephys.pipeline import lab, subject, session, probe, ephys diff --git a/notebooks/py_scripts/05-explore.py b/notebooks/py_scripts/05-explore.py index f45d48bc..e41d4e2d 100644 --- a/notebooks/py_scripts/05-explore.py +++ b/notebooks/py_scripts/05-explore.py @@ -1,14 +1,13 @@ # --- # jupyter: # jupytext: -# formats: ipynb,py_scripts//py # text_representation: # extension: .py # format_name: light # format_version: '1.5' -# jupytext_version: 1.14.0 +# jupytext_version: 1.13.7 # kernelspec: -# display_name: Python 3.10.4 64-bit ('python3p10') +# display_name: Python 3.9.12 ('ele') # language: python # name: python3 # --- @@ -18,8 +17,7 @@ # This notebook will describe the steps for interacting with the data ingested into `workflow-array-ephys`. import os - -os.chdir("..") +if os.path.basename(os.getcwd()) == "notebooks": os.chdir("..") # + import datajoint as dj diff --git a/notebooks/py_scripts/06-drop-optional.py b/notebooks/py_scripts/06-drop-optional.py index a5a232d2..31a3ff23 100644 --- a/notebooks/py_scripts/06-drop-optional.py +++ b/notebooks/py_scripts/06-drop-optional.py @@ -1,14 +1,13 @@ # --- # jupyter: # jupytext: -# formats: ipynb,py_scripts//py # text_representation: # extension: .py # format_name: light # format_version: '1.5' -# jupytext_version: 1.14.0 +# jupytext_version: 1.13.7 # kernelspec: -# display_name: Python 3.10.4 64-bit ('python3p10') +# display_name: Python 3.9.12 ('ele') # language: python # name: python3 # --- @@ -22,8 +21,7 @@ # Change into the parent directory to find the `dj_local_conf.json` file. import os - -os.chdir("..") +if os.path.basename(os.getcwd()) == "notebooks": os.chdir("..") from workflow_array_ephys.pipeline import * diff --git a/notebooks/py_scripts/07-downstream-analysis.py b/notebooks/py_scripts/07-downstream-analysis.py index 1560a341..222e9c43 100644 --- a/notebooks/py_scripts/07-downstream-analysis.py +++ b/notebooks/py_scripts/07-downstream-analysis.py @@ -1,14 +1,13 @@ # --- # jupyter: # jupytext: -# formats: ipynb,py_scripts//py # text_representation: # extension: .py # format_name: light # format_version: '1.5' -# jupytext_version: 1.14.0 +# jupytext_version: 1.13.7 # kernelspec: -# display_name: Python 3.10.4 64-bit ('python3p10') +# display_name: Python 3.9.12 ('ele') # language: python # name: python3 # --- @@ -23,17 +22,10 @@ # First, let's change directories to find the `dj_local_conf` file. import os - -# change to the upper level folder to detect dj_local_conf.json -if os.path.basename(os.getcwd()) == "notebooks": - os.chdir("..") -assert os.path.basename(os.getcwd()) == "workflow-array-ephys", ( - "Please move to the " + "workflow directory" -) +if os.path.basename(os.getcwd()) == "notebooks": os.chdir("..") # We'll be working with long tables, so we'll make visualization easier with a limit import datajoint as dj - dj.config["display.limit"] = 10 # Next, we populate the python namespace with the required schemas diff --git a/notebooks/py_scripts/08-electrode-localization.py b/notebooks/py_scripts/08-electrode-localization.py index 9b10ac92..a38f5a2d 100644 --- a/notebooks/py_scripts/08-electrode-localization.py +++ b/notebooks/py_scripts/08-electrode-localization.py @@ -1,14 +1,13 @@ # --- # jupyter: # jupytext: -# formats: ipynb,py_scripts//py # text_representation: # extension: .py # format_name: light # format_version: '1.5' -# jupytext_version: 1.14.0 +# jupytext_version: 1.13.7 # kernelspec: -# display_name: Python 3.10.4 64-bit ('python3p10') +# display_name: Python 3.9.12 ('ele') # language: python # name: python3 # --- @@ -17,18 +16,13 @@ # Change into the parent directory to find the `dj_local_conf.json` file. -import os - # change to the upper level folder to detect dj_local_conf.json +import os if os.path.basename(os.getcwd()) == "notebooks": os.chdir("..") -assert os.path.basename(os.getcwd()) == "workflow-array-ephys", ( - "Please move to the " + "workflow directory" -) # We'll be working with long tables, so we'll make visualization easier with a limit import datajoint as dj - dj.config["display.limit"] = 10 # + [markdown] tags=[] jp-MarkdownHeadingCollapsed=true jp-MarkdownHeadingCollapsed=true tags=[] diff --git a/notebooks/py_scripts/09-NWB-export.py b/notebooks/py_scripts/09-NWB-export.py index 43dce65c..e2ca8a5b 100644 --- a/notebooks/py_scripts/09-NWB-export.py +++ b/notebooks/py_scripts/09-NWB-export.py @@ -1,12 +1,11 @@ # --- # jupyter: # jupytext: -# formats: ipynb,py_scripts//py # text_representation: # extension: .py # format_name: light # format_version: '1.5' -# jupytext_version: 1.14.0 +# jupytext_version: 1.13.7 # kernelspec: # display_name: Python 3.10.4 64-bit ('python3p10') # language: python @@ -22,39 +21,30 @@ # First, let's change directories to find the `dj_local_conf` file. import os - # change to the upper level folder to detect dj_local_conf.json -if os.path.basename(os.getcwd()) == "notebooks": - os.chdir("..") -assert os.path.basename(os.getcwd()) == "workflow-array-ephys", ( - "Please move to the " + "workflow directory" -) +if os.path.basename(os.getcwd())=='notebooks': os.chdir('..') # We'll be working with long tables, so we'll make visualization easier with a limit import datajoint as dj +dj.config['display.limit']=10 -dj.config["display.limit"] = 10 - -# If you haven't already populated the `lab`, `subject`, `session`, `probe`, and `ephys` schemas, please do so now with [04-automate](./04-automate-optional.ipynb). Note: exporting `ephys` data is currently only supported on the `no_curation` schema. +# If you haven't already populated the `lab`, `subject`, `session`, `probe`, and `ephys` schemas, please do so now with [04-automate](./04-automate-optional.ipynb). Note: exporting `ephys` data is currently only supported on the `no_curation` schema. from workflow_array_ephys.pipeline import lab, subject, session, probe, ephys -from workflow_array_ephys.export import ( - element_lab_to_nwb_dict, - subject_to_nwb, - session_to_nwb, - ecephys_session_to_nwb, - write_nwb, -) +from workflow_array_ephys.export import (element_lab_to_nwb_dict, subject_to_nwb, + session_to_nwb, ecephys_session_to_nwb, + write_nwb) from element_interface.dandi import upload_to_dandi # ## Export to NWB # # We'll use the following keys to demonstrate export functions. -lab_key = {"lab": "LabA"} -protocol_key = {"protocol": "ProtA"} -project_key = {"project": "ProjA"} -session_key = {"subject": "subject5", "session_datetime": "2018-07-03 20:32:28"} +lab_key={"lab": "LabA"} +protocol_key={"protocol": "ProtA"} +project_key={"project": "ProjA"} +session_key={"subject": "subject5", + "session_datetime": "2018-07-03 20:32:28"} # ### Upstream Elements # @@ -67,13 +57,12 @@ # Note: `pynwb` will display a warning regarding timezone information - datetime fields are assumed to be in local time, and will be converted to UTC. # -print("Lab:\n") -element_lab_to_nwb_dict( - lab_key=lab_key, protocol_key=protocol_key, project_key=project_key -) -print("\nAnimal:\n") +print('Lab:\n') +element_lab_to_nwb_dict(lab_key=lab_key, protocol_key=protocol_key, + project_key=project_key) +print('\nAnimal:\n') subject_to_nwb(session_key=session_key) -print("\nSession:\n") +print('\nSession:\n') session_to_nwb(session_key=session_key) # ### Element Array Electrophysiology @@ -83,17 +72,15 @@ help(ecephys_session_to_nwb) -nwbfile = ecephys_session_to_nwb( - session_key=session_key, - raw=True, - spikes=True, - lfp="dj", - end_frame=100, - lab_key=lab_key, - project_key=project_key, - protocol_key=protocol_key, - nwbfile_kwargs=None, -) +nwbfile = ecephys_session_to_nwb(session_key=session_key, + raw=True, + spikes=True, + lfp="dj", + end_frame=100, + lab_key=lab_key, + project_key=project_key, + protocol_key=protocol_key, + nwbfile_kwargs=None) nwbfile @@ -101,7 +88,7 @@ # + import time - + write_nwb(nwbfile, f'./temp_nwb/{time.strftime("_test_%Y%m%d-%H%M%S.nwb")}') # - @@ -116,16 +103,17 @@ # # These values can be added to your `dj.config` as follows: -dj.config["custom"]["dandiset_id"] = "" -dj.config["custom"]["dandi.api"] = "<40-character alphanumeric string>" +dj.config['custom']['dandiset_id']="" +dj.config['custom']['dandi.api']="<40-character alphanumeric string>" # This would facilitate routine updating of your dandiset. upload_to_dandi( data_directory="./temp_nwb/", - dandiset_id=dj.config["custom"]["dandiset_id"], + dandiset_id=dj.config['custom']['dandiset_id'], staging=True, working_directory="./temp_nwb/", - api_key=dj.config["custom"]["dandi.api"], - sync=False, -) + api_key=dj.config['custom']['dandi.api'], + sync=False) + + diff --git a/requirements.txt b/requirements.txt index e5cc9a7a..340936e5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,11 @@ datajoint>=0.13.0 -element-array-ephys>=0.1.0b0 +element-array-ephys @ git+https://github.com/datajoint/element-array-ephys@run_kilosort element-lab>=0.1.0b0 element-animal>=0.1.0b0 element-session>=0.1.0b0 element-event @ git+https://github.com/datajoint/element-event.git element-interface @ git+https://github.com/datajoint/element-interface.git ipykernel>=6.0.1 +nwb-conversion-tools==0.11.1 +spikeinterface==0.93.0 +neo==0.10.2 \ No newline at end of file diff --git a/workflow_array_ephys/export.py b/workflow_array_ephys/export.py index a2f5e0f1..3cf76a3a 100644 --- a/workflow_array_ephys/export.py +++ b/workflow_array_ephys/export.py @@ -2,14 +2,20 @@ from element_lab.export.nwb import element_lab_to_nwb_dict from element_animal.export.nwb import subject_to_nwb from element_session.export.nwb import session_to_nwb - -# Import ephys NWB export functions -from element_array_ephys.export.nwb import ecephys_session_to_nwb, write_nwb +from .pipeline import ephys_mode __all__ = [ "element_lab_to_nwb_dict", "subject_to_nwb", "session_to_nwb", - "ecephys_session_to_nwb", - "write_nwb", ] + +# Import ephys NWB export functions +if ephys_mode == "no-curation": + from element_array_ephys.export.nwb import ecephys_session_to_nwb, write_nwb +else: + print( + f"Warning: ephys export requires the no-curation ephys_mode. To use it,\n\t" + + "try setting datajoint.config['custom']['ephys_mode'] to 'no-curation'\n\t" + + "and restarting your kernel." + ) diff --git a/workflow_array_ephys/pipeline.py b/workflow_array_ephys/pipeline.py index 6d412e36..e498038e 100644 --- a/workflow_array_ephys/pipeline.py +++ b/workflow_array_ephys/pipeline.py @@ -16,9 +16,6 @@ get_electrode_localization_dir, ) -# session and ephys nwb exports check for these in linking_module -from .export import element_lab_to_nwb_dict, subject_to_nwb, session_to_nwb - if "custom" not in dj.config: dj.config["custom"] = {} diff --git a/workflow_array_ephys/version.py b/workflow_array_ephys/version.py index 0822c478..fd331289 100644 --- a/workflow_array_ephys/version.py +++ b/workflow_array_ephys/version.py @@ -2,4 +2,4 @@ Package metadata Update the Docker image tag in `docker-compose.yaml` to match """ -__version__ = "0.2.0" +__version__ = "0.2.1"