diff --git a/DANDI User Guide.ipynb b/DANDI User Guide.ipynb
index f439afc..e6413f1 100644
--- a/DANDI User Guide.ipynb
+++ b/DANDI User Guide.ipynb
@@ -15,7 +15,7 @@
"This is part 3 of the DANDI User Training on Nov 1, 2021.\n",
"\n",
"## Neurodata Without Borders (NWB)\n",
- "NWB is a BRAIN Initiative-backed data standard designed to package all of the data and metadata associated with neurophysiology experiments into a single file that enables sharing and re-analysis of the data. Neurophysiology data submitted to the DANDI archive is required to be in NWB. This includes extracellular and intracellular electrophysiology, optical physiology, and behavior. NWB defines a data organization schema that ensure the crucial metadata is packaged in a standardized way. This includes not just the neurophysiology recordings, but also metadata about the subjects, equipment, task structure, etc.\n",
+ "DANDI has chosen NWB as our supported format for exchanging neurophysiology data. NWB is a BRAIN Initiative-backed data standard designed to package all of the data and metadata associated with neurophysiology experiments into a single file that enables sharing and re-analysis of the data. This includes extracellular and intracellular electrophysiology, optical physiology, and behavior. NWB defines a data organization schema that ensure the crucial metadata is packaged in a standardized way. This includes not just the neurophysiology recordings, but also metadata about the subjects, equipment, task structure, etc.\n",
"\n",
"\n",
"\n",
@@ -26,7 +26,7 @@
},
{
"cell_type": "code",
- "execution_count": 5,
+ "execution_count": 22,
"id": "aa5bd1ad-cdb4-413e-8a8f-c8340fb8f739",
"metadata": {},
"outputs": [],
@@ -41,61 +41,61 @@
"from pynwb.ecephys import ElectricalSeries\n",
"from pynwb import NWBHDF5IO\n",
"\n",
+ "def create_nwbfile(subject_id):\n",
+ " nwbfile = NWBFile(\n",
+ " session_description='my first synthetic recording',\n",
+ " identifier=str(uuid.uuid4()),\n",
+ " session_start_time=datetime.now(tzlocal()),\n",
+ " experimenter='Dr. Bilbo Baggins',\n",
+ " lab='Bag End Laboratory',\n",
+ " institution='University of Middle Earth at the Shire',\n",
+ " experiment_description='I went on an adventure with thirteen dwarves to reclaim vast treasures.',\n",
+ " session_id='001',\n",
+ " )\n",
+ "\n",
+ " nwbfile.subject = Subject(\n",
+ " subject_id=subject_id,\n",
+ " species='Mus musculus',\n",
+ " age='P90D',\n",
+ " )\n",
+ "\n",
+ " device = nwbfile.create_device(name='trodes_rig123')\n",
"\n",
- "nwbfile = NWBFile(\n",
- " session_description='my first synthetic recording',\n",
- " identifier=str(uuid.uuid4()),\n",
- " session_start_time=datetime.now(tzlocal()),\n",
- " experimenter='Dr. Bilbo Baggins',\n",
- " lab='Bag End Laboratory',\n",
- " institution='University of Middle Earth at the Shire',\n",
- " experiment_description='I went on an adventure with thirteen dwarves to reclaim vast treasures.',\n",
- " session_id='LONELYMTN-001'\n",
- ")\n",
- "\n",
- "nwbfile.subject = Subject(\n",
- " subject_id='001',\n",
- " species='Mus musculus',\n",
- " age='P90D',\n",
- ")\n",
- "\n",
- "device = nwbfile.create_device(name='trodes_rig123')\n",
- "electrode_name = 'tetrode1'\n",
- "description = \"an example tetrode\"\n",
- "location = \"hippocampus\"\n",
- "\n",
- "electrode_group = nwbfile.create_electrode_group(\n",
- " electrode_name,\n",
- " description=description,\n",
- " location=location,\n",
- " device=device\n",
- ")\n",
- "\n",
- "for idx in [1, 2, 3, 4]:\n",
- " nwbfile.add_electrode(\n",
- " id=idx,\n",
- " x=1.0, y=2.0, z=3.0,\n",
- " imp=float(-idx),\n",
- " location='CA1', filtering='none',\n",
- " group=electrode_group\n",
+ " electrode_group = nwbfile.create_electrode_group(\n",
+ " name='tetrode1',\n",
+ " description=\"an example tetrode\",\n",
+ " location=\"hippocampus\",\n",
+ " device=device,\n",
" )\n",
+ "\n",
+ " for _ in range(4):\n",
+ " nwbfile.add_electrode(\n",
+ " x=1.0, y=2.0, z=3.0,\n",
+ " imp=np.nan,\n",
+ " location='CA1',\n",
+ " filtering='none',\n",
+ " group=electrode_group,\n",
+ " )\n",
+ "\n",
+ " electrode_table_region = nwbfile.create_electrode_table_region([0, 2], 'the first and third electrodes')\n",
+ "\n",
+ "\n",
+ " rate = 10.0\n",
+ " data_len = 1000\n",
+ " ephys_data = np.random.rand(data_len * 2).reshape((data_len, 2))\n",
+ " ephys_timestamps = np.arange(data_len) / rate\n",
+ "\n",
+ " ephys_ts = ElectricalSeries(\n",
+ " name='test_ephys_data',\n",
+ " data=ephys_data,\n",
+ " electrodes=electrode_table_region,\n",
+ " timestamps=ephys_timestamps,\n",
+ " description=\"Random numbers generated with numpy.random.rand\"\n",
+ " )\n",
+ " nwbfile.add_acquisition(ephys_ts)\n",
" \n",
- "electrode_table_region = nwbfile.create_electrode_table_region([0, 2], 'the first and third electrodes')\n",
- "\n",
- "\n",
- "rate = 10.0\n",
- "data_len = 1000\n",
- "ephys_data = np.random.rand(data_len * 2).reshape((data_len, 2))\n",
- "ephys_timestamps = np.arange(data_len) / rate\n",
- "\n",
- "ephys_ts = ElectricalSeries(\n",
- " 'test_ephys_data',\n",
- " ephys_data,\n",
- " electrode_table_region,\n",
- " timestamps=ephys_timestamps,\n",
- " description=\"Random numbers generated with numpy.random.rand\"\n",
- ")\n",
- "nwbfile.add_acquisition(ephys_ts)"
+ " return nwbfile\n",
+ " \n"
]
},
{
@@ -111,57 +111,32 @@
"| Writing extracellular electrophysiology | [23 min video](https://www.youtube.com/watch?v=rlywed3ar-s&ab_channel=NeurodataWithoutBorders)
[Jupyter notebook](https://github.com/NeurodataWithoutBorders/nwb_tutorial/blob/master/HCK08/ecephys_tutorial.ipynb) | [46 min video](https://www.youtube.com/watch?v=W8t4_quIl1k&ab_channel=NeurodataWithoutBorders)
[Written tutorial](https://neurodatawithoutborders.github.io/matnwb/tutorials/html/ecephys.html) |\n",
"| Writing intracellular electrophysiology | [Jupyter notebook](https://neurodatawithoutborders.github.io/matnwb/tutorials/html/ecephys.html) | [Written tutorial](https://neurodatawithoutborders.github.io/matnwb/tutorials/html/icephys.html) |\n",
"| Writing optical physiology | [31 min video](https://www.youtube.com/watch?v=HPjSxBjdFpM&ab_channel=NeurodataWithoutBorders)
[Jupyter notebook](https://github.com/NeurodataWithoutBorders/nwb_tutorial/blob/master/HCK08/ophys_tutorial.ipynb) | [39 min video](https://www.youtube.com/watch?v=OBidHdocnTc&ab_channel=NeurodataWithoutBorders)
[Written tutorial](https://neurodatawithoutborders.github.io/matnwb/tutorials/html/ophys.html) |\n",
- "| Advanced write | [26 min video](https://www.youtube.com/watch?v=wduZHfNOaNg&ab_channel=NeurodataWithoutBorders) | [16 min video](https://www.youtube.com/watch?v=PIE_F4iVv98&ab_channel=NeurodataWithoutBorders)
[Written tutorial](https://neurodatawithoutborders.github.io/matnwb/tutorials/html/dataPipe.html) |"
- ]
- },
- {
- "cell_type": "markdown",
- "id": "2b15685a-e439-4d90-a0fe-cc8e4762ed44",
- "metadata": {},
- "source": [
- "## Visualization NWB files\n",
+ "| Advanced write | [26 min video](https://www.youtube.com/watch?v=wduZHfNOaNg&ab_channel=NeurodataWithoutBorders) | [16 min video](https://www.youtube.com/watch?v=PIE_F4iVv98&ab_channel=NeurodataWithoutBorders)
[Written tutorial](https://neurodatawithoutborders.github.io/matnwb/tutorials/html/dataPipe.html) |\n",
"\n",
- "NWB Widgets is a library of interactive data visualizations that works automatically with any NWB file. This can be very useful for visually confirming any conversion."
+ "If you think NWB might not meet your needs, please contact us on our [help desk](https://github.com/dandi/helpdesk/discussions)!"
]
},
{
"cell_type": "code",
- "execution_count": 6,
- "id": "2d5165d5-74a2-4b1e-9e04-548ee844dfcd",
+ "execution_count": 20,
+ "id": "b9ebc17f-4b15-4c2c-b028-3b284785f22e",
"metadata": {},
"outputs": [
{
- "name": "stderr",
+ "name": "stdout",
"output_type": "stream",
"text": [
- "/opt/conda/lib/python3.9/site-packages/xarray/backends/cfgrib_.py:27: UserWarning: Failed to load cfgrib - most likely there is a problem accessing the ecCodes library. Try `import cfgrib` to get the full error message\n",
- " warnings.warn(\n"
+ "/home/jovyan/dandi-notebooks\n"
]
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "b4d4726a47784c8a9cd7b3be055c8ccc",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "VBox(children=(HBox(children=(Label(value='session_description:', layout=Layout(max_height='40px', max_width='…"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
}
],
"source": [
- "from nwbwidgets import nwb2widget\n",
- "\n",
- "nwb2widget(nwbfile)"
+ "!pwd"
]
},
{
"cell_type": "code",
- "execution_count": 8,
+ "execution_count": 24,
"id": "2fb3ab28-5e0a-4484-b0fa-9d4fc113e658",
"metadata": {},
"outputs": [],
@@ -170,6 +145,8 @@
"\n",
"mkdir('../data')\n",
"\n",
+ "nwbfile = create_nwbfile(subject_id='001')\n",
+ "\n",
"with NWBHDF5IO('../data/ecephys_example.nwb', 'w') as io:\n",
" io.write(nwbfile)"
]
@@ -208,6 +185,7 @@
"1. Go to DANDI staging (https://gui-staging.dandiarchive.org/) and use your GitHub account to log in.\n",
"\n",
"1. Click on your initials in the upper right and copy your API key.\n",
+ "\n",
"\n",
"\n",
"1. Now create a new dataset by clicking the NEW DATASET button\n",
@@ -233,24 +211,22 @@
"export DANDI_API_KEY=d71c0db17827aac067896f612f48af667890000\n",
"```\n",
"\n",
- "Confirm that this worked with\n",
+ "Confirm that this worked with the following line, which should print your key.\n",
"\n",
"```bash\n",
"echo $DANDI_API_KEY\n",
"```\n",
"\n",
- "which should print out your key.\n",
- "\n",
"6. Copy the URL from the dataset you have created (something like `https://gui-staging.dandiarchive.org/#/dandiset/100507`)\n",
"\n",
"and in the Terminal window, run\n",
"\n",
"```bash\n",
- "dandi download https://gui-staging.dandiarchive.org/#/dandiset/100507 # <-- your dandiset ID here\n",
+ "dandi download \"https://gui-staging.dandiarchive.org/#/dandiset/100507\" # <-- your dandiset ID here\n",
"cd 100507 # <-- your dandiset ID here\n",
"dandi organize ../data -f dry\n",
"dandi organize ../data\n",
- "DANDI_DEVEL=1 dandi upload -i dandi-staging # (on the non-staging server, this line would simply be dandi upload)\n",
+ "dandi upload -i dandi-staging # (on the non-staging server, this line would simply be: dandi upload)\n",
"```\n",
"\n",
"7. Now refresh the landing page of your dandiset and you should see that it is no longer empty.\n",
@@ -260,13 +236,143 @@
"You can explore all the NWB files within by clicking Files\n",
"\n",
"\n",
- "\n"
+ "\n",
+ "8. Adding more data to a dandiset"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 26,
+ "id": "53cbeb15-679d-432b-8b9d-daf943829590",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "nwbfile = create_nwbfile(subject_id='002')\n",
+ "\n",
+ "with NWBHDF5IO('../data/ecephys_example2.nwb', 'w') as io:\n",
+ " io.write(nwbfile)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "36623a61-98f9-4dcb-915c-5a97cffa9a33",
+ "metadata": {},
+ "source": [
+ "Then go back to terminal and run\n",
+ "\n",
+ "```bash\n",
+ "dandi organize ../data\n",
+ "dandi upload -i dandi-staging\n",
+ "```"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "7d980104-839e-44ae-8031-ae3d20517eb7",
+ "metadata": {},
+ "source": [
+ "## Part 3, Visualization NWB files\n",
+ "\n",
+ "NWB Widgets is a library of interactive data visualizations that works automatically with any NWB file. This can be very useful for visually confirming any conversion."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "id": "b141a5a5-1e84-4796-b53e-ded9360ece51",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "application/vnd.jupyter.widget-view+json": {
+ "model_id": "202325d29684459f9af43420939b054d",
+ "version_major": 2,
+ "version_minor": 0
+ },
+ "text/plain": [
+ "VBox(children=(HBox(children=(Label(value='session_description:', layout=Layout(max_height='40px', max_width='…"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "from nwbwidgets import nwb2widget\n",
+ "\n",
+ "nwb2widget(nwbfile)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "1fa27484-8e53-4a6d-a755-a5718806c69f",
+ "metadata": {},
+ "source": [
+ "It can also be used to explore any file shared on DANDI. You can use the DANDI API to access the s3 path of any file and stream it directly into NWB Widgets."
]
},
+ {
+ "cell_type": "code",
+ "execution_count": 11,
+ "id": "bbc614dc-577a-47fd-bbe6-51a45c263f21",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# calcium imaging, Giocomo Lab (30 GB)\n",
+ "#dandiset_id, filepath = \"000054\", \"sub-F2/sub-F2_ses-20190407T210000_behavior+ophys.nwb\"\n",
+ "\n",
+ "# neuropixel, Giocomo Lab (46 GB)\n",
+ "#dandiset_id, filepath = \"000053\", \"sub-npI1/sub-npI1_ses-20190415_behavior+ecephys.nwb\"\n",
+ "\n",
+ "# ecephys, Buzsaki Lab (15.2 GB)\n",
+ "dandiset_id, filepath = \"000003\", \"sub-YutaMouse41/sub-YutaMouse41_ses-YutaMouse41-150831_behavior+ecephys.nwb\""
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 27,
+ "id": "1fa74623-67c1-46af-8b06-6a6f33e721f1",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "application/vnd.jupyter.widget-view+json": {
+ "model_id": "edce05c4de4d4cd7a3a9162b27fe2eda",
+ "version_major": 2,
+ "version_minor": 0
+ },
+ "text/plain": [
+ "VBox(children=(HBox(children=(Label(value='session_description:', layout=Layout(max_height='40px', max_width='…"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "from dandi.dandiapi import DandiAPIClient\n",
+ "\n",
+ "with DandiAPIClient() as client:\n",
+ " asset = client.get_dandiset(dandiset_id, \"draft\").get_asset_by_path(filepath)\n",
+ " s3_url = asset.get_content_url(follow_redirects=1, strip_query=True)\n",
+ "\n",
+ "io = NWBHDF5IO(s3_url, mode='r', load_namespaces=True, driver='ros3')\n",
+ "nwb = io.read()\n",
+ "nwb2widget(nwb)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "2c564363-710d-449b-9b4d-98e93504118f",
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
{
"cell_type": "code",
"execution_count": null,
- "id": "615b1f9f-8dcb-4153-96df-570cd3cbce49",
+ "id": "c2b6b9ec-2f72-446d-bd50-3bc19e70befc",
"metadata": {},
"outputs": [],
"source": []