Skip to content

Commit

Permalink
update user guide
Browse files Browse the repository at this point in the history
  • Loading branch information
@bendichter
bendichter committed Oct 27, 2021
1 parent 9d46dff commit 0c67b47
Showing 1 changed file with 200 additions and 94 deletions.
294 changes: 200 additions & 94 deletions DANDI User Guide.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
"This is part 3 of the DANDI User Training on Nov 1, 2021.\n",
"\n",
"## Neurodata Without Borders (NWB)\n",
"NWB is a BRAIN Initiative-backed data standard designed to package all of the data and metadata associated with neurophysiology experiments into a single file that enables sharing and re-analysis of the data. Neurophysiology data submitted to the DANDI archive is required to be in NWB. This includes extracellular and intracellular electrophysiology, optical physiology, and behavior. NWB defines a data organization schema that ensure the crucial metadata is packaged in a standardized way. This includes not just the neurophysiology recordings, but also metadata about the subjects, equipment, task structure, etc.\n",
"DANDI has chosen NWB as our supported format for exchanging neurophysiology data. NWB is a BRAIN Initiative-backed data standard designed to package all of the data and metadata associated with neurophysiology experiments into a single file that enables sharing and re-analysis of the data. This includes extracellular and intracellular electrophysiology, optical physiology, and behavior. NWB defines a data organization schema that ensure the crucial metadata is packaged in a standardized way. This includes not just the neurophysiology recordings, but also metadata about the subjects, equipment, task structure, etc.\n",
"\n",
"\n",
"![image.png](attachment:867900b4-7979-4346-9893-81edbba268bd.png)\n",
Expand All @@ -26,7 +26,7 @@
},
{
"cell_type": "code",
"execution_count": 5,
"execution_count": 22,
"id": "aa5bd1ad-cdb4-413e-8a8f-c8340fb8f739",
"metadata": {},
"outputs": [],
Expand All @@ -41,61 +41,61 @@
"from pynwb.ecephys import ElectricalSeries\n",
"from pynwb import NWBHDF5IO\n",
"\n",
"def create_nwbfile(subject_id):\n",
" nwbfile = NWBFile(\n",
" session_description='my first synthetic recording',\n",
" identifier=str(uuid.uuid4()),\n",
" session_start_time=datetime.now(tzlocal()),\n",
" experimenter='Dr. Bilbo Baggins',\n",
" lab='Bag End Laboratory',\n",
" institution='University of Middle Earth at the Shire',\n",
" experiment_description='I went on an adventure with thirteen dwarves to reclaim vast treasures.',\n",
" session_id='001',\n",
" )\n",
"\n",
" nwbfile.subject = Subject(\n",
" subject_id=subject_id,\n",
" species='Mus musculus',\n",
" age='P90D',\n",
" )\n",
"\n",
" device = nwbfile.create_device(name='trodes_rig123')\n",
"\n",
"nwbfile = NWBFile(\n",
" session_description='my first synthetic recording',\n",
" identifier=str(uuid.uuid4()),\n",
" session_start_time=datetime.now(tzlocal()),\n",
" experimenter='Dr. Bilbo Baggins',\n",
" lab='Bag End Laboratory',\n",
" institution='University of Middle Earth at the Shire',\n",
" experiment_description='I went on an adventure with thirteen dwarves to reclaim vast treasures.',\n",
" session_id='LONELYMTN-001'\n",
")\n",
"\n",
"nwbfile.subject = Subject(\n",
" subject_id='001',\n",
" species='Mus musculus',\n",
" age='P90D',\n",
")\n",
"\n",
"device = nwbfile.create_device(name='trodes_rig123')\n",
"electrode_name = 'tetrode1'\n",
"description = \"an example tetrode\"\n",
"location = \"hippocampus\"\n",
"\n",
"electrode_group = nwbfile.create_electrode_group(\n",
" electrode_name,\n",
" description=description,\n",
" location=location,\n",
" device=device\n",
")\n",
"\n",
"for idx in [1, 2, 3, 4]:\n",
" nwbfile.add_electrode(\n",
" id=idx,\n",
" x=1.0, y=2.0, z=3.0,\n",
" imp=float(-idx),\n",
" location='CA1', filtering='none',\n",
" group=electrode_group\n",
" electrode_group = nwbfile.create_electrode_group(\n",
" name='tetrode1',\n",
" description=\"an example tetrode\",\n",
" location=\"hippocampus\",\n",
" device=device,\n",
" )\n",
"\n",
" for _ in range(4):\n",
" nwbfile.add_electrode(\n",
" x=1.0, y=2.0, z=3.0,\n",
" imp=np.nan,\n",
" location='CA1',\n",
" filtering='none',\n",
" group=electrode_group,\n",
" )\n",
"\n",
" electrode_table_region = nwbfile.create_electrode_table_region([0, 2], 'the first and third electrodes')\n",
"\n",
"\n",
" rate = 10.0\n",
" data_len = 1000\n",
" ephys_data = np.random.rand(data_len * 2).reshape((data_len, 2))\n",
" ephys_timestamps = np.arange(data_len) / rate\n",
"\n",
" ephys_ts = ElectricalSeries(\n",
" name='test_ephys_data',\n",
" data=ephys_data,\n",
" electrodes=electrode_table_region,\n",
" timestamps=ephys_timestamps,\n",
" description=\"Random numbers generated with numpy.random.rand\"\n",
" )\n",
" nwbfile.add_acquisition(ephys_ts)\n",
" \n",
"electrode_table_region = nwbfile.create_electrode_table_region([0, 2], 'the first and third electrodes')\n",
"\n",
"\n",
"rate = 10.0\n",
"data_len = 1000\n",
"ephys_data = np.random.rand(data_len * 2).reshape((data_len, 2))\n",
"ephys_timestamps = np.arange(data_len) / rate\n",
"\n",
"ephys_ts = ElectricalSeries(\n",
" 'test_ephys_data',\n",
" ephys_data,\n",
" electrode_table_region,\n",
" timestamps=ephys_timestamps,\n",
" description=\"Random numbers generated with numpy.random.rand\"\n",
")\n",
"nwbfile.add_acquisition(ephys_ts)"
" return nwbfile\n",
" \n"
]
},
{
Expand All @@ -111,57 +111,32 @@
"| Writing extracellular electrophysiology | [23 min video](https://www.youtube.com/watch?v=rlywed3ar-s&ab_channel=NeurodataWithoutBorders)<br>[Jupyter notebook](https://github.com/NeurodataWithoutBorders/nwb_tutorial/blob/master/HCK08/ecephys_tutorial.ipynb) | [46 min video](https://www.youtube.com/watch?v=W8t4_quIl1k&ab_channel=NeurodataWithoutBorders)<br>[Written tutorial](https://neurodatawithoutborders.github.io/matnwb/tutorials/html/ecephys.html) |\n",
"| Writing intracellular electrophysiology | [Jupyter notebook](https://neurodatawithoutborders.github.io/matnwb/tutorials/html/ecephys.html) | [Written tutorial](https://neurodatawithoutborders.github.io/matnwb/tutorials/html/icephys.html) |\n",
"| Writing optical physiology | [31 min video](https://www.youtube.com/watch?v=HPjSxBjdFpM&ab_channel=NeurodataWithoutBorders)<br>[Jupyter notebook](https://github.com/NeurodataWithoutBorders/nwb_tutorial/blob/master/HCK08/ophys_tutorial.ipynb) | [39 min video](https://www.youtube.com/watch?v=OBidHdocnTc&ab_channel=NeurodataWithoutBorders)<br>[Written tutorial](https://neurodatawithoutborders.github.io/matnwb/tutorials/html/ophys.html) |\n",
"| Advanced write | [26 min video](https://www.youtube.com/watch?v=wduZHfNOaNg&ab_channel=NeurodataWithoutBorders) | [16 min video](https://www.youtube.com/watch?v=PIE_F4iVv98&ab_channel=NeurodataWithoutBorders)<br>[Written tutorial](https://neurodatawithoutborders.github.io/matnwb/tutorials/html/dataPipe.html) |"
]
},
{
"cell_type": "markdown",
"id": "2b15685a-e439-4d90-a0fe-cc8e4762ed44",
"metadata": {},
"source": [
"## Visualization NWB files\n",
"| Advanced write | [26 min video](https://www.youtube.com/watch?v=wduZHfNOaNg&ab_channel=NeurodataWithoutBorders) | [16 min video](https://www.youtube.com/watch?v=PIE_F4iVv98&ab_channel=NeurodataWithoutBorders)<br>[Written tutorial](https://neurodatawithoutborders.github.io/matnwb/tutorials/html/dataPipe.html) |\n",
"\n",
"NWB Widgets is a library of interactive data visualizations that works automatically with any NWB file. This can be very useful for visually confirming any conversion."
"If you think NWB might not meet your needs, please contact us on our [help desk](https://github.com/dandi/helpdesk/discussions)!"
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "2d5165d5-74a2-4b1e-9e04-548ee844dfcd",
"execution_count": 20,
"id": "b9ebc17f-4b15-4c2c-b028-3b284785f22e",
"metadata": {},
"outputs": [
{
"name": "stderr",
"name": "stdout",
"output_type": "stream",
"text": [
"/opt/conda/lib/python3.9/site-packages/xarray/backends/cfgrib_.py:27: UserWarning: Failed to load cfgrib - most likely there is a problem accessing the ecCodes library. Try `import cfgrib` to get the full error message\n",
" warnings.warn(\n"
"/home/jovyan/dandi-notebooks\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "b4d4726a47784c8a9cd7b3be055c8ccc",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"VBox(children=(HBox(children=(Label(value='session_description:', layout=Layout(max_height='40px', max_width='…"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"from nwbwidgets import nwb2widget\n",
"\n",
"nwb2widget(nwbfile)"
"!pwd"
]
},
{
"cell_type": "code",
"execution_count": 8,
"execution_count": 24,
"id": "2fb3ab28-5e0a-4484-b0fa-9d4fc113e658",
"metadata": {},
"outputs": [],
Expand All @@ -170,6 +145,8 @@
"\n",
"mkdir('../data')\n",
"\n",
"nwbfile = create_nwbfile(subject_id='001')\n",
"\n",
"with NWBHDF5IO('../data/ecephys_example.nwb', 'w') as io:\n",
" io.write(nwbfile)"
]
Expand Down Expand Up @@ -208,6 +185,7 @@
"1. Go to DANDI staging (https://gui-staging.dandiarchive.org/) and use your GitHub account to log in.\n",
"\n",
"1. Click on your initials in the upper right and copy your API key.\n",
"\n",
"![image.png](attachment:32a72db5-d216-4a7d-b4f0-bb5ade3d0f14.png)\n",
"\n",
"1. Now create a new dataset by clicking the NEW DATASET button\n",
Expand All @@ -233,24 +211,22 @@
"export DANDI_API_KEY=d71c0db17827aac067896f612f48af667890000\n",
"```\n",
"\n",
"Confirm that this worked with\n",
"Confirm that this worked with the following line, which should print your key.\n",
"\n",
"```bash\n",
"echo $DANDI_API_KEY\n",
"```\n",
"\n",
"which should print out your key.\n",
"\n",
"6. Copy the URL from the dataset you have created (something like `https://gui-staging.dandiarchive.org/#/dandiset/100507`)\n",
"\n",
"and in the Terminal window, run\n",
"\n",
"```bash\n",
"dandi download https://gui-staging.dandiarchive.org/#/dandiset/100507 # <-- your dandiset ID here\n",
"dandi download \"https://gui-staging.dandiarchive.org/#/dandiset/100507\" # <-- your dandiset ID here\n",
"cd 100507 # <-- your dandiset ID here\n",
"dandi organize ../data -f dry\n",
"dandi organize ../data\n",
"DANDI_DEVEL=1 dandi upload -i dandi-staging # (on the non-staging server, this line would simply be dandi upload)\n",
"dandi upload -i dandi-staging # (on the non-staging server, this line would simply be: dandi upload)\n",
"```\n",
"\n",
"7. Now refresh the landing page of your dandiset and you should see that it is no longer empty.\n",
Expand All @@ -260,13 +236,143 @@
"You can explore all the NWB files within by clicking Files\n",
"\n",
"![image.png](attachment:38187baa-a811-4051-b657-7b21c0f484d6.png)\n",
"\n"
"\n",
"8. Adding more data to a dandiset"
]
},
{
"cell_type": "code",
"execution_count": 26,
"id": "53cbeb15-679d-432b-8b9d-daf943829590",
"metadata": {},
"outputs": [],
"source": [
"nwbfile = create_nwbfile(subject_id='002')\n",
"\n",
"with NWBHDF5IO('../data/ecephys_example2.nwb', 'w') as io:\n",
" io.write(nwbfile)"
]
},
{
"cell_type": "markdown",
"id": "36623a61-98f9-4dcb-915c-5a97cffa9a33",
"metadata": {},
"source": [
"Then go back to terminal and run\n",
"\n",
"```bash\n",
"dandi organize ../data\n",
"dandi upload -i dandi-staging\n",
"```"
]
},
{
"cell_type": "markdown",
"id": "7d980104-839e-44ae-8031-ae3d20517eb7",
"metadata": {},
"source": [
"## Part 3, Visualization NWB files\n",
"\n",
"NWB Widgets is a library of interactive data visualizations that works automatically with any NWB file. This can be very useful for visually confirming any conversion."
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "b141a5a5-1e84-4796-b53e-ded9360ece51",
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "202325d29684459f9af43420939b054d",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"VBox(children=(HBox(children=(Label(value='session_description:', layout=Layout(max_height='40px', max_width='…"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"from nwbwidgets import nwb2widget\n",
"\n",
"nwb2widget(nwbfile)"
]
},
{
"cell_type": "markdown",
"id": "1fa27484-8e53-4a6d-a755-a5718806c69f",
"metadata": {},
"source": [
"It can also be used to explore any file shared on DANDI. You can use the DANDI API to access the s3 path of any file and stream it directly into NWB Widgets."
]
},
{
"cell_type": "code",
"execution_count": 11,
"id": "bbc614dc-577a-47fd-bbe6-51a45c263f21",
"metadata": {},
"outputs": [],
"source": [
"# calcium imaging, Giocomo Lab (30 GB)\n",
"#dandiset_id, filepath = \"000054\", \"sub-F2/sub-F2_ses-20190407T210000_behavior+ophys.nwb\"\n",
"\n",
"# neuropixel, Giocomo Lab (46 GB)\n",
"#dandiset_id, filepath = \"000053\", \"sub-npI1/sub-npI1_ses-20190415_behavior+ecephys.nwb\"\n",
"\n",
"# ecephys, Buzsaki Lab (15.2 GB)\n",
"dandiset_id, filepath = \"000003\", \"sub-YutaMouse41/sub-YutaMouse41_ses-YutaMouse41-150831_behavior+ecephys.nwb\""
]
},
{
"cell_type": "code",
"execution_count": 27,
"id": "1fa74623-67c1-46af-8b06-6a6f33e721f1",
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "edce05c4de4d4cd7a3a9162b27fe2eda",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"VBox(children=(HBox(children=(Label(value='session_description:', layout=Layout(max_height='40px', max_width='…"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"from dandi.dandiapi import DandiAPIClient\n",
"\n",
"with DandiAPIClient() as client:\n",
" asset = client.get_dandiset(dandiset_id, \"draft\").get_asset_by_path(filepath)\n",
" s3_url = asset.get_content_url(follow_redirects=1, strip_query=True)\n",
"\n",
"io = NWBHDF5IO(s3_url, mode='r', load_namespaces=True, driver='ros3')\n",
"nwb = io.read()\n",
"nwb2widget(nwb)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "2c564363-710d-449b-9b4d-98e93504118f",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "615b1f9f-8dcb-4153-96df-570cd3cbce49",
"id": "c2b6b9ec-2f72-446d-bd50-3bc19e70befc",
"metadata": {},
"outputs": [],
"source": []
Expand Down

0 comments on commit 0c67b47

Please sign in to comment.