Dataset Publication Tutorial

docs/conf.py

@@ -16,7 +16,7 @@
     "sphinx.ext.intersphinx",  # Allows links to other sphinx project documentation sites
     "sphinx_search.extension",  # Allows for auto search function the documentation
     "sphinx.ext.viewcode",  # Shows source code in the documentation
-    "sphinx.ext.extlinks",  # Allows to use shorter external links defined in the extlinks variable.
+"sphinx.ext.extlinks",  # Allows to use shorter external links defined in the extlinks variable.
 ]
 
 templates_path = ["_templates"]

docs/conf_extlinks.py

@@ -6,6 +6,8 @@
     "matnwb-src": ("https://github.com/NeurodataWithoutBorders/matnwb/%s", "%s"),
     "nwb-overview": ("https://nwb-overview.readthedocs.io/en/latest/%s", "%s"),
     "path-expansion-guide": ("https://neuroconv.readthedocs.io/en/main/user_guide/expand_path.html%s", "%s"),
+    "dandi-staging": ("https://gui-staging.dandiarchive.org/%s", "%s"),
+    "dandi-archive": ("https://dandiarchive.org/%s", "%s"),
     "conda-install": (
         "https://docs.conda.io/projects/conda/en/latest/user-guide/install/index.html#regular-installation%s",
         "%s",

docs/tutorials/dataset_publication.rst

@@ -1,3 +1,80 @@
 Dataset Publication
 =======================================
-Coming soon...
+
+For this tutorial, we'll be adapting the previous :doc:`Multi-Session Tutorial </tutorials/multiple_sessions>` to publish our data to the DANDI Archive.
+
+.. note::
+    This tutorial focuses on uploading to the Staging server.
+
+    **When working with real data, you'll want to publish to the Main Archive**. In this case, follow the same steps outlined here—except replace the Staging server with the Main Archive.
+
+.. note::
+    Gaining access to DANDI requires approval from the archive administrators. Separate approval is required for both the main archive and the staging server.
+
+    **This tutorial requires an account on the** :dandi-staging:`DANDI staging server <>`.
+
+    We’re going to use the Staging server for this tutorial so we don’t crowd the main DANDI Archive with `synthetic` datasets! However, you’ll want to publish your `real` data on the main server—which will require a separate approval process.
+
+    Once you receive notice that your account was approved, you can move on to the next steps.
+
+Workflow Setup
+--------------
+1. Resume the conversion via the **Convert** page
+
+2. Navigate to the **Workflow** page.
+
+   a. Specify that you’d like to publish your data to the :dandi-archive:`DANDI Archive <>`.
+
+3. Navigate back to the **Conversion Review** page
+
+You'll now notice that the **Exit Pipeline** button has been replaced with **Next**, allowing you to move forward with publication on the DANDI Archive.
+
+DANDI Upload
+------------
+You’ll need to specify your DANDI API keys if you haven’t uploaded from the GUIDE before. These keys are unique between the Main and Staging servers.
+
+.. figure:: ../assets/tutorials/dandi/api-tokens.png
+  :align: center
+  :alt: A pop-up asking for DANDI API keys
+
+To get your API key, visit the :dandi-staging:`staging website <>` and click on the profile icon in the top-right corner. This will show a dropdown with a copy button, which will assign your API key to the clipboard.
+
+.. figure:: ../assets/dandi/api-token-location.png
+  :align: center
+  :alt: DANDI staging API key added
+
+Submit this to the Staging API Key input on the GUIDE.
+
+.. figure:: ../assets/tutorials/dandi/api-token-added.png
+  :align: center
+  :alt: DANDI staging API key added
+
+
+Once you have specified your Staging API Key, the **Dandiset** input will allow you to select any existing Dandiset associated with your account by ID (e.g., "207698") or name (e.g., "NWB GUIDE Test").
+
+Continue to the next page to trigger your upload to the DANDI Archive.
+
+.. figure:: ../assets/tutorials/dandi/dandiset-id.png
+  :align: center
+  :alt: DANDI upload page with Dandiset ID specified
+
+Creating a Dandiset from the GUIDE
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+If you do not already own a Dandiset on staging, you will need to create one. Press the Create New Dandiset button to open a pop-up that guides you through the required fields for Dandiset creation.
+
+.. figure:: ../assets/tutorials/dandi/create-dandiset.png
+  :align: center
+  :alt: Dandiset creation pop-up
+
+
+Once this pop-up form is submitted, the Dandiset input will now contain your new Dandiset.
+
+Final Review
+------------
+Once your upload to the DANDI Archive is complete, you will be able to review a quick overview of the associated Dandiset and a list of the uploaded files from this pipeline.
+
+.. figure:: ../assets/tutorials/dandi/review-page.png
+  :align: center
+  :alt: DANDI upload review page
+
+Congratulations on your first upload to the DANDI Archive from the GUIDE!

src/renderer/src/stories/List.ts

@@ -156,14 +156,18 @@ export class List extends LitElement {
       const oldObject = this.object
       this.#updateObject()
 
-      this.onChange({
-        items: this.#items,
-        object: this.object
-      },
-      {
-        items: oldList,
-        object: oldObject
-      })
+      if (this.#initialized) {
+
+        this.onChange({
+          items: this.#items,
+          object: this.object
+        },
+        {
+          items: oldList,
+          object: oldObject
+        })
+      }
+
       this.requestUpdate('items', oldList)
     }
 
@@ -177,6 +181,8 @@ export class List extends LitElement {
 
     declare listStyles: any
 
+    #initialized = false
+
     allowDrop = (ev) => ev.preventDefault();
 
 
@@ -230,6 +236,8 @@ export class List extends LitElement {
 
       if (props.onChange) this.onChange = props.onChange
 
+      this.#initialized = true
+
     }
 
     add = (item: ListItemType) => {

src/renderer/src/stories/pages/guided-mode/data/GuidedStructure.js

@@ -52,10 +52,7 @@ export class GuidedStructurePage extends Page {
         },
     });
 
-    list = new List({
-        emptyMessage: defaultEmptyMessage,
-        onChange: () => (this.unsavedUpdates = "conversions"),
-    });
+    list = new List({ emptyMessage: defaultEmptyMessage });
 
     addButton = new Button();
 
@@ -146,6 +143,8 @@ export class GuidedStructurePage extends Page {
 
         this.list.emptyMessage = defaultEmptyMessage;
 
+        const items = [];
+
         for (const [key, name] of Object.entries(interfaces)) {
             let found = this.search.options?.find((item) => item.value === name);
 
@@ -158,17 +157,27 @@ export class GuidedStructurePage extends Page {
                 };
             }
 
-            this.list.add({ ...found, key }); // Add previously selected items
+            items.push({ ...found, key });
         }
 
+        const ogList = this.list;
+
+        this.list = new List({
+            items,
+            emptyMessage: defaultEmptyMessage,
+            onChange: () => (this.unsavedUpdates = "conversions"),
+        });
+
+        this.list.style.display = "inline-block";
+
+        ogList.replaceWith(this.list);
+
         this.addButton.removeAttribute("hidden");
         super.updated(); // Call if updating data
     }
 
     render() {
         // Reset list
-        this.list.style.display = "inline-block";
-        this.list.clear();
         this.addButton.setAttribute("hidden", "");
 
         return html`

tests/e2e/config.ts

@@ -4,7 +4,7 @@ import { homedir } from 'node:os'
 import { existsSync } from 'node:fs'
 
 import paths from "../../paths.config.json" assert { type: "json" };
-import { connect } from '../puppeteer';
+import { connect as connectToElectron } from '../puppeteer';
 
 // ------------------------------------------------------------------
 // ------------------------ Path Definitions ------------------------
@@ -119,4 +119,4 @@ export const publish = dandiInfo.token ? true : false
 if (!publish) console.log('No DANDI API key provided. Will skip dataset publication step...')
 
 
-export const references = connect()
+export const references = connectToElectron()

tests/e2e/e2e.test.ts

@@ -5,8 +5,19 @@ import { mkdirSync, existsSync, rmSync } from 'node:fs'
 import { join } from 'node:path'
 
 import * as config from './config'
-import runWorkflow from './workflow'
-import { evaluate, takeScreenshot } from './utils'
+import runWorkflow, { uploadToDandi } from './workflow'
+import { evaluate, takeScreenshot, to, toNextPage } from './utils'
+
+const x = 250 // Sidebar size
+const width = config.windowDims.width - x
+
+const datasetScreenshotClip = {
+  x,
+  y: 0,
+  width,
+  height: 220
+}
+
 
 beforeAll(() => {
 
@@ -22,8 +33,6 @@ beforeAll(() => {
 
 describe('E2E Test', () => {
 
-  // NOTE: This is where you should be connecting...
-
   test('Ensure number of test pipelines starts at zero', async () => {
 
     await sleep(500) // Wait for full notification to render
@@ -34,58 +43,49 @@ describe('E2E Test', () => {
     expect(nPipelines).toBe(0)
   })
 
-  describe('Manually run through the pipeline', async () => {
-
-    const datasetTestFunction = config.regenerateTestData ? test : test.skip
 
-    datasetTestFunction('Create tutorial dataset', async () => {
+  const datasetTestFunction = config.regenerateTestData ? test : test.skip
 
-      const x = 250 // Sidebar size
-      const width = config.windowDims.width - x
+  datasetTestFunction('Create tutorial dataset', async () => {
 
-      const screenshotClip = {
-        x,
-        y: 0,
-        width,
-        height: 220
-      }
 
+    await evaluate(async () => {
 
-      await evaluate(async () => {
+      // Transition to settings page
+      const dashboard = document.querySelector('nwb-dashboard')
+      dashboard.sidebar.select('settings')
 
-        // Transition to settings page
-        const dashboard = document.querySelector('nwb-dashboard')
-        dashboard.sidebar.select('settings')
+      // Generate test data
+      const page = dashboard.page
+      page.deleteTestData()
+    })
 
-        // Generate test data
-        const page = dashboard.page
-        page.deleteTestData()
-      })
+    await takeScreenshot('dataset-creation', 300, { clip: datasetScreenshotClip })
 
-      await takeScreenshot('dataset-creation', 300, { clip: screenshotClip })
+    const outputLocation = await evaluate(async () => {
+      const dashboard = document.querySelector('nwb-dashboard')
+      const page = dashboard.page
+      const outputLocation = await page.generateTestData()
+      page.requestUpdate()
+      return outputLocation
+    })
 
-      const outputLocation = await evaluate(async () => {
-        const dashboard = document.querySelector('nwb-dashboard')
-        const page = dashboard.page
-        const outputLocation = await page.generateTestData()
-        page.requestUpdate()
-        return outputLocation
-      })
+    // Take image after dataset generation
+    await takeScreenshot('dataset-created', 500, { clip: datasetScreenshotClip })
 
-      // Take image after dataset generation
-      await takeScreenshot('dataset-created', 500, { clip: screenshotClip })
+    expect(existsSync(outputLocation)).toBe(true)
 
-      expect(existsSync(outputLocation)).toBe(true)
+    // Navigate back to the home page
+    let pageId = await evaluate(() => {
+      const dashboard = document.querySelector('nwb-dashboard')
+      dashboard.sidebar.select('/')
+      return dashboard.page.info.id
+    })
 
-      // Navigate back to the home page
-      let pageId = await evaluate(() => {
-        const dashboard = document.querySelector('nwb-dashboard')
-        dashboard.sidebar.select('/')
-        return dashboard.page.info.id
-      })
+    expect(pageId).toBe('/')
+  })
 
-      expect(pageId).toBe('/')
-    })
+  describe('Run through several pipeline workflows', async () => {
 
     describe('Complete a single-session workflow', async () => {
       const subdirectory = 'single'
@@ -109,6 +109,58 @@ describe('E2E Test', () => {
       })
     })
 
+    describe('Upload the multi-session output to DANDI', async () => {
+
+      const subdirectory = 'dandi'
+
+      test('Restart pipeline', async () => {
+
+        await evaluate(async () => {
+          const pipelines = document.getElementById('guided-div-resume-progress-cards').children
+          const found = Array.from(pipelines).find(card => card.info.project.name === 'Multi Session Workflow')
+          console.log(found, Array.from(pipelines))
+          found.querySelector('button').click()
+        })
+
+      })
+
+      test('Update the workflow to allow DANDI upload', async () => {
+
+        await sleep(1000)
+        await to('//workflow')
+
+        await evaluate(async ( workflow ) => {
+          const dashboard = document.querySelector('nwb-dashboard')
+          const page = dashboard.page
+
+          for (let key in workflow) {
+            const input = page.form.getFormElement([ key ])
+            input.updateData(workflow[key])
+          }
+
+          page.form.requestUpdate() // Ensure the form is updated visually
+
+          await page.save()
+
+        }, { upload_to_dandi: true })
+
+        await toNextPage('structure') // Save data without a popup
+        await to('//conversion')
+
+        // Do not prompt to save
+        await evaluate(() => {
+          const dashboard = document.querySelector('nwb-dashboard')
+          const page = dashboard.page
+          page.unsavedUpdates = false
+        })
+
+        await to('//upload') // NOTE: It would be nice to avoid having to re-run the conversion...
+
+      })
+
+      uploadToDandi(subdirectory) // Upload to DANDI if the API key is provided
+
+    })
 
   })