Merge branch 'main' into file-or-folder-handler

.github/workflows/Build-and-deploy-mac.yml

@@ -22,7 +22,6 @@ jobs:
       - uses: conda-incubator/setup-miniconda@v2
         with:
           activate-environment: nwb-guide
-          #mamba-version: "*"
           environment-file: environments/environment-MAC.yml
           auto-activate-base: false
 

.github/workflows/pyflask-build-and-dist-tests.yml

@@ -30,19 +30,16 @@ jobs:
             os: ubuntu-latest
             label: environments/environment-Linux.yml
             prefix: /usr/share/miniconda3/envs/nwb-guide
-            shorthand: unix
 
-          - python-version: "3.10"
+          - python-version: "3.9"
             os: macos-latest
             label: environments/environment-Mac.yml
             prefix: /Users/runner/miniconda3/envs/nwb-guide
-            shorthand: unix
 
-          - python-version: "3.10"
+          - python-version: "3.9"
             os: windows-latest
             label: environments/environment-Windows.yml
             prefix: C:\Miniconda3\envs\nwb-guide
-            shorthand: win
 
     steps:
       - uses: actions/checkout@v3
@@ -88,15 +85,7 @@ jobs:
         run: rm -f /Users/runner/miniconda3/envs/nwb-guide/lib/python3.9/site-packages/sonpy/linux/sonpy.so
 
       - name: Build PyFlask distribution
-        run: npm run build:flask:${{ matrix.shorthand }}
-
-      #- if: matrix.os == 'windows-latest'
-      #  name: Run test on build executable
-      #  run: node tests/testPyinstallerExecutable.js --script ./build/nwb-guide/nwb-guide.exe
-
-      #- if: matrix.os != 'windows-latest'
-      #  name: Run test on build executable
-      #  run: node tests/testPyinstallerExecutable.js --script ./build/nwb-guide/nwb-guide
+        run: npm run build:flask
 
       - name: Run test on distributed executable
         run: node tests/testPyinstallerExecutable.js

.gitignore

@@ -19,8 +19,6 @@ semantic.json
 build/
 yarn.lock
 
-*.spec
-
 *.pyc
 src/.DS_Store
 .DS_Store

nwb-guide.spec

@@ -0,0 +1,74 @@
+# -*- mode: python ; coding: utf-8 -*-
+import sys
+from pathlib import Path
+
+sys.setrecursionlimit(sys.getrecursionlimit() * 5)
+
+from PyInstaller.utils.hooks import collect_data_files
+from PyInstaller.utils.hooks import collect_all
+
+datas = [('./paths.config.json', '.'), ('./package.json', '.')]
+binaries = []
+hiddenimports = ['scipy._distributor_init', 'scipy._lib.messagestream', 'scipy._lib._ccallback', 'scipy._lib._testutils', 'email_validator']
+datas += collect_data_files('jsonschema_specifications')
+tmp_ret = collect_all('nwbinspector')
+datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2]
+tmp_ret = collect_all('neuroconv')
+datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2]
+tmp_ret = collect_all('pynwb')
+datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2]
+tmp_ret = collect_all('hdmf')
+datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2]
+tmp_ret = collect_all('ndx_dandi_icephys')
+datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2]
+tmp_ret = collect_all('ci_info')
+datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2]
+
+
+block_cipher = None
+
+
+a = Analysis(
+    [f"{Path('pyflask') / 'app.py'}"],
+    pathex=[],
+    binaries=binaries,
+    datas=datas,
+    hiddenimports=hiddenimports,
+    hookspath=[],
+    hooksconfig={},
+    runtime_hooks=[],
+    excludes=[],
+    win_no_prefer_redirects=False,
+    win_private_assemblies=False,
+    cipher=block_cipher,
+    noarchive=False,
+)
+pyz = PYZ(a.pure, a.zipped_data, cipher=block_cipher)
+
+exe = EXE(
+    pyz,
+    a.scripts,
+    [],
+    exclude_binaries=True,
+    name='nwb-guide',
+    debug=False,
+    bootloader_ignore_signals=False,
+    strip=False,
+    upx=True,
+    console=True,
+    disable_windowed_traceback=False,
+    argv_emulation=False,
+    target_arch=None,
+    codesign_identity=None,
+    entitlements_file=None,
+)
+coll = COLLECT(
+    exe,
+    a.binaries,
+    a.zipfiles,
+    a.datas,
+    strip=False,
+    upx=True,
+    upx_exclude=[],
+    name='nwb-guide',
+)

package.json

@@ -11,13 +11,14 @@
     "dev:app": "vite src/renderer",
     "dev:server": "cd pyflask && python -m flask run --port 4242",
     "build": "npm run build:app",
+    "echo": "python -c \"print('hello')\"",
     "build:app": "electron-vite build --outDir build",
-    "build:win": "npm run build && npm run build:flask:win && npm run build:electron:win",
-    "build:mac": "npm run build && npm run build:flask:unix && npm run build:electron:mac",
-    "build:linux": "npm run build && npm run build:flask:unix && npm run build:electron:linux",
-    "build:flask:base": "python -m PyInstaller --log-level DEBUG --name nwb-guide --onedir --clean --noconfirm ./pyflask/app.py --distpath ./build/flask --collect-data jsonschema_specifications --collect-all nwbinspector --collect-all neuroconv --collect-all pynwb --collect-all hdmf --collect-all ci_info --collect-all ndx_dandi_icephys --hidden-import scipy._distributor_init --hidden-import scipy._lib.messagestream --hidden-import scipy._lib._ccallback --hidden-import scipy._lib._testutils --hidden-import email_validator",
-    "build:flask:win": "npm run build:flask:base -- --add-data ./paths.config.json;. --add-data ./package.json;.",
-    "build:flask:unix": "npm run build:flask:base -- --add-data ./paths.config.json:. --add-data ./package.json:.",
+    "build:win": "npm run build && npm run build:flask && npm run build:electron:win",
+    "build:mac": "npm run build && npm run build:flask && npm run build:electron:mac",
+    "build:linux": "npm run build && npm run build:flask && npm run build:electron:linux",
+    "build:flask": "python -m PyInstaller nwb-guide.spec --log-level DEBUG --clean --noconfirm --distpath ./build/flask",
+    "build:flask:spec:base": "pyi-makespec --name nwb-guide --onedir --collect-data jsonschema_specifications --collect-all nwbinspector --collect-all neuroconv --collect-all pynwb --collect-all hdmf --collect-all ndx_dandi_icephys --collect-all ci_info --hidden-import scipy._distributor_init --hidden-import scipy._lib.messagestream --hidden-import scipy._lib._ccallback --hidden-import scipy._lib._testutils --hidden-import email_validator ./pyflask/app.py",
+    "build:flask:spec": "npm run build:flask:spec:base && python prepare_pyinstaller_spec.py",
     "build:electron:win": "electron-builder build --win --publish never",
     "build:electron:mac": "electron-builder build --mac --publish never",
     "build:electron:linux": "electron-builder build --linux --publish never",

prepare_pyinstaller_spec.py

@@ -0,0 +1,30 @@
+"""
+Calling `pyi-makespec` regenerates the base spec file, but we need to extend the recursion limit.
+
+This script is run automatically as a part of `npm run build:flask:spec` after the `pyi-makespec` call.
+"""
+from pathlib import Path
+
+with open(file=Path(__file__).parent / "nwb-guide.spec", mode="r") as io:
+    lines = io.readlines()
+
+lines.insert(1, "import sys\n")
+lines.insert(2, "from pathlib import Path\n")
+lines.insert(3, "\n")
+lines.insert(4, "sys.setrecursionlimit(sys.getrecursionlimit() * 5)\n")
+lines.insert(5, "\n")
+
+# Originally this was a separate `npm` command per platform to account for CLI syntax differences between ; and :
+# The spec file is, however, the same across platforms
+data_line_index = lines.index("datas = []\n")
+lines[data_line_index] = "datas = [('./paths.config.json', '.'), ('./package.json', '.')]\n"
+
+# Another platform specific difference is the app.py location
+app_py_line_index = next(index for index, line in enumerate(lines) if "app.py" in line)
+app_py_line = "    [f\"{Path('pyflask') / 'app.py'}\"],\n"
+lines[app_py_line_index] = app_py_line
+
+with open(file=Path(__file__).parent / "nwb-guide.spec", mode="w") as io:
+    io.writelines(lines)
+
+print("Sucessfully injected recursion depth extension and json paths!")

schemas/json/dandi/global.json

@@ -1,10 +1,20 @@
 {
+    "description": "Log in to DANDI, click on your user initials in the top-right corner, and copy your API key from the resulting pop-up. <br/><small><b>Note:</b> <a href='https://dandiarchive.org' target='_blank'>The main archive</a> and <a href='https://gui-staging.dandiarchive.org'  target='_blank'>the staging (testing) server</a> have different API keys.</small>",
     "properties": {
-        "api_key": {
-            "type": "string",
-            "format": "password",
-            "description": "Log in to DANDI, click on your user initials in the top-right corner, and copy your API key from the resulting pop-up. <br/><small><b>Note:</b> <a href='https://dandiarchive.org' target='_blank'>The main archive</a> and <a href='https://gui-staging.dandiarchive.org'  target='_blank'>the staging (testing) server</a> have different API keys.</small>"
+        "api_keys": {
+            "properties": {
+                "main_api_key": {
+                    "type": "string",
+                    "format": "password",
+                    "description": "From the <a href='https://dandiarchive.org' target='_blank'>main archive</a>"
+                },
+                "staging_api_key": {
+                    "type": "string",
+                    "format": "password",
+                    "description": "From the <a href='https://gui-staging.dandiarchive.org' target='_blank'>staging (testing) server</a>"
+                }
+            },
+            "required": ["main_api_key", "staging_api_key"]
         }
-    },
-    "required": ["api_key"]
+    }
 }

schemas/json/dandi/upload.json

@@ -1,7 +1,8 @@
 {
     "properties": {
         "dandiset_id": {
-            "type": "string"
+            "type": "string",
+            "description": "The unique identifier for your dandiset. Will automatically determine whether to upload to the main DANDI archive or the development staging server."
         },
         "cleanup": {
             "type": "boolean",

src/renderer/src/stories/JSONSchemaForm.js

@@ -114,6 +114,22 @@ pre {
   .required.conditional label:after {
     color: transparent;
   }
+
+  h4 {
+    margin: 0;
+    margin-bottom: 5px;
+    padding-bottom: 5px;
+    border-bottom: 1px solid gainsboro;
+  }
+
+  .guided--text-input-instructions {
+    font-size: 13px;
+    width: 100%;
+    padding-top: 4px;
+    color: dimgray !important;
+    margin: 0 0 1em;
+    line-height: 1.4285em;
+}
 `;
 
 document.addEventListener("dragover", (e) => {
@@ -238,10 +254,10 @@ export class JSONSchemaForm extends LitElement {
 
         // NOTE: Forms with nested forms will handle their own state updates
         if (!value) {
-            delete resultParent[name];
+            if (fullPath.length === 1) delete resultParent[name];
             delete resolvedParent[name];
         } else {
-            resultParent[name] = value;
+            if (fullPath.length === 1) resultParent[name] = value;
             resolvedParent[name] = value;
         }
 
@@ -468,18 +484,39 @@ export class JSONSchemaForm extends LitElement {
         }
     };
 
-    #getRenderable = (schema = {}, required, path) => {
+    #getRenderable = (schema = {}, required, path, recursive = false) => {
         const entries = Object.entries(schema.properties ?? {});
 
-        return entries.filter(([key, value]) => {
-            if (!value.properties && key === "definitions") return false; // Skip definitions
-            if (this.ignore.includes(key)) return false;
-            if (this.showLevelOverride >= path.length) return true;
-            if (required[key]) return true;
-            if (this.#getLink([...this.#base, ...path, key])) return true;
-            if (!this.onlyRequired) return true;
-            return false;
-        });
+        const isArrayOfArrays = (arr) => !!arr.find((v) => Array.isArray(v));
+
+        const flattenRecursedValues = (arr) => {
+            const newArr = [];
+            arr.forEach((o) => {
+                if (isArrayOfArrays(o)) newArr.push(...o);
+                else newArr.push(o);
+            });
+
+            return newArr;
+        };
+
+        const isRenderable = (key, value) => {
+            if (recursive && value.properties) return this.#getRenderable(value, required[key], [...path, key], true);
+            else return [key, value];
+        };
+
+        const res = entries
+            .map(([key, value]) => {
+                if (!value.properties && key === "definitions") return false; // Skip definitions
+                if (this.ignore.includes(key)) return false;
+                if (this.showLevelOverride >= path.length) return isRenderable(key, value);
+                if (required[key]) return isRenderable(key, value);
+                if (this.#getLink([...this.#base, ...path, key])) return isRenderable(key, value);
+                if (!this.onlyRequired) return isRenderable(key, value);
+                return false;
+            })
+            .filter((o) => !!o);
+
+        return flattenRecursedValues(res); // Flatten on the last pass
     };
 
     validateOnChange = () => {};
@@ -757,7 +794,7 @@ export class JSONSchemaForm extends LitElement {
                 const accordion = new Accordion({
                     sections: {
                         [headerName]: {
-                            subtitle: `${this.#getRenderable(info, required[name], fullPath).length} fields`,
+                            subtitle: `${this.#getRenderable(info, required[name], fullPath, true).length} fields`,
                             content: this.#nestedForms[name],
                         },
                     },
@@ -836,7 +873,10 @@ export class JSONSchemaForm extends LitElement {
 
         return html`
             <div>
-                ${false ? html`<h2>${schema.title}</h2>` : ""} ${false ? html`<p>${schema.description}</p>` : ""}
+                ${schema.description
+                    ? html`<h4>Description</h4>
+                          <p class="guided--text-input-instructions">${unsafeHTML(schema.description)}</p>`
+                    : ""}
                 ${this.#render(schema, this.resolved, this.#requirements)}
             </div>
         `;

src/renderer/src/stories/JSONSchemaInput.js

@@ -1,4 +1,5 @@
 import { LitElement, css, html } from "lit";
+import { unsafeHTML } from "lit/directives/unsafe-html.js";
 import { FilesystemSelector } from "./FileSystemSelector";
 
 import { BasicTable } from "./BasicTable";
@@ -142,7 +143,7 @@ export class JSONSchemaInput extends LitElement {
             ${input}
             ${info.description
                 ? html`<p class="guided--text-input-instructions">
-                      ${capitalize(info.description)}${info.description.slice(-1)[0] === "." ? "" : "."}
+                      ${unsafeHTML(capitalize(info.description))}${info.description.slice(-1)[0] === "." ? "" : "."}
                   </p>`
                 : ""}
         `;
@@ -191,7 +192,14 @@ export class JSONSchemaInput extends LitElement {
 
                     // NOTE: This is likely an incorrect declaration of the table validation call
                     validateOnChange: (key, parent, v) => {
-                        return validateOnChange && this.#triggerValidation(key, this.form.tables[name], path); // this.form.validateOnChange(key, parent, fullPath, v);
+                        return (
+                            validateOnChange &&
+                            (this.onValidate
+                                ? this.onValidate()
+                                : this.form
+                                ? this.form.validateOnChange(key, parent, fullPath, v)
+                                : "")
+                        );
                     },
 
                     onStatusChange: () => this.form?.checkStatus(), // Check status on all elements

src/renderer/src/stories/forms/utils.ts

@@ -1,4 +1,6 @@
-export const capitalize = (str: string) => str[0].toUpperCase() + str.slice(1)
+const toCapitalizeAll = ['nwb', 'api', 'id']
+
+export const capitalize = (str: string) => toCapitalizeAll.includes(str.toLowerCase()) ? str.toUpperCase() : str[0].toUpperCase() + str.slice(1)
 
 
 export const header = (headerStr: string) => {

src/renderer/src/stories/pages/Page.js

@@ -17,7 +17,7 @@ export class Page extends LitElement {
     //     );
     // }
 
-    info = { globalState: {} };
+    info = { globalState: {}, states: { saved: false } };
 
     constructor(info = {}) {
         super();

src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js

@@ -58,18 +58,19 @@ export class GuidedMetadataPage extends ManagedPage {
     createForm = ({ subject, session, info }) => {
         // const results = createResults({ subject, info }, this.info.globalState);
 
-        const globalState = this.info.globalState;
+        const { globalState } = this.info;
 
         const results = info.metadata; // Edited form info
 
         // Define the appropriate global metadata to fill empty values in the form
-        const aggregateGlobalMetadata = resolveGlobalOverrides(subject, this.info.globalState);
+        const aggregateGlobalMetadata = resolveGlobalOverrides(subject, globalState);
 
         // Define the correct instance identifier
         const instanceId = `sub-${subject}/ses-${session}`;
 
         // Ignore specific metadata in the form by removing their schema value
         const schema = globalState.schema.metadata[subject][session];
+        delete schema.description;
         delete schema.properties.NWBFile.properties.source_script;
         delete schema.properties.NWBFile.properties.source_script_file_name;