Merge branch 'main' into fix_pydantic_version

.github/workflows/Build-and-deploy-mac.yml

@@ -22,7 +22,6 @@ jobs:
       - uses: conda-incubator/setup-miniconda@v2
         with:
           activate-environment: nwb-guide
-          #mamba-version: "*"
           environment-file: environments/environment-MAC.yml
           auto-activate-base: false
 

.github/workflows/pyflask-build-and-dist-tests.yml

@@ -30,19 +30,16 @@ jobs:
             os: ubuntu-latest
             label: environments/environment-Linux.yml
             prefix: /usr/share/miniconda3/envs/nwb-guide
-            shorthand: unix
 
-          - python-version: "3.10"
+          - python-version: "3.9"
             os: macos-latest
             label: environments/environment-Mac.yml
             prefix: /Users/runner/miniconda3/envs/nwb-guide
-            shorthand: unix
 
-          - python-version: "3.10"
+          - python-version: "3.9"
             os: windows-latest
             label: environments/environment-Windows.yml
             prefix: C:\Miniconda3\envs\nwb-guide
-            shorthand: win
 
     steps:
       - uses: actions/checkout@v3
@@ -88,15 +85,7 @@ jobs:
         run: rm -f /Users/runner/miniconda3/envs/nwb-guide/lib/python3.9/site-packages/sonpy/linux/sonpy.so
 
       - name: Build PyFlask distribution
-        run: npm run build:flask:${{ matrix.shorthand }}
-
-      #- if: matrix.os == 'windows-latest'
-      #  name: Run test on build executable
-      #  run: node tests/testPyinstallerExecutable.js --script ./build/nwb-guide/nwb-guide.exe
-
-      #- if: matrix.os != 'windows-latest'
-      #  name: Run test on build executable
-      #  run: node tests/testPyinstallerExecutable.js --script ./build/nwb-guide/nwb-guide
+        run: npm run build:flask
 
       - name: Run test on distributed executable
         run: node tests/testPyinstallerExecutable.js

.gitignore

@@ -19,8 +19,6 @@ semantic.json
 build/
 yarn.lock
 
-*.spec
-
 *.pyc
 src/.DS_Store
 .DS_Store

nwb-guide.spec

@@ -0,0 +1,74 @@
+# -*- mode: python ; coding: utf-8 -*-
+import sys
+from pathlib import Path
+
+sys.setrecursionlimit(sys.getrecursionlimit() * 5)
+
+from PyInstaller.utils.hooks import collect_data_files
+from PyInstaller.utils.hooks import collect_all
+
+datas = [('./paths.config.json', '.'), ('./package.json', '.')]
+binaries = []
+hiddenimports = ['scipy._distributor_init', 'scipy._lib.messagestream', 'scipy._lib._ccallback', 'scipy._lib._testutils', 'email_validator']
+datas += collect_data_files('jsonschema_specifications')
+tmp_ret = collect_all('nwbinspector')
+datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2]
+tmp_ret = collect_all('neuroconv')
+datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2]
+tmp_ret = collect_all('pynwb')
+datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2]
+tmp_ret = collect_all('hdmf')
+datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2]
+tmp_ret = collect_all('ndx_dandi_icephys')
+datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2]
+tmp_ret = collect_all('ci_info')
+datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2]
+
+
+block_cipher = None
+
+
+a = Analysis(
+    [f"{Path('pyflask') / 'app.py'}"],
+    pathex=[],
+    binaries=binaries,
+    datas=datas,
+    hiddenimports=hiddenimports,
+    hookspath=[],
+    hooksconfig={},
+    runtime_hooks=[],
+    excludes=[],
+    win_no_prefer_redirects=False,
+    win_private_assemblies=False,
+    cipher=block_cipher,
+    noarchive=False,
+)
+pyz = PYZ(a.pure, a.zipped_data, cipher=block_cipher)
+
+exe = EXE(
+    pyz,
+    a.scripts,
+    [],
+    exclude_binaries=True,
+    name='nwb-guide',
+    debug=False,
+    bootloader_ignore_signals=False,
+    strip=False,
+    upx=True,
+    console=True,
+    disable_windowed_traceback=False,
+    argv_emulation=False,
+    target_arch=None,
+    codesign_identity=None,
+    entitlements_file=None,
+)
+coll = COLLECT(
+    exe,
+    a.binaries,
+    a.zipfiles,
+    a.datas,
+    strip=False,
+    upx=True,
+    upx_exclude=[],
+    name='nwb-guide',
+)

package.json

@@ -11,13 +11,14 @@
     "dev:app": "vite src/renderer",
     "dev:server": "cd pyflask && python -m flask run --port 4242",
     "build": "npm run build:app",
+    "echo": "python -c \"print('hello')\"",
     "build:app": "electron-vite build --outDir build",
-    "build:win": "npm run build && npm run build:flask:win && npm run build:electron:win",
-    "build:mac": "npm run build && npm run build:flask:unix && npm run build:electron:mac",
-    "build:linux": "npm run build && npm run build:flask:unix && npm run build:electron:linux",
-    "build:flask:base": "python -m PyInstaller --log-level DEBUG --name nwb-guide --onedir --clean --noconfirm ./pyflask/app.py --distpath ./build/flask --collect-data jsonschema_specifications --collect-all nwbinspector --collect-all neuroconv --collect-all pynwb --collect-all hdmf --collect-all ci_info --collect-all ndx_dandi_icephys --hidden-import scipy._distributor_init --hidden-import scipy._lib.messagestream --hidden-import scipy._lib._ccallback --hidden-import scipy._lib._testutils --hidden-import email_validator",
-    "build:flask:win": "npm run build:flask:base -- --add-data ./paths.config.json;. --add-data ./package.json;.",
-    "build:flask:unix": "npm run build:flask:base -- --add-data ./paths.config.json:. --add-data ./package.json:.",
+    "build:win": "npm run build && npm run build:flask && npm run build:electron:win",
+    "build:mac": "npm run build && npm run build:flask && npm run build:electron:mac",
+    "build:linux": "npm run build && npm run build:flask && npm run build:electron:linux",
+    "build:flask": "python -m PyInstaller nwb-guide.spec --log-level DEBUG --clean --noconfirm --distpath ./build/flask",
+    "build:flask:spec:base": "pyi-makespec --name nwb-guide --onedir --collect-data jsonschema_specifications --collect-all nwbinspector --collect-all neuroconv --collect-all pynwb --collect-all hdmf --collect-all ndx_dandi_icephys --collect-all ci_info --hidden-import scipy._distributor_init --hidden-import scipy._lib.messagestream --hidden-import scipy._lib._ccallback --hidden-import scipy._lib._testutils --hidden-import email_validator ./pyflask/app.py",
+    "build:flask:spec": "npm run build:flask:spec:base && python prepare_pyinstaller_spec.py",
     "build:electron:win": "electron-builder build --win --publish never",
     "build:electron:mac": "electron-builder build --mac --publish never",
     "build:electron:linux": "electron-builder build --linux --publish never",
@@ -57,7 +58,7 @@
       {
         "ext": "nwb",
         "name": "NWB File",
-        "role": "Editor"
+        "role": "Viewer"
       }
     ],
     "files": [

prepare_pyinstaller_spec.py

@@ -0,0 +1,30 @@
+"""
+Calling `pyi-makespec` regenerates the base spec file, but we need to extend the recursion limit.
+
+This script is run automatically as a part of `npm run build:flask:spec` after the `pyi-makespec` call.
+"""
+from pathlib import Path
+
+with open(file=Path(__file__).parent / "nwb-guide.spec", mode="r") as io:
+    lines = io.readlines()
+
+lines.insert(1, "import sys\n")
+lines.insert(2, "from pathlib import Path\n")
+lines.insert(3, "\n")
+lines.insert(4, "sys.setrecursionlimit(sys.getrecursionlimit() * 5)\n")
+lines.insert(5, "\n")
+
+# Originally this was a separate `npm` command per platform to account for CLI syntax differences between ; and :
+# The spec file is, however, the same across platforms
+data_line_index = lines.index("datas = []\n")
+lines[data_line_index] = "datas = [('./paths.config.json', '.'), ('./package.json', '.')]\n"
+
+# Another platform specific difference is the app.py location
+app_py_line_index = next(index for index, line in enumerate(lines) if "app.py" in line)
+app_py_line = "    [f\"{Path('pyflask') / 'app.py'}\"],\n"
+lines[app_py_line_index] = app_py_line
+
+with open(file=Path(__file__).parent / "nwb-guide.spec", mode="w") as io:
+    io.writelines(lines)
+
+print("Sucessfully injected recursion depth extension and json paths!")

pyflask/app.py

@@ -2,15 +2,18 @@
 import sys
 import json
 import multiprocessing
+from os.path import sep
 from logging import Formatter, DEBUG
 from logging.handlers import RotatingFileHandler
 from pathlib import Path
+from urllib.parse import unquote
+
 
 # https://stackoverflow.com/questions/32672596/pyinstaller-loads-script-multiple-times#comment103216434_32677108
 multiprocessing.freeze_support()
 
 
-from flask import Flask, request, send_from_directory
+from flask import Flask, request, send_from_directory, send_file
 from flask_cors import CORS
 from flask_restx import Api, Resource
 
@@ -52,6 +55,29 @@
 api.add_namespace(neuroconv_api)
 api.init_app(app)
 
+registered = {}
+
+
+@app.route("/files")
+def get_all_files():
+    return list(registered.keys())
+
+
+@app.route("/files/<path:path>", methods=["GET", "POST"])
+def handle_file_request(path):
+    if request.method == "GET":
+        if registered[path]:
+            return send_file(unquote(path))
+        else:
+            app.abort(404, "Resource is not accessible.")
+
+    else:
+        if ".nwb" in path:
+            registered[path] = True
+            return request.base_url
+        else:
+            app.abort(400, str("Path does not point to an NWB file."))
+
 
 @app.route("/conversions/<path:path>")
 def send_conversions(path):

schemas/json/dandi/global.json

@@ -1,9 +1,20 @@
 {
+    "description": "Log in to DANDI, click on your user initials in the top-right corner, and copy your API key from the resulting pop-up. <br/><small><b>Note:</b> <a href='https://dandiarchive.org' target='_blank'>The main archive</a> and <a href='https://gui-staging.dandiarchive.org'  target='_blank'>the staging (testing) server</a> have different API keys.</small>",
     "properties": {
-        "api_key": {
-            "type": "string",
-            "format": "password"
+        "api_keys": {
+            "properties": {
+                "main_api_key": {
+                    "type": "string",
+                    "format": "password",
+                    "description": "From the <a href='https://dandiarchive.org' target='_blank'>main archive</a>"
+                },
+                "staging_api_key": {
+                    "type": "string",
+                    "format": "password",
+                    "description": "From the <a href='https://gui-staging.dandiarchive.org' target='_blank'>staging (testing) server</a>"
+                }
+            },
+            "required": ["main_api_key", "staging_api_key"]
         }
-    },
-    "required": ["api_key"]
+    }
 }

schemas/json/dandi/upload.json

@@ -1,7 +1,8 @@
 {
     "properties": {
         "dandiset_id": {
-            "type": "string"
+            "type": "string",
+            "description": "The unique identifier for your dandiset. Will automatically determine whether to upload to the main DANDI archive or the development staging server."
         },
         "cleanup": {
             "type": "boolean",

src/main/main.ts

@@ -70,6 +70,7 @@ let globals: {
   },
 
   set mainWindowReady(v) {
+    if (!globals.mainWindow) throw new Error('Main window cannot be ready. It does not exist...')
     mainWindowReady = v
     if (v) readyQueue.forEach(f => onWindowReady(f))
     readyQueue = []
@@ -80,7 +81,7 @@ function send(this: BrowserWindow, ...args: any[]) {
   return this.webContents.send(...args)
 }
 
-const onWindowReady = (f: (win: BrowserWindow) => any) => (mainWindowReady) ? f(globals.mainWindow) : readyQueue.push(f)
+const onWindowReady = (f: (win: BrowserWindow) => any) => (globals.mainWindowReady) ? f(globals.mainWindow) : readyQueue.push(f)
 
 
 // Pass all important log functions to the application
@@ -355,14 +356,18 @@ function initialize() {
   else app.on("ready", onAppReady)
 }
 
-function onFileOpened(_, path: string) {
+function isValidFile(filepath: string) {
+  return !fs.existsSync(filepath) && path.extname(filepath) === '.nwb'
+}
+
+function onFileOpened(_, filepath: string) {
     restoreWindow() || initialize(); // Ensure the application is properly visible
-    onWindowReady((win) => win.webContents.send('fileOpened', path))
+    onWindowReady((win) => send.call(win, 'fileOpened', filepath))
 }
 
 if (isWindows && process.argv.length >= 2) {
   const openFilePath = process.argv[1];
-  if (openFilePath !== "") onFileOpened(null, openFilePath)
+  if (isValidFile(openFilePath)) onFileOpened(null, openFilePath)
 }
 
 // Make this app a single instance app.
@@ -383,7 +388,7 @@ function makeSingleInstance() {
   else app.on("second-instance", () => restoreWindow());
 }
 
-if (process.platform === 'darwin') initialize();
+initialize();
 
 app.on('activate', () => {
   if (BrowserWindow.getAllWindows().length === 0) initialize()

src/renderer/assets/css/global.css

@@ -8,6 +8,15 @@
     /* src: local('Source Code Pro'), local('SourceCodePro'), url(fonts/SourceCodePro-Regular.ttf) format('truetype'); */
 }
 
+/*  Notfy */
+.notyf__toast {
+    max-width: clamp(300px, 40vw, 500px) !important;
+}
+
+.notyf__message {
+    word-wrap: break-word;
+}
+
 /* Global ---------------------------- */
 
 * {

src/renderer/src/electron/index.js

@@ -1,3 +1,4 @@
+import { updateURLParams } from "../../utils/url.js";
 import isElectron from "./check.js";
 
 export { isElectron };
@@ -21,8 +22,12 @@ if (isElectron) {
         remote = require("@electron/remote");
         app = remote.app;
 
-        electron.ipcRenderer.on("fileOpened", (info, ...args) => {
-            console.log("File opened!", ...args);
+        electron.ipcRenderer.on("fileOpened", (info, filepath) => {
+            updateURLParams({ file: filepath });
+            const dashboard = document.querySelector("nwb-dashboard");
+            const activePage = dashboard.getAttribute("activePage");
+            if (activePage === "preview") dashboard.requestUpdate();
+            else dashboard.setAttribute("activePage", "preview");
         });
 
         ["log", "warn", "error"].forEach((method) =>

src/renderer/src/pages.js

@@ -22,11 +22,14 @@ import { TutorialPage } from "./stories/pages/tutorial/Tutorial";
 import tutorialIcon from "./stories/assets/exploration.svg?raw";
 import uploadIcon from "./stories/assets/dandi.svg?raw";
 import inspectIcon from "./stories/assets/inspect.svg?raw";
+import neurosiftIcon from "./stories/assets/neurosift-logo.svg?raw";
+
 import settingsIcon from "./stories/assets/settings.svg?raw";
 
 import { UploadsPage } from "./stories/pages/uploads/UploadsPage";
 import { SettingsPage } from "./stories/pages/settings/SettingsPage";
 import { InspectPage } from "./stories/pages/inspect/InspectPage";
+import { PreviewPage } from "./stories/pages/preview/PreviewPage";
 
 let dashboard = document.querySelector("nwb-dashboard");
 if (!dashboard) dashboard = new Dashboard();
@@ -165,14 +168,18 @@ const pages = {
             }),
         },
     }),
-    uploads: new UploadsPage({
-        label: "Uploads",
-        icon: uploadIcon,
-    }),
     inspect: new InspectPage({
         label: "Inspect",
         icon: inspectIcon,
     }),
+    preview: new PreviewPage({
+        label: "Neurosift",
+        icon: neurosiftIcon,
+    }),
+    uploads: new UploadsPage({
+        label: "Uploads",
+        icon: uploadIcon,
+    }),
     tutorial: new TutorialPage({
         label: "Tutorial",
         icon: tutorialIcon,