Merge branch 'main' into show-dandi-results-after-upload

.github/workflows/Build-and-deploy-mac.yml

@@ -33,6 +33,9 @@ jobs:
       - name: Install package.json modules and their dependencies
         run: npm install --ignore-scripts
 
+      - name: Remove bad sonpy file (might make Spike2 format unusable on Mac - should exclude from selection)
+        run: rm -f /usr/local/miniconda/envs/nwb-guide/lib/python3.9/site-packages/sonpy/linux/sonpy.so
+
       - uses: apple-actions/import-codesign-certs@v1
         with:
           # https://developer.apple.com/account/resources/certificates/add
@@ -56,5 +59,4 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           appleId: ${{ secrets.APPLE_ID }}  # currently this is set to Ryan's Apple ID and password
           appleIdPassword: ${{ secrets.APPLE_PASSWORD }}
-        run: |
-          npm run deploy:mac
+        run: npm run deploy:mac

.github/workflows/pyflask-build-and-dist-tests.yml

@@ -83,6 +83,10 @@ jobs:
 
       - run: npm ci
 
+      # fix for macos build
+      - if: matrix.os == 'macos-latest'
+        run: rm -f /Users/runner/miniconda3/envs/nwb-guide/lib/python3.9/site-packages/sonpy/linux/sonpy.so
+
       - name: Build PyFlask distribution
         run: npm run build:flask:${{ matrix.shorthand }}
 

.pre-commit-config.yaml

@@ -6,7 +6,7 @@ repos:
     -   id: end-of-file-fixer
     -   id: trailing-whitespace
 -   repo: https://github.com/psf/black
-    rev: 23.7.0
+    rev: 23.9.1
     hooks:
     -   id: black
         exclude: ^docs/

docs/format_support.rst

@@ -0,0 +1,7 @@
+Ecosystem Format Support
+=======================================
+The following is a live record of all the supported formats in the NWB GUIDE and underlying ecosystem.
+
+.. raw:: html
+
+    <iframe style="width: 100%; height: 50vh; border: none;" src="https://catalystneuro.github.io/format-support-table/"></iframe>

docs/index.rst

@@ -19,3 +19,4 @@ The resulting files are fully compliant with the best practices expected of the
    :maxdepth: 2
 
    developer_guide
+   format_support

environments/environment-Linux.yml

@@ -1,31 +1,22 @@
 name: nwb-guide
 channels:
-  - defaults
   - conda-forge
+  - defaults
 dependencies:
-  - python = 3.9.17
-  - PyInstaller = 4.7
-  - nodejs = 16.14.2
-  - libgcc = 7.2.0
-  - git = 2.20.1
+  - python = 3.9.18
+  - PyInstaller = 5.13.0
+  - nodejs = 18.16.1
+  - numcodecs = 0.11.0
+  # install these from conda-forge so that dependent packages get included in the distributable
+  - jsonschema = 4.18.0  # installs jsonschema-specifications
+  - pydantic[email] = 2.0.2  # installs email-validator
   - pip
   - pip:
-      - chardet == 4.0.0
-      - pandas == 1.4.2
-      - openpyxl == 3.0.3
-      - pennsieve == 6.1.2
-      - configparser == 4.0.2
-      - python-docx == 0.8.10
-      - xlrd == 1.2.0
-      - biopython == 1.79
-      - flask == 2.0.2
-      - flask_restx
-      - protobuf == 3.20.0
-      - gevent == 21.12.0
-      - werkzeug == 2.0.2
-      - opencv-python == 4.5.3.56
+      - chardet == 5.1.0
+      - configparser == 6.0.0
+      - flask == 2.3.2
+      - flask-cors == 4.0.0
+      - flask_restx == 1.1.0
       - neuroconv @ git+https://github.com/catalystneuro/neuroconv.git@main#neuroconv[full]
-      - hdmf >= 3.7.0
-      - pytest == 7.2.2
+      - pytest == 7.4.0
       - pytest-cov == 4.1.0
-      - flask-cors === 3.0.10

environments/environment-MAC-arm64.yml

@@ -3,21 +3,23 @@ channels:
   - conda-forge
   - defaults
 dependencies:
-  - python = 3.9.17
+  - python = 3.9.18
   - PyInstaller = 5.13.0
   - nodejs = 18.16.1
   - numcodecs = 0.11.0
   - lxml = 4.9.3  # pypi build fails due to x64/arm64 mismatch so install from conda-forge
   - pyedflib = 0.1.32  # pypi build fails due to x64/arm64 mismatch so install from conda-forge
-  - jsonschema = 4.18.0  # this installs jsonschema-specifications which is needed for validation
-  - pydantic[email] = 2.0.2  # email validator is used by dandi
   - numpy  # may have x64/arm64 mismatch issues so install from conda-forge
+  # install these from conda-forge so that dependent packages get included in the distributable
+  - jsonschema = 4.18.0  # installs jsonschema-specifications
+  - pydantic[email] = 2.0.2  # installs email-validator
   - pip
   - pip:
       - chardet == 5.1.0
       - configparser == 6.0.0
       - flask == 2.3.2
-      - flask_restx == 1.1.0
       - flask-cors == 4.0.0
-      - neuroconv @ git+https://github.com/catalystneuro/neuroconv.git@main#neuroconv[ecephys]
+      - flask_restx == 1.1.0
+      - neuroconv @ git+https://github.com/catalystneuro/neuroconv.git@main#neuroconv[full]
       - pytest == 7.4.0
+      - pytest-cov == 4.1.0

environments/environment-MAC.yml

@@ -1,22 +1,22 @@
 name: nwb-guide
 channels:
-  - defaults
   - conda-forge
+  - defaults
 dependencies:
-  - python = 3.9.17
-  - nodejs = 18.16.1
+  - python = 3.9.18
   - PyInstaller = 5.13.0
+  - nodejs = 18.16.1
   - numcodecs = 0.11.0
-  - jsonschema = 4.18.0  # this installs jsonschema-specifications which is needed for validation
-  - pydantic[email] = 2.0.2  # email validator is used by dandi
+  # install these from conda-forge so that dependent packages get included in the distributable
+  - jsonschema = 4.18.0  # installs jsonschema-specifications
+  - pydantic[email] = 2.0.2  # installs email-validator
   - pip
   - pip:
       - chardet == 5.1.0
       - configparser == 6.0.0
       - flask == 2.3.2
-      - flask-cors === 3.0.10
+      - flask-cors == 4.0.0
       - flask_restx == 1.1.0
       - neuroconv @ git+https://github.com/catalystneuro/neuroconv.git@main#neuroconv[full]
-      - hdmf >= 3.7.0
-      - pytest == 7.2.2
+      - pytest == 7.4.0
       - pytest-cov == 4.1.0

environments/environment-Windows.yml

@@ -20,3 +20,4 @@ dependencies:
       - hdmf >= 3.7.0
       - pytest == 7.2.2
       - pytest-cov == 4.1.0
+      - email-validator == 2.0.0

package.json

@@ -1,7 +1,7 @@
 {
   "name": "nwb-guide",
   "productName": "NWB GUIDE",
-  "version": "0.0.4",
+  "version": "0.0.5",
   "description": "",
   "main": "./build/main/main.js",
   "scripts": {
@@ -15,16 +15,17 @@
     "build:win": "npm run build && npm run build:flask:win && npm run build:electron:win",
     "build:mac": "npm run build && npm run build:flask:unix && npm run build:electron:mac",
     "build:linux": "npm run build && npm run build:flask:unix && npm run build:electron:linux",
-    "build:flask:base": "python -m PyInstaller --name nwb-guide --onedir --clean --noconfirm ./pyflask/app.py --distpath ./build/flask --collect-data jsonschema_specifications --collect-all nwbinspector --collect-all neuroconv --collect-all pynwb --collect-all hdmf --collect-all ci_info --hidden-import scipy._distributor_init --hidden-import scipy._lib.messagestream --hidden-import scipy._lib._ccallback --hidden-import scipy._lib._testutils",
+    "build:flask:base": "python -m PyInstaller --log-level DEBUG --name nwb-guide --onedir --clean --noconfirm ./pyflask/app.py --distpath ./build/flask --collect-data jsonschema_specifications --collect-all nwbinspector --collect-all neuroconv --collect-all pynwb --collect-all hdmf --collect-all ci_info --collect-all ndx_dandi_icephys --hidden-import scipy._distributor_init --hidden-import scipy._lib.messagestream --hidden-import scipy._lib._ccallback --hidden-import scipy._lib._testutils --hidden-import email_validator",
     "build:flask:win": "npm run build:flask:base -- --add-data ./paths.config.json;. --add-data ./package.json;.",
-    "build:flask:unix": "npm run build:flask:base -- --add-data ./paths.config.json:. --add-data ./package.json:. --collect-all ndx_dandi_icephys",
+    "build:flask:unix": "npm run build:flask:base -- --add-data ./paths.config.json:. --add-data ./package.json:.",
     "build:electron:win": "electron-builder build --win --publish never",
     "build:electron:mac": "electron-builder build --mac --publish never",
     "build:electron:linux": "electron-builder build --linux --publish never",
     "test": "npm run test:app && npm run test:server",
     "test:app": "vitest run",
     "test:server": "pytest pyflask/tests/ -s",
-    "test:executable": "concurrently -n EXE,TEST --kill-others \"node tests/testPyinstallerExecutable.js --port 3434 --forever\" \"pytest pyflask/tests/ -s --target http://localhost:3434\"",
+    "wait3s": "node -e \"setTimeout(() => process.exit(0),3000)\"",
+    "test:executable": "concurrently -n EXE,TEST --kill-others --success first \"node tests/testPyinstallerExecutable.js --port 3434 --forever\" \"npm run wait3s && pytest pyflask/tests/ -s --target http://localhost:3434\"",
     "test:coverage": "npm run coverage:app && npm run coverage:server",
     "coverage:app": "vitest run --coverage",
     "coverage:server": "pytest pyflask/tests/ -s --cov=pyflask --cov-report=xml",

pyflask/apis/startup.py

@@ -1,6 +1,8 @@
 """API endpoint definitions for startup operations."""
 from flask_restx import Namespace, Resource
 
+from errorHandlers import notBadRequestException
+
 startup_api = Namespace("startup", description="API for startup commands related to the NWB GUIDE.")
 
 parser = startup_api.parser()
@@ -19,3 +21,25 @@ class Echo(Resource):
     def get(self):
         args = parser.parse_args()
         return args["arg"]
+
+
+@startup_api.route("/preload-imports")
+class PreloadImports(Resource):
+    """
+    Preload various imports on startup instead of waiting for them later on.
+
+    Python caches all modules that have been imported at least once in the same kernel,
+    even if their namespace is not always exposed to a given scope. This means that later imports
+    simply expose the cached namespaces to their scope instead of retriggering the entire import.
+    """
+
+    @startup_api.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"})
+    def get(self):
+        try:
+            import neuroconv
+
+            return True
+        except Exception as exception:
+            if notBadRequestException(exception=exception):
+                startup_api.abort(500, str(exception))
+            raise exception

pyflask/tests/test_neuroconv.py

@@ -2,9 +2,8 @@
 from utils import get, post, get_converter_output_schema
 
 
-# --------------------- Tests ---------------------
-# Accesses the dictionary of all interfaces and their metadata
 def test_get_all_interfaces(client):
+    """Accesses the dictionary of all interfaces and their metadata."""
     validate(
         get("neuroconv", client),
         schema={
@@ -23,14 +22,14 @@ def test_get_all_interfaces(client):
     )
 
 
-# Test single interface schema request
 def test_single_schema_request(client):
+    """Test single interface schema request."""
     interfaces = {"myname": "SpikeGLXRecordingInterface"}
     validate(post("neuroconv/schema", interfaces, client), schema=get_converter_output_schema(interfaces))
 
 
-# Uses the NWBConverter Class to combine multiple interfaces
 def test_multiple_schema_request(client):
+    """Uses the NWBConverter Class to combine multiple interfaces."""
     interfaces = {"myname": "SpikeGLXRecordingInterface", "myphyinterface": "PhySortingInterface"}
     data = post("/neuroconv/schema", interfaces, client)
     validate(data, schema=get_converter_output_schema(interfaces))

pyflask/tests/test_startup.py

@@ -0,0 +1,7 @@
+from utils import get, post, get_converter_output_schema
+
+
+def test_preload_imports(client):
+    """Verify that the preload import endpoint returned good status."""
+    result = get("startup/preload-imports", client)
+    assert result == True

src/main/main.ts

@@ -383,7 +383,7 @@ function makeSingleInstance() {
   else app.on("second-instance", () => restoreWindow());
 }
 
-if (process.platform === 'darwin') initialize();
+initialize();
 
 app.on('activate', () => {
   if (BrowserWindow.getAllWindows().length === 0) initialize()

src/renderer/src/dependencies/simple.js

@@ -12,4 +12,11 @@ export const homeDirectory = app?.getPath("home") ?? "";
 export const appDirectory = homeDirectory ? joinPath(homeDirectory, paths.root) : "";
 export const guidedProgressFilePath = homeDirectory ? joinPath(appDirectory, ...paths.subfolders.progress) : "";
 
+export const stubSaveFolderPath = homeDirectory
+    ? joinPath(homeDirectory, paths["root"], ...paths.subfolders.stubs)
+    : "";
+export const conversionSaveFolderPath = homeDirectory
+    ? joinPath(homeDirectory, paths["root"], ...paths.subfolders.conversions)
+    : "";
+
 export const isStorybook = window.location.href.includes("iframe.html");

src/renderer/src/index.ts

@@ -94,18 +94,22 @@ async function checkInternetConnection() {
     return hasInternet
 };
 
-// Check if the Pysoda server is live
+// Check if the Flask server is live
 const serverIsLiveStartup = async () => {
   const echoResponse = await fetch(`${baseUrl}/startup/echo?arg=server ready`).then(res => res.json()).catch(e => e)
   return echoResponse === "server ready" ? true : false;
 };
 
+// Preload Flask imports for faster on-demand operations
+const preloadFlaskImports = async () => await fetch(`${baseUrl}/startup/preload-imports`).then(res => res.json()).catch(e => e)
+
 let openPythonStatusNotyf: undefined | any;
 
 async function pythonServerOpened() {
 
   // Confirm requests are actually received by the server
   const isLive = await serverIsLiveStartup()
+  if (isLive) await preloadFlaskImports() // initiate preload of Flask imports
   if (!isLive) return pythonServerClosed()
 
   // Update server status and throw a notification

src/renderer/src/progress/index.js

@@ -1,6 +1,13 @@
 import Swal from "sweetalert2";
 
-import { guidedProgressFilePath, reloadPageToHome, isStorybook, appDirectory } from "../dependencies/simple.js";
+import {
+    guidedProgressFilePath,
+    reloadPageToHome,
+    isStorybook,
+    appDirectory,
+    stubSaveFolderPath,
+    conversionSaveFolderPath,
+} from "../dependencies/simple.js";
 import { fs } from "../electron/index.js";
 import { joinPath, runOnLoad } from "../globals.js";
 import { merge } from "../stories/pages/utils.js";
@@ -102,14 +109,14 @@ export function resume(name) {
 
 export const remove = async (name) => {
     const result = await Swal.fire({
-        title: `Are you sure you would like to delete NWB GUIDE progress made on the dataset: ${name}?`,
-        text: "Your progress file will be deleted permanently, and all existing progress will be lost.",
+        title: `Are you sure you would like to delete this conversion pipeline?`,
+        html: `All related files will be deleted permanently, and existing progress will be lost.`,
         icon: "warning",
         heightAuto: false,
         showCancelButton: true,
         confirmButtonColor: "#3085d6",
         cancelButtonColor: "#d33",
-        confirmButtonText: "Delete progress file",
+        confirmButtonText: `Delete ${name}`,
         cancelButtonText: "Cancel",
         focusCancel: true,
     });
@@ -119,9 +126,17 @@ export const remove = async (name) => {
         const progressFilePathToDelete = joinPath(guidedProgressFilePath, name + ".json");
 
         //delete the progress file
-        if (fs) fs.unlinkSync(progressFilePathToDelete, (err) => console.log(err));
+        if (fs) fs.unlinkSync(progressFilePathToDelete);
         else localStorage.removeItem(progressFilePathToDelete);
 
+        if (fs) {
+            // delete default stub location
+            fs.rmSync(joinPath(stubSaveFolderPath, name), { recursive: true, force: true });
+
+            // delete default conversion location
+            fs.rmSync(joinPath(conversionSaveFolderPath, name), { recursive: true, force: true });
+        }
+
         return true;
     }
 

src/renderer/src/stories/pages/guided-mode/ProgressCard.js

@@ -126,7 +126,7 @@ export class ProgressCard extends LitElement {
                         @click=${(ev) => progress.deleteProgressCard(ev.target)}
                     >
                         <i class="fas fa-trash mr-sm-1"></i>
-                        Delete progress file
+                        Delete pipeline
                     </h2>
                 </div>
             </div>

src/renderer/src/stories/pages/guided-mode/data/GuidedStructure.js

@@ -75,9 +75,9 @@ export class GuidedStructurePage extends Page {
     };
 
     async updated() {
-        const selected = this.info.globalState.interfaces;
+        const { interfaces = {} } = this.info.globalState;
 
-        if (Object.keys(selected).length > 0) this.list.emptyMessage = "Loading valid interfaces...";
+        if (Object.keys(interfaces).length > 0) this.list.emptyMessage = "Loading valid interfaces...";
 
         this.search.options = await fetch(`${baseUrl}/neuroconv`)
             .then((res) => res.json())
@@ -93,7 +93,7 @@ export class GuidedStructurePage extends Page {
             )
             .catch((e) => console.error(e));
 
-        for (const [key, name] of Object.entries(selected || {})) {
+        for (const [key, name] of Object.entries(interfaces)) {
             let found = this.search.options?.find((o) => o.value === name);
 
             // If not found, spoof based on the key and names provided previously