Merge branch 'main' into dandi-api-key-instructions

docs/format_support.rst

@@ -0,0 +1,7 @@
+Ecosystem Format Support
+=======================================
+The following is a live record of all the supported formats in the NWB GUIDE and underlying ecosystem.
+
+.. raw:: html
+
+    <iframe style="width: 100%; height: 50vh; border: none;" src="https://catalystneuro.github.io/format-support-table/"></iframe>

docs/index.rst

@@ -19,3 +19,4 @@ The resulting files are fully compliant with the best practices expected of the
    :maxdepth: 2
 
    developer_guide
+   format_support

environments/environment-Windows.yml

@@ -20,3 +20,4 @@ dependencies:
       - hdmf >= 3.7.0
       - pytest == 7.2.2
       - pytest-cov == 4.1.0
+      - email-validator == 2.0.0

package.json

@@ -15,7 +15,7 @@
     "build:win": "npm run build && npm run build:flask:win && npm run build:electron:win",
     "build:mac": "npm run build && npm run build:flask:unix && npm run build:electron:mac",
     "build:linux": "npm run build && npm run build:flask:unix && npm run build:electron:linux",
-    "build:flask:base": "python -m PyInstaller --name nwb-guide --onedir --clean --noconfirm ./pyflask/app.py --distpath ./build/flask --collect-data jsonschema_specifications --collect-all nwbinspector --collect-all neuroconv --collect-all pynwb --collect-all hdmf --collect-all ci_info --hidden-import scipy._distributor_init --hidden-import scipy._lib.messagestream --hidden-import scipy._lib._ccallback --hidden-import scipy._lib._testutils",
+    "build:flask:base": "python -m PyInstaller --name nwb-guide --onedir --clean --noconfirm ./pyflask/app.py --distpath ./build/flask --collect-data jsonschema_specifications --collect-all nwbinspector --collect-all neuroconv --collect-all pynwb --collect-all hdmf --collect-all ci_info --collect-all ndx_dandi_icephys --hidden-import scipy._distributor_init --hidden-import scipy._lib.messagestream --hidden-import scipy._lib._ccallback --hidden-import scipy._lib._testutils --hidden-import email_validator",
     "build:flask:win": "npm run build:flask:base -- --add-data ./paths.config.json;. --add-data ./package.json;.",
     "build:flask:unix": "npm run build:flask:base -- --add-data ./paths.config.json:. --add-data ./package.json:. --collect-all ndx_dandi_icephys",
     "build:electron:win": "electron-builder build --win --publish never",
@@ -24,7 +24,8 @@
     "test": "npm run test:app && npm run test:server",
     "test:app": "vitest run",
     "test:server": "pytest pyflask/tests/ -s",
-    "test:executable": "concurrently -n EXE,TEST --kill-others \"node tests/testPyinstallerExecutable.js --port 3434 --forever\" \"pytest pyflask/tests/ -s --target http://localhost:3434\"",
+    "wait3s": "node -e \"setTimeout(() => process.exit(0),3000)\"",
+    "test:executable": "concurrently -n EXE,TEST --kill-others --success first \"node tests/testPyinstallerExecutable.js --port 3434 --forever\" \"npm run wait3s && pytest pyflask/tests/ -s --target http://localhost:3434\"",
     "test:coverage": "npm run coverage:app && npm run coverage:server",
     "coverage:app": "vitest run --coverage",
     "coverage:server": "pytest pyflask/tests/ -s --cov=pyflask --cov-report=xml",

pyflask/apis/startup.py

@@ -1,6 +1,8 @@
 """API endpoint definitions for startup operations."""
 from flask_restx import Namespace, Resource
 
+from errorHandlers import notBadRequestException
+
 startup_api = Namespace("startup", description="API for startup commands related to the NWB GUIDE.")
 
 parser = startup_api.parser()
@@ -19,3 +21,25 @@ class Echo(Resource):
     def get(self):
         args = parser.parse_args()
         return args["arg"]
+
+
+@startup_api.route("/preload-imports")
+class PreloadImports(Resource):
+    """
+    Preload various imports on startup instead of waiting for them later on.
+
+    Python caches all modules that have been imported at least once in the same kernel,
+    even if their namespace is not always exposed to a given scope. This means that later imports
+    simply expose the cached namespaces to their scope instead of retriggering the entire import.
+    """
+
+    @startup_api.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"})
+    def get(self):
+        try:
+            import neuroconv
+
+            return True
+        except Exception as exception:
+            if notBadRequestException(exception=exception):
+                startup_api.abort(500, str(exception))
+            raise exception

pyflask/tests/test_neuroconv.py

@@ -2,9 +2,8 @@
 from utils import get, post, get_converter_output_schema
 
 
-# --------------------- Tests ---------------------
-# Accesses the dictionary of all interfaces and their metadata
 def test_get_all_interfaces(client):
+    """Accesses the dictionary of all interfaces and their metadata."""
     validate(
         get("neuroconv", client),
         schema={
@@ -23,14 +22,14 @@ def test_get_all_interfaces(client):
     )
 
 
-# Test single interface schema request
 def test_single_schema_request(client):
+    """Test single interface schema request."""
     interfaces = {"myname": "SpikeGLXRecordingInterface"}
     validate(post("neuroconv/schema", interfaces, client), schema=get_converter_output_schema(interfaces))
 
 
-# Uses the NWBConverter Class to combine multiple interfaces
 def test_multiple_schema_request(client):
+    """Uses the NWBConverter Class to combine multiple interfaces."""
     interfaces = {"myname": "SpikeGLXRecordingInterface", "myphyinterface": "PhySortingInterface"}
     data = post("/neuroconv/schema", interfaces, client)
     validate(data, schema=get_converter_output_schema(interfaces))

pyflask/tests/test_startup.py

@@ -0,0 +1,7 @@
+from utils import get, post, get_converter_output_schema
+
+
+def test_preload_imports(client):
+    """Verify that the preload import endpoint returned good status."""
+    result = get("startup/preload-imports", client)
+    assert result == True

src/renderer/src/dependencies/simple.js

@@ -12,4 +12,11 @@ export const homeDirectory = app?.getPath("home") ?? "";
 export const appDirectory = homeDirectory ? joinPath(homeDirectory, paths.root) : "";
 export const guidedProgressFilePath = homeDirectory ? joinPath(appDirectory, ...paths.subfolders.progress) : "";
 
+export const stubSaveFolderPath = homeDirectory
+    ? joinPath(homeDirectory, paths["root"], ...paths.subfolders.stubs)
+    : "";
+export const conversionSaveFolderPath = homeDirectory
+    ? joinPath(homeDirectory, paths["root"], ...paths.subfolders.conversions)
+    : "";
+
 export const isStorybook = window.location.href.includes("iframe.html");

src/renderer/src/index.ts

@@ -94,18 +94,22 @@ async function checkInternetConnection() {
     return hasInternet
 };
 
-// Check if the Pysoda server is live
+// Check if the Flask server is live
 const serverIsLiveStartup = async () => {
   const echoResponse = await fetch(`${baseUrl}/startup/echo?arg=server ready`).then(res => res.json()).catch(e => e)
   return echoResponse === "server ready" ? true : false;
 };
 
+// Preload Flask imports for faster on-demand operations
+const preloadFlaskImports = async () => await fetch(`${baseUrl}/startup/preload-imports`).then(res => res.json()).catch(e => e)
+
 let openPythonStatusNotyf: undefined | any;
 
 async function pythonServerOpened() {
 
   // Confirm requests are actually received by the server
   const isLive = await serverIsLiveStartup()
+  if (isLive) await preloadFlaskImports() // initiate preload of Flask imports
   if (!isLive) return pythonServerClosed()
 
   // Update server status and throw a notification

src/renderer/src/progress/index.js

@@ -1,6 +1,13 @@
 import Swal from "sweetalert2";
 
-import { guidedProgressFilePath, reloadPageToHome, isStorybook, appDirectory } from "../dependencies/simple.js";
+import {
+    guidedProgressFilePath,
+    reloadPageToHome,
+    isStorybook,
+    appDirectory,
+    stubSaveFolderPath,
+    conversionSaveFolderPath,
+} from "../dependencies/simple.js";
 import { fs } from "../electron/index.js";
 import { joinPath, runOnLoad } from "../globals.js";
 import { merge } from "../stories/pages/utils.js";
@@ -102,14 +109,14 @@ export function resume(name) {
 
 export const remove = async (name) => {
     const result = await Swal.fire({
-        title: `Are you sure you would like to delete NWB GUIDE progress made on the dataset: ${name}?`,
-        text: "Your progress file will be deleted permanently, and all existing progress will be lost.",
+        title: `Are you sure you would like to delete this conversion pipeline?`,
+        html: `All related files will be deleted permanently, and existing progress will be lost.`,
         icon: "warning",
         heightAuto: false,
         showCancelButton: true,
         confirmButtonColor: "#3085d6",
         cancelButtonColor: "#d33",
-        confirmButtonText: "Delete progress file",
+        confirmButtonText: `Delete ${name}`,
         cancelButtonText: "Cancel",
         focusCancel: true,
     });
@@ -119,9 +126,17 @@ export const remove = async (name) => {
         const progressFilePathToDelete = joinPath(guidedProgressFilePath, name + ".json");
 
         //delete the progress file
-        if (fs) fs.unlinkSync(progressFilePathToDelete, (err) => console.log(err));
+        if (fs) fs.unlinkSync(progressFilePathToDelete);
         else localStorage.removeItem(progressFilePathToDelete);
 
+        if (fs) {
+            // delete default stub location
+            fs.rmSync(joinPath(stubSaveFolderPath, name), { recursive: true, force: true });
+
+            // delete default conversion location
+            fs.rmSync(joinPath(conversionSaveFolderPath, name), { recursive: true, force: true });
+        }
+
         return true;
     }
 

src/renderer/src/stories/pages/guided-mode/ProgressCard.js

@@ -126,7 +126,7 @@ export class ProgressCard extends LitElement {
                         @click=${(ev) => progress.deleteProgressCard(ev.target)}
                     >
                         <i class="fas fa-trash mr-sm-1"></i>
-                        Delete progress file
+                        Delete pipeline
                     </h2>
                 </div>
             </div>

src/renderer/src/stories/pages/guided-mode/data/GuidedStructure.js

@@ -75,9 +75,9 @@ export class GuidedStructurePage extends Page {
     };
 
     async updated() {
-        const selected = this.info.globalState.interfaces;
+        const { interfaces = {} } = this.info.globalState;
 
-        if (Object.keys(selected).length > 0) this.list.emptyMessage = "Loading valid interfaces...";
+        if (Object.keys(interfaces).length > 0) this.list.emptyMessage = "Loading valid interfaces...";
 
         this.search.options = await fetch(`${baseUrl}/neuroconv`)
             .then((res) => res.json())
@@ -93,7 +93,7 @@ export class GuidedStructurePage extends Page {
             )
             .catch((e) => console.error(e));
 
-        for (const [key, name] of Object.entries(selected || {})) {
+        for (const [key, name] of Object.entries(interfaces)) {
             let found = this.search.options?.find((o) => o.value === name);
 
             // If not found, spoof based on the key and names provided previously