diff --git a/docs/format_support.rst b/docs/format_support.rst
new file mode 100644
index 000000000..d2f35083e
--- /dev/null
+++ b/docs/format_support.rst
@@ -0,0 +1,7 @@
+Ecosystem Format Support
+=======================================
+The following is a live record of all the supported formats in the NWB GUIDE and underlying ecosystem.
+
+.. raw:: html
+
+
diff --git a/docs/index.rst b/docs/index.rst
index 57fa8202f..432034626 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -19,3 +19,4 @@ The resulting files are fully compliant with the best practices expected of the
:maxdepth: 2
developer_guide
+ format_support
diff --git a/environments/environment-Windows.yml b/environments/environment-Windows.yml
index 7faeba0ee..eb846dee2 100644
--- a/environments/environment-Windows.yml
+++ b/environments/environment-Windows.yml
@@ -20,3 +20,4 @@ dependencies:
- hdmf >= 3.7.0
- pytest == 7.2.2
- pytest-cov == 4.1.0
+ - email-validator == 2.0.0
diff --git a/package.json b/package.json
index bad2cfe3e..6d545d081 100644
--- a/package.json
+++ b/package.json
@@ -15,7 +15,7 @@
"build:win": "npm run build && npm run build:flask:win && npm run build:electron:win",
"build:mac": "npm run build && npm run build:flask:unix && npm run build:electron:mac",
"build:linux": "npm run build && npm run build:flask:unix && npm run build:electron:linux",
- "build:flask:base": "python -m PyInstaller --name nwb-guide --onedir --clean --noconfirm ./pyflask/app.py --distpath ./build/flask --collect-data jsonschema_specifications --collect-all nwbinspector --collect-all neuroconv --collect-all pynwb --collect-all hdmf --collect-all ci_info --hidden-import scipy._distributor_init --hidden-import scipy._lib.messagestream --hidden-import scipy._lib._ccallback --hidden-import scipy._lib._testutils",
+ "build:flask:base": "python -m PyInstaller --name nwb-guide --onedir --clean --noconfirm ./pyflask/app.py --distpath ./build/flask --collect-data jsonschema_specifications --collect-all nwbinspector --collect-all neuroconv --collect-all pynwb --collect-all hdmf --collect-all ci_info --collect-all ndx_dandi_icephys --hidden-import scipy._distributor_init --hidden-import scipy._lib.messagestream --hidden-import scipy._lib._ccallback --hidden-import scipy._lib._testutils --hidden-import email_validator",
"build:flask:win": "npm run build:flask:base -- --add-data ./paths.config.json;. --add-data ./package.json;.",
"build:flask:unix": "npm run build:flask:base -- --add-data ./paths.config.json:. --add-data ./package.json:. --collect-all ndx_dandi_icephys",
"build:electron:win": "electron-builder build --win --publish never",
@@ -24,7 +24,8 @@
"test": "npm run test:app && npm run test:server",
"test:app": "vitest run",
"test:server": "pytest pyflask/tests/ -s",
- "test:executable": "concurrently -n EXE,TEST --kill-others \"node tests/testPyinstallerExecutable.js --port 3434 --forever\" \"pytest pyflask/tests/ -s --target http://localhost:3434\"",
+ "wait3s": "node -e \"setTimeout(() => process.exit(0),3000)\"",
+ "test:executable": "concurrently -n EXE,TEST --kill-others --success first \"node tests/testPyinstallerExecutable.js --port 3434 --forever\" \"npm run wait3s && pytest pyflask/tests/ -s --target http://localhost:3434\"",
"test:coverage": "npm run coverage:app && npm run coverage:server",
"coverage:app": "vitest run --coverage",
"coverage:server": "pytest pyflask/tests/ -s --cov=pyflask --cov-report=xml",
diff --git a/pyflask/apis/startup.py b/pyflask/apis/startup.py
index 3824a9465..d72ae3cdc 100644
--- a/pyflask/apis/startup.py
+++ b/pyflask/apis/startup.py
@@ -1,6 +1,8 @@
"""API endpoint definitions for startup operations."""
from flask_restx import Namespace, Resource
+from errorHandlers import notBadRequestException
+
startup_api = Namespace("startup", description="API for startup commands related to the NWB GUIDE.")
parser = startup_api.parser()
@@ -19,3 +21,25 @@ class Echo(Resource):
def get(self):
args = parser.parse_args()
return args["arg"]
+
+
+@startup_api.route("/preload-imports")
+class PreloadImports(Resource):
+ """
+ Preload various imports on startup instead of waiting for them later on.
+
+ Python caches all modules that have been imported at least once in the same kernel,
+ even if their namespace is not always exposed to a given scope. This means that later imports
+ simply expose the cached namespaces to their scope instead of retriggering the entire import.
+ """
+
+ @startup_api.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"})
+ def get(self):
+ try:
+ import neuroconv
+
+ return True
+ except Exception as exception:
+ if notBadRequestException(exception=exception):
+ startup_api.abort(500, str(exception))
+ raise exception
diff --git a/pyflask/tests/test_neuroconv.py b/pyflask/tests/test_neuroconv.py
index 8c363c004..4f0e793f5 100644
--- a/pyflask/tests/test_neuroconv.py
+++ b/pyflask/tests/test_neuroconv.py
@@ -2,9 +2,8 @@
from utils import get, post, get_converter_output_schema
-# --------------------- Tests ---------------------
-# Accesses the dictionary of all interfaces and their metadata
def test_get_all_interfaces(client):
+ """Accesses the dictionary of all interfaces and their metadata."""
validate(
get("neuroconv", client),
schema={
@@ -23,14 +22,14 @@ def test_get_all_interfaces(client):
)
-# Test single interface schema request
def test_single_schema_request(client):
+ """Test single interface schema request."""
interfaces = {"myname": "SpikeGLXRecordingInterface"}
validate(post("neuroconv/schema", interfaces, client), schema=get_converter_output_schema(interfaces))
-# Uses the NWBConverter Class to combine multiple interfaces
def test_multiple_schema_request(client):
+ """Uses the NWBConverter Class to combine multiple interfaces."""
interfaces = {"myname": "SpikeGLXRecordingInterface", "myphyinterface": "PhySortingInterface"}
data = post("/neuroconv/schema", interfaces, client)
validate(data, schema=get_converter_output_schema(interfaces))
diff --git a/pyflask/tests/test_startup.py b/pyflask/tests/test_startup.py
new file mode 100644
index 000000000..42cead8c7
--- /dev/null
+++ b/pyflask/tests/test_startup.py
@@ -0,0 +1,7 @@
+from utils import get, post, get_converter_output_schema
+
+
+def test_preload_imports(client):
+ """Verify that the preload import endpoint returned good status."""
+ result = get("startup/preload-imports", client)
+ assert result == True
diff --git a/src/renderer/src/dependencies/simple.js b/src/renderer/src/dependencies/simple.js
index 35c3adafd..97f8d2148 100644
--- a/src/renderer/src/dependencies/simple.js
+++ b/src/renderer/src/dependencies/simple.js
@@ -12,4 +12,11 @@ export const homeDirectory = app?.getPath("home") ?? "";
export const appDirectory = homeDirectory ? joinPath(homeDirectory, paths.root) : "";
export const guidedProgressFilePath = homeDirectory ? joinPath(appDirectory, ...paths.subfolders.progress) : "";
+export const stubSaveFolderPath = homeDirectory
+ ? joinPath(homeDirectory, paths["root"], ...paths.subfolders.stubs)
+ : "";
+export const conversionSaveFolderPath = homeDirectory
+ ? joinPath(homeDirectory, paths["root"], ...paths.subfolders.conversions)
+ : "";
+
export const isStorybook = window.location.href.includes("iframe.html");
diff --git a/src/renderer/src/index.ts b/src/renderer/src/index.ts
index efbc4b093..c13b6f258 100644
--- a/src/renderer/src/index.ts
+++ b/src/renderer/src/index.ts
@@ -94,18 +94,22 @@ async function checkInternetConnection() {
return hasInternet
};
-// Check if the Pysoda server is live
+// Check if the Flask server is live
const serverIsLiveStartup = async () => {
const echoResponse = await fetch(`${baseUrl}/startup/echo?arg=server ready`).then(res => res.json()).catch(e => e)
return echoResponse === "server ready" ? true : false;
};
+// Preload Flask imports for faster on-demand operations
+const preloadFlaskImports = async () => await fetch(`${baseUrl}/startup/preload-imports`).then(res => res.json()).catch(e => e)
+
let openPythonStatusNotyf: undefined | any;
async function pythonServerOpened() {
// Confirm requests are actually received by the server
const isLive = await serverIsLiveStartup()
+ if (isLive) await preloadFlaskImports() // initiate preload of Flask imports
if (!isLive) return pythonServerClosed()
// Update server status and throw a notification
diff --git a/src/renderer/src/progress/index.js b/src/renderer/src/progress/index.js
index b5c7c2121..b3dfb9ea8 100644
--- a/src/renderer/src/progress/index.js
+++ b/src/renderer/src/progress/index.js
@@ -1,6 +1,13 @@
import Swal from "sweetalert2";
-import { guidedProgressFilePath, reloadPageToHome, isStorybook, appDirectory } from "../dependencies/simple.js";
+import {
+ guidedProgressFilePath,
+ reloadPageToHome,
+ isStorybook,
+ appDirectory,
+ stubSaveFolderPath,
+ conversionSaveFolderPath,
+} from "../dependencies/simple.js";
import { fs } from "../electron/index.js";
import { joinPath, runOnLoad } from "../globals.js";
import { merge } from "../stories/pages/utils.js";
@@ -102,14 +109,14 @@ export function resume(name) {
export const remove = async (name) => {
const result = await Swal.fire({
- title: `Are you sure you would like to delete NWB GUIDE progress made on the dataset: ${name}?`,
- text: "Your progress file will be deleted permanently, and all existing progress will be lost.",
+ title: `Are you sure you would like to delete this conversion pipeline?`,
+ html: `All related files will be deleted permanently, and existing progress will be lost.`,
icon: "warning",
heightAuto: false,
showCancelButton: true,
confirmButtonColor: "#3085d6",
cancelButtonColor: "#d33",
- confirmButtonText: "Delete progress file",
+ confirmButtonText: `Delete ${name}`,
cancelButtonText: "Cancel",
focusCancel: true,
});
@@ -119,9 +126,17 @@ export const remove = async (name) => {
const progressFilePathToDelete = joinPath(guidedProgressFilePath, name + ".json");
//delete the progress file
- if (fs) fs.unlinkSync(progressFilePathToDelete, (err) => console.log(err));
+ if (fs) fs.unlinkSync(progressFilePathToDelete);
else localStorage.removeItem(progressFilePathToDelete);
+ if (fs) {
+ // delete default stub location
+ fs.rmSync(joinPath(stubSaveFolderPath, name), { recursive: true, force: true });
+
+ // delete default conversion location
+ fs.rmSync(joinPath(conversionSaveFolderPath, name), { recursive: true, force: true });
+ }
+
return true;
}
diff --git a/src/renderer/src/stories/pages/guided-mode/ProgressCard.js b/src/renderer/src/stories/pages/guided-mode/ProgressCard.js
index 062ae8fe3..d66b0e8de 100644
--- a/src/renderer/src/stories/pages/guided-mode/ProgressCard.js
+++ b/src/renderer/src/stories/pages/guided-mode/ProgressCard.js
@@ -126,7 +126,7 @@ export class ProgressCard extends LitElement {
@click=${(ev) => progress.deleteProgressCard(ev.target)}
>
- Delete progress file
+ Delete pipeline
diff --git a/src/renderer/src/stories/pages/guided-mode/data/GuidedStructure.js b/src/renderer/src/stories/pages/guided-mode/data/GuidedStructure.js
index 93938a8ca..b4c24266d 100644
--- a/src/renderer/src/stories/pages/guided-mode/data/GuidedStructure.js
+++ b/src/renderer/src/stories/pages/guided-mode/data/GuidedStructure.js
@@ -75,9 +75,9 @@ export class GuidedStructurePage extends Page {
};
async updated() {
- const selected = this.info.globalState.interfaces;
+ const { interfaces = {} } = this.info.globalState;
- if (Object.keys(selected).length > 0) this.list.emptyMessage = "Loading valid interfaces...";
+ if (Object.keys(interfaces).length > 0) this.list.emptyMessage = "Loading valid interfaces...";
this.search.options = await fetch(`${baseUrl}/neuroconv`)
.then((res) => res.json())
@@ -93,7 +93,7 @@ export class GuidedStructurePage extends Page {
)
.catch((e) => console.error(e));
- for (const [key, name] of Object.entries(selected || {})) {
+ for (const [key, name] of Object.entries(interfaces)) {
let found = this.search.options?.find((o) => o.value === name);
// If not found, spoof based on the key and names provided previously