diff --git a/docs/format_support.rst b/docs/format_support.rst new file mode 100644 index 000000000..d2f35083e --- /dev/null +++ b/docs/format_support.rst @@ -0,0 +1,7 @@ +Ecosystem Format Support +======================================= +The following is a live record of all the supported formats in the NWB GUIDE and underlying ecosystem. + +.. raw:: html + + diff --git a/docs/index.rst b/docs/index.rst index 57fa8202f..432034626 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -19,3 +19,4 @@ The resulting files are fully compliant with the best practices expected of the :maxdepth: 2 developer_guide + format_support diff --git a/environments/environment-Windows.yml b/environments/environment-Windows.yml index 7faeba0ee..eb846dee2 100644 --- a/environments/environment-Windows.yml +++ b/environments/environment-Windows.yml @@ -20,3 +20,4 @@ dependencies: - hdmf >= 3.7.0 - pytest == 7.2.2 - pytest-cov == 4.1.0 + - email-validator == 2.0.0 diff --git a/package.json b/package.json index bad2cfe3e..6d545d081 100644 --- a/package.json +++ b/package.json @@ -15,7 +15,7 @@ "build:win": "npm run build && npm run build:flask:win && npm run build:electron:win", "build:mac": "npm run build && npm run build:flask:unix && npm run build:electron:mac", "build:linux": "npm run build && npm run build:flask:unix && npm run build:electron:linux", - "build:flask:base": "python -m PyInstaller --name nwb-guide --onedir --clean --noconfirm ./pyflask/app.py --distpath ./build/flask --collect-data jsonschema_specifications --collect-all nwbinspector --collect-all neuroconv --collect-all pynwb --collect-all hdmf --collect-all ci_info --hidden-import scipy._distributor_init --hidden-import scipy._lib.messagestream --hidden-import scipy._lib._ccallback --hidden-import scipy._lib._testutils", + "build:flask:base": "python -m PyInstaller --name nwb-guide --onedir --clean --noconfirm ./pyflask/app.py --distpath ./build/flask --collect-data jsonschema_specifications --collect-all nwbinspector --collect-all neuroconv --collect-all pynwb --collect-all hdmf --collect-all ci_info --collect-all ndx_dandi_icephys --hidden-import scipy._distributor_init --hidden-import scipy._lib.messagestream --hidden-import scipy._lib._ccallback --hidden-import scipy._lib._testutils --hidden-import email_validator", "build:flask:win": "npm run build:flask:base -- --add-data ./paths.config.json;. --add-data ./package.json;.", "build:flask:unix": "npm run build:flask:base -- --add-data ./paths.config.json:. --add-data ./package.json:. --collect-all ndx_dandi_icephys", "build:electron:win": "electron-builder build --win --publish never", @@ -24,7 +24,8 @@ "test": "npm run test:app && npm run test:server", "test:app": "vitest run", "test:server": "pytest pyflask/tests/ -s", - "test:executable": "concurrently -n EXE,TEST --kill-others \"node tests/testPyinstallerExecutable.js --port 3434 --forever\" \"pytest pyflask/tests/ -s --target http://localhost:3434\"", + "wait3s": "node -e \"setTimeout(() => process.exit(0),3000)\"", + "test:executable": "concurrently -n EXE,TEST --kill-others --success first \"node tests/testPyinstallerExecutable.js --port 3434 --forever\" \"npm run wait3s && pytest pyflask/tests/ -s --target http://localhost:3434\"", "test:coverage": "npm run coverage:app && npm run coverage:server", "coverage:app": "vitest run --coverage", "coverage:server": "pytest pyflask/tests/ -s --cov=pyflask --cov-report=xml", diff --git a/pyflask/apis/startup.py b/pyflask/apis/startup.py index 3824a9465..d72ae3cdc 100644 --- a/pyflask/apis/startup.py +++ b/pyflask/apis/startup.py @@ -1,6 +1,8 @@ """API endpoint definitions for startup operations.""" from flask_restx import Namespace, Resource +from errorHandlers import notBadRequestException + startup_api = Namespace("startup", description="API for startup commands related to the NWB GUIDE.") parser = startup_api.parser() @@ -19,3 +21,25 @@ class Echo(Resource): def get(self): args = parser.parse_args() return args["arg"] + + +@startup_api.route("/preload-imports") +class PreloadImports(Resource): + """ + Preload various imports on startup instead of waiting for them later on. + + Python caches all modules that have been imported at least once in the same kernel, + even if their namespace is not always exposed to a given scope. This means that later imports + simply expose the cached namespaces to their scope instead of retriggering the entire import. + """ + + @startup_api.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"}) + def get(self): + try: + import neuroconv + + return True + except Exception as exception: + if notBadRequestException(exception=exception): + startup_api.abort(500, str(exception)) + raise exception diff --git a/pyflask/tests/test_neuroconv.py b/pyflask/tests/test_neuroconv.py index 8c363c004..4f0e793f5 100644 --- a/pyflask/tests/test_neuroconv.py +++ b/pyflask/tests/test_neuroconv.py @@ -2,9 +2,8 @@ from utils import get, post, get_converter_output_schema -# --------------------- Tests --------------------- -# Accesses the dictionary of all interfaces and their metadata def test_get_all_interfaces(client): + """Accesses the dictionary of all interfaces and their metadata.""" validate( get("neuroconv", client), schema={ @@ -23,14 +22,14 @@ def test_get_all_interfaces(client): ) -# Test single interface schema request def test_single_schema_request(client): + """Test single interface schema request.""" interfaces = {"myname": "SpikeGLXRecordingInterface"} validate(post("neuroconv/schema", interfaces, client), schema=get_converter_output_schema(interfaces)) -# Uses the NWBConverter Class to combine multiple interfaces def test_multiple_schema_request(client): + """Uses the NWBConverter Class to combine multiple interfaces.""" interfaces = {"myname": "SpikeGLXRecordingInterface", "myphyinterface": "PhySortingInterface"} data = post("/neuroconv/schema", interfaces, client) validate(data, schema=get_converter_output_schema(interfaces)) diff --git a/pyflask/tests/test_startup.py b/pyflask/tests/test_startup.py new file mode 100644 index 000000000..42cead8c7 --- /dev/null +++ b/pyflask/tests/test_startup.py @@ -0,0 +1,7 @@ +from utils import get, post, get_converter_output_schema + + +def test_preload_imports(client): + """Verify that the preload import endpoint returned good status.""" + result = get("startup/preload-imports", client) + assert result == True diff --git a/src/renderer/src/dependencies/simple.js b/src/renderer/src/dependencies/simple.js index 35c3adafd..97f8d2148 100644 --- a/src/renderer/src/dependencies/simple.js +++ b/src/renderer/src/dependencies/simple.js @@ -12,4 +12,11 @@ export const homeDirectory = app?.getPath("home") ?? ""; export const appDirectory = homeDirectory ? joinPath(homeDirectory, paths.root) : ""; export const guidedProgressFilePath = homeDirectory ? joinPath(appDirectory, ...paths.subfolders.progress) : ""; +export const stubSaveFolderPath = homeDirectory + ? joinPath(homeDirectory, paths["root"], ...paths.subfolders.stubs) + : ""; +export const conversionSaveFolderPath = homeDirectory + ? joinPath(homeDirectory, paths["root"], ...paths.subfolders.conversions) + : ""; + export const isStorybook = window.location.href.includes("iframe.html"); diff --git a/src/renderer/src/index.ts b/src/renderer/src/index.ts index efbc4b093..c13b6f258 100644 --- a/src/renderer/src/index.ts +++ b/src/renderer/src/index.ts @@ -94,18 +94,22 @@ async function checkInternetConnection() { return hasInternet }; -// Check if the Pysoda server is live +// Check if the Flask server is live const serverIsLiveStartup = async () => { const echoResponse = await fetch(`${baseUrl}/startup/echo?arg=server ready`).then(res => res.json()).catch(e => e) return echoResponse === "server ready" ? true : false; }; +// Preload Flask imports for faster on-demand operations +const preloadFlaskImports = async () => await fetch(`${baseUrl}/startup/preload-imports`).then(res => res.json()).catch(e => e) + let openPythonStatusNotyf: undefined | any; async function pythonServerOpened() { // Confirm requests are actually received by the server const isLive = await serverIsLiveStartup() + if (isLive) await preloadFlaskImports() // initiate preload of Flask imports if (!isLive) return pythonServerClosed() // Update server status and throw a notification diff --git a/src/renderer/src/progress/index.js b/src/renderer/src/progress/index.js index b5c7c2121..b3dfb9ea8 100644 --- a/src/renderer/src/progress/index.js +++ b/src/renderer/src/progress/index.js @@ -1,6 +1,13 @@ import Swal from "sweetalert2"; -import { guidedProgressFilePath, reloadPageToHome, isStorybook, appDirectory } from "../dependencies/simple.js"; +import { + guidedProgressFilePath, + reloadPageToHome, + isStorybook, + appDirectory, + stubSaveFolderPath, + conversionSaveFolderPath, +} from "../dependencies/simple.js"; import { fs } from "../electron/index.js"; import { joinPath, runOnLoad } from "../globals.js"; import { merge } from "../stories/pages/utils.js"; @@ -102,14 +109,14 @@ export function resume(name) { export const remove = async (name) => { const result = await Swal.fire({ - title: `Are you sure you would like to delete NWB GUIDE progress made on the dataset: ${name}?`, - text: "Your progress file will be deleted permanently, and all existing progress will be lost.", + title: `Are you sure you would like to delete this conversion pipeline?`, + html: `All related files will be deleted permanently, and existing progress will be lost.`, icon: "warning", heightAuto: false, showCancelButton: true, confirmButtonColor: "#3085d6", cancelButtonColor: "#d33", - confirmButtonText: "Delete progress file", + confirmButtonText: `Delete ${name}`, cancelButtonText: "Cancel", focusCancel: true, }); @@ -119,9 +126,17 @@ export const remove = async (name) => { const progressFilePathToDelete = joinPath(guidedProgressFilePath, name + ".json"); //delete the progress file - if (fs) fs.unlinkSync(progressFilePathToDelete, (err) => console.log(err)); + if (fs) fs.unlinkSync(progressFilePathToDelete); else localStorage.removeItem(progressFilePathToDelete); + if (fs) { + // delete default stub location + fs.rmSync(joinPath(stubSaveFolderPath, name), { recursive: true, force: true }); + + // delete default conversion location + fs.rmSync(joinPath(conversionSaveFolderPath, name), { recursive: true, force: true }); + } + return true; } diff --git a/src/renderer/src/stories/pages/guided-mode/ProgressCard.js b/src/renderer/src/stories/pages/guided-mode/ProgressCard.js index 062ae8fe3..d66b0e8de 100644 --- a/src/renderer/src/stories/pages/guided-mode/ProgressCard.js +++ b/src/renderer/src/stories/pages/guided-mode/ProgressCard.js @@ -126,7 +126,7 @@ export class ProgressCard extends LitElement { @click=${(ev) => progress.deleteProgressCard(ev.target)} > - Delete progress file + Delete pipeline diff --git a/src/renderer/src/stories/pages/guided-mode/data/GuidedStructure.js b/src/renderer/src/stories/pages/guided-mode/data/GuidedStructure.js index 93938a8ca..b4c24266d 100644 --- a/src/renderer/src/stories/pages/guided-mode/data/GuidedStructure.js +++ b/src/renderer/src/stories/pages/guided-mode/data/GuidedStructure.js @@ -75,9 +75,9 @@ export class GuidedStructurePage extends Page { }; async updated() { - const selected = this.info.globalState.interfaces; + const { interfaces = {} } = this.info.globalState; - if (Object.keys(selected).length > 0) this.list.emptyMessage = "Loading valid interfaces..."; + if (Object.keys(interfaces).length > 0) this.list.emptyMessage = "Loading valid interfaces..."; this.search.options = await fetch(`${baseUrl}/neuroconv`) .then((res) => res.json()) @@ -93,7 +93,7 @@ export class GuidedStructurePage extends Page { ) .catch((e) => console.error(e)); - for (const [key, name] of Object.entries(selected || {})) { + for (const [key, name] of Object.entries(interfaces)) { let found = this.search.options?.find((o) => o.value === name); // If not found, spoof based on the key and names provided previously