From ee76961fcfdbbb40a09b6d617ec23ef863f3bb71 Mon Sep 17 00:00:00 2001 From: Garrett Date: Wed, 6 Sep 2023 09:24:47 -0700 Subject: [PATCH 01/36] Update index.rst --- docs/index.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/index.rst b/docs/index.rst index 57fa8202f..166949267 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -19,3 +19,9 @@ The resulting files are fully compliant with the best practices expected of the :maxdepth: 2 developer_guide + +To see all the file types supported by the NWB GUIDE, visit the following website: + +.. raw:: html + + \ No newline at end of file From bb95024b4f4f67ddc69f4551d1c5097db490366c Mon Sep 17 00:00:00 2001 From: Garrett Date: Wed, 6 Sep 2023 09:33:35 -0700 Subject: [PATCH 02/36] Update index.rst --- docs/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index 166949267..c6a513a83 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -24,4 +24,4 @@ To see all the file types supported by the NWB GUIDE, visit the following websit .. raw:: html - \ No newline at end of file + \ No newline at end of file From 7e88bfffcda43ba2c6205c8b9ce17e7c3b17ab7f Mon Sep 17 00:00:00 2001 From: Garrett Date: Wed, 6 Sep 2023 09:41:12 -0700 Subject: [PATCH 03/36] Move to separate page --- docs/index.rst | 7 +------ docs/support.rst | 5 +++++ 2 files changed, 6 insertions(+), 6 deletions(-) create mode 100644 docs/support.rst diff --git a/docs/index.rst b/docs/index.rst index c6a513a83..952fdc2f3 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -19,9 +19,4 @@ The resulting files are fully compliant with the best practices expected of the :maxdepth: 2 developer_guide - -To see all the file types supported by the NWB GUIDE, visit the following website: - -.. raw:: html - - \ No newline at end of file + support \ No newline at end of file diff --git a/docs/support.rst b/docs/support.rst new file mode 100644 index 000000000..ddefced66 --- /dev/null +++ b/docs/support.rst @@ -0,0 +1,5 @@ +Ecosystem Format Support +======================================= +.. raw:: html + + \ No newline at end of file From df7cf71f0ca36fb6984901761c0e6761d7b0c062 Mon Sep 17 00:00:00 2001 From: Garrett Date: Wed, 6 Sep 2023 09:41:54 -0700 Subject: [PATCH 04/36] Update support.rst --- docs/support.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/support.rst b/docs/support.rst index ddefced66..130d02ec3 100644 --- a/docs/support.rst +++ b/docs/support.rst @@ -2,4 +2,4 @@ Ecosystem Format Support ======================================= .. raw:: html - \ No newline at end of file + \ No newline at end of file From eaa84774a4d8709f48a4b4a18bee148d7bd82b46 Mon Sep 17 00:00:00 2001 From: Garrett Date: Wed, 6 Sep 2023 09:51:55 -0700 Subject: [PATCH 05/36] Use the published sheet --- docs/support.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/support.rst b/docs/support.rst index 130d02ec3..38c26bad0 100644 --- a/docs/support.rst +++ b/docs/support.rst @@ -1,5 +1,7 @@ Ecosystem Format Support ======================================= +The following Google Sheet is a live record of all the supported formats in the NWB GUIDE and underlying ecosystem. + .. raw:: html - \ No newline at end of file + From d72d9fdd11264bd0605ba2e0eb9be04e3292d570 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 6 Sep 2023 17:25:09 +0000 Subject: [PATCH 06/36] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- docs/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index 952fdc2f3..895337c80 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -19,4 +19,4 @@ The resulting files are fully compliant with the best practices expected of the :maxdepth: 2 developer_guide - support \ No newline at end of file + support From 0aad5e9d67a84d31c7002b834b2ef07690e38b56 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Wed, 6 Sep 2023 13:25:42 -0400 Subject: [PATCH 07/36] Update index.ts --- src/renderer/src/index.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/renderer/src/index.ts b/src/renderer/src/index.ts index efbc4b093..54d841024 100644 --- a/src/renderer/src/index.ts +++ b/src/renderer/src/index.ts @@ -94,18 +94,25 @@ async function checkInternetConnection() { return hasInternet }; -// Check if the Pysoda server is live +// Check if the Flask server is live const serverIsLiveStartup = async () => { const echoResponse = await fetch(`${baseUrl}/startup/echo?arg=server ready`).then(res => res.json()).catch(e => e) return echoResponse === "server ready" ? true : false; }; +// Preload Flask imports for faster on-demand operations +const preloadFlaskImports = async () => { + const result = await fetch(`${baseUrl}/startup/preload-imports`).then(res => res.json()).catch(e => e) + return result +}; + let openPythonStatusNotyf: undefined | any; async function pythonServerOpened() { // Confirm requests are actually received by the server const isLive = await serverIsLiveStartup() + if (isLive) return preloadFlaskImports() // initiate preload of Flask imports if (!isLive) return pythonServerClosed() // Update server status and throw a notification From c78eff6910e3c8296931ae1d15fc21d23f9e8b7a Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Wed, 6 Sep 2023 13:31:57 -0400 Subject: [PATCH 08/36] Update startup.py --- pyflask/apis/startup.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/pyflask/apis/startup.py b/pyflask/apis/startup.py index 3824a9465..28e20d48b 100644 --- a/pyflask/apis/startup.py +++ b/pyflask/apis/startup.py @@ -1,6 +1,8 @@ """API endpoint definitions for startup operations.""" from flask_restx import Namespace, Resource +from errorHandlers import notBadRequestException + startup_api = Namespace("startup", description="API for startup commands related to the NWB GUIDE.") parser = startup_api.parser() @@ -19,3 +21,23 @@ class Echo(Resource): def get(self): args = parser.parse_args() return args["arg"] + +@startup_api.route("/preload-imports") +class PreloadImports(Resource): + """ + Preload various imports on startup instead of waiting for them later on. + + Python caches all modules that have been imported at least once in the same kernel, + even if their namespace is not always exposed to a given scope. This means that later imports + simply expose the cached namespaces to their scope instead of retriggering the entire import. + """ + @startup_api.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"}) + def get(self): + try: + import neuroconv + + return True + except Exception as exception: + if notBadRequestException(exception=exception): + startup_api.abort(500, str(exception)) + raise exception From 3fab5e5a6cd2d693cf603f5ee3bad813d85cbf2d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 6 Sep 2023 17:33:18 +0000 Subject: [PATCH 09/36] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pyflask/apis/startup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyflask/apis/startup.py b/pyflask/apis/startup.py index 28e20d48b..d72ae3cdc 100644 --- a/pyflask/apis/startup.py +++ b/pyflask/apis/startup.py @@ -22,6 +22,7 @@ def get(self): args = parser.parse_args() return args["arg"] + @startup_api.route("/preload-imports") class PreloadImports(Resource): """ @@ -31,6 +32,7 @@ class PreloadImports(Resource): even if their namespace is not always exposed to a given scope. This means that later imports simply expose the cached namespaces to their scope instead of retriggering the entire import. """ + @startup_api.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"}) def get(self): try: From 8d2a6ab24ee198c6553d0b50bcbdb8191496a040 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Wed, 6 Sep 2023 14:11:18 -0400 Subject: [PATCH 10/36] Update test_neuroconv.py --- pyflask/tests/test_neuroconv.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/pyflask/tests/test_neuroconv.py b/pyflask/tests/test_neuroconv.py index 8c363c004..4f0e793f5 100644 --- a/pyflask/tests/test_neuroconv.py +++ b/pyflask/tests/test_neuroconv.py @@ -2,9 +2,8 @@ from utils import get, post, get_converter_output_schema -# --------------------- Tests --------------------- -# Accesses the dictionary of all interfaces and their metadata def test_get_all_interfaces(client): + """Accesses the dictionary of all interfaces and their metadata.""" validate( get("neuroconv", client), schema={ @@ -23,14 +22,14 @@ def test_get_all_interfaces(client): ) -# Test single interface schema request def test_single_schema_request(client): + """Test single interface schema request.""" interfaces = {"myname": "SpikeGLXRecordingInterface"} validate(post("neuroconv/schema", interfaces, client), schema=get_converter_output_schema(interfaces)) -# Uses the NWBConverter Class to combine multiple interfaces def test_multiple_schema_request(client): + """Uses the NWBConverter Class to combine multiple interfaces.""" interfaces = {"myname": "SpikeGLXRecordingInterface", "myphyinterface": "PhySortingInterface"} data = post("/neuroconv/schema", interfaces, client) validate(data, schema=get_converter_output_schema(interfaces)) From aa3e202791e75b72531599f1ea3d0e40f4a2fcae Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Wed, 6 Sep 2023 14:12:34 -0400 Subject: [PATCH 11/36] Create test_startup.py --- pyflask/tests/test_startup.py | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 pyflask/tests/test_startup.py diff --git a/pyflask/tests/test_startup.py b/pyflask/tests/test_startup.py new file mode 100644 index 000000000..a99fa6e2a --- /dev/null +++ b/pyflask/tests/test_startup.py @@ -0,0 +1,7 @@ +from utils import get, post, get_converter_output_schema + + +def test_preload_imports(client): + """Verify that the preload import endpoint returned good status.""" + result = get("startup", client) + assert result == True From 1ca85232b4ec9c61e102849b5e31c5a84d6ae381 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Wed, 6 Sep 2023 14:13:04 -0400 Subject: [PATCH 12/36] Update test_startup.py --- pyflask/tests/test_startup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyflask/tests/test_startup.py b/pyflask/tests/test_startup.py index a99fa6e2a..42cead8c7 100644 --- a/pyflask/tests/test_startup.py +++ b/pyflask/tests/test_startup.py @@ -3,5 +3,5 @@ def test_preload_imports(client): """Verify that the preload import endpoint returned good status.""" - result = get("startup", client) + result = get("startup/preload-imports", client) assert result == True From 483c9f1819a616ac698595960dad9552b8940452 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Wed, 6 Sep 2023 15:29:51 -0400 Subject: [PATCH 13/36] Update environment-Windows.yml --- environments/environment-Windows.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/environments/environment-Windows.yml b/environments/environment-Windows.yml index 7faeba0ee..eb846dee2 100644 --- a/environments/environment-Windows.yml +++ b/environments/environment-Windows.yml @@ -20,3 +20,4 @@ dependencies: - hdmf >= 3.7.0 - pytest == 7.2.2 - pytest-cov == 4.1.0 + - email-validator == 2.0.0 From 19781793d74a6c0fd4e61756bf641e4713f03f13 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Wed, 6 Sep 2023 16:30:23 -0400 Subject: [PATCH 14/36] add hidden import --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index bad2cfe3e..45d4c17ae 100644 --- a/package.json +++ b/package.json @@ -15,7 +15,7 @@ "build:win": "npm run build && npm run build:flask:win && npm run build:electron:win", "build:mac": "npm run build && npm run build:flask:unix && npm run build:electron:mac", "build:linux": "npm run build && npm run build:flask:unix && npm run build:electron:linux", - "build:flask:base": "python -m PyInstaller --name nwb-guide --onedir --clean --noconfirm ./pyflask/app.py --distpath ./build/flask --collect-data jsonschema_specifications --collect-all nwbinspector --collect-all neuroconv --collect-all pynwb --collect-all hdmf --collect-all ci_info --hidden-import scipy._distributor_init --hidden-import scipy._lib.messagestream --hidden-import scipy._lib._ccallback --hidden-import scipy._lib._testutils", + "build:flask:base": "python -m PyInstaller --name nwb-guide --onedir --clean --noconfirm ./pyflask/app.py --distpath ./build/flask --collect-data jsonschema_specifications --collect-all nwbinspector --collect-all neuroconv --collect-all pynwb --collect-all hdmf --collect-all ci_info --hidden-import scipy._distributor_init --hidden-import scipy._lib.messagestream --hidden-import scipy._lib._ccallback --hidden-import scipy._lib._testutils --hidden-import email_validator", "build:flask:win": "npm run build:flask:base -- --add-data ./paths.config.json;. --add-data ./package.json;.", "build:flask:unix": "npm run build:flask:base -- --add-data ./paths.config.json:. --add-data ./package.json:. --collect-all ndx_dandi_icephys", "build:electron:win": "electron-builder build --win --publish never", From afa547e2630738530fe3e8f587890dfdcd17ca12 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Wed, 6 Sep 2023 16:48:32 -0400 Subject: [PATCH 15/36] past the pydantic problem; add schema files for ndx_dandi_icephys --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 45d4c17ae..e61702881 100644 --- a/package.json +++ b/package.json @@ -15,7 +15,7 @@ "build:win": "npm run build && npm run build:flask:win && npm run build:electron:win", "build:mac": "npm run build && npm run build:flask:unix && npm run build:electron:mac", "build:linux": "npm run build && npm run build:flask:unix && npm run build:electron:linux", - "build:flask:base": "python -m PyInstaller --name nwb-guide --onedir --clean --noconfirm ./pyflask/app.py --distpath ./build/flask --collect-data jsonschema_specifications --collect-all nwbinspector --collect-all neuroconv --collect-all pynwb --collect-all hdmf --collect-all ci_info --hidden-import scipy._distributor_init --hidden-import scipy._lib.messagestream --hidden-import scipy._lib._ccallback --hidden-import scipy._lib._testutils --hidden-import email_validator", + "build:flask:base": "python -m PyInstaller --name nwb-guide --onedir --clean --noconfirm ./pyflask/app.py --distpath ./build/flask --collect-data jsonschema_specifications --collect-all nwbinspector --collect-all neuroconv --collect-all pynwb --collect-all hdmf --collect-all ci_info --collect-all ndx_dandi_icephys --hidden-import scipy._distributor_init --hidden-import scipy._lib.messagestream --hidden-import scipy._lib._ccallback --hidden-import scipy._lib._testutils --hidden-import email_validator", "build:flask:win": "npm run build:flask:base -- --add-data ./paths.config.json;. --add-data ./package.json;.", "build:flask:unix": "npm run build:flask:base -- --add-data ./paths.config.json:. --add-data ./package.json:. --collect-all ndx_dandi_icephys", "build:electron:win": "electron-builder build --win --publish never", From e1f13248d790b8cfef0825e3eb7d74adb944a042 Mon Sep 17 00:00:00 2001 From: Garrett Michael Flynn <46533749+GarrettMFlynn@users.noreply.github.com> Date: Thu, 7 Sep 2023 08:36:16 -0700 Subject: [PATCH 16/36] Do not return after fetch request and ensure interfaces are defined --- src/renderer/src/index.ts | 7 ++----- .../src/stories/pages/guided-mode/data/GuidedStructure.js | 6 +++--- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/src/renderer/src/index.ts b/src/renderer/src/index.ts index 54d841024..c13b6f258 100644 --- a/src/renderer/src/index.ts +++ b/src/renderer/src/index.ts @@ -101,10 +101,7 @@ const serverIsLiveStartup = async () => { }; // Preload Flask imports for faster on-demand operations -const preloadFlaskImports = async () => { - const result = await fetch(`${baseUrl}/startup/preload-imports`).then(res => res.json()).catch(e => e) - return result -}; +const preloadFlaskImports = async () => await fetch(`${baseUrl}/startup/preload-imports`).then(res => res.json()).catch(e => e) let openPythonStatusNotyf: undefined | any; @@ -112,7 +109,7 @@ async function pythonServerOpened() { // Confirm requests are actually received by the server const isLive = await serverIsLiveStartup() - if (isLive) return preloadFlaskImports() // initiate preload of Flask imports + if (isLive) await preloadFlaskImports() // initiate preload of Flask imports if (!isLive) return pythonServerClosed() // Update server status and throw a notification diff --git a/src/renderer/src/stories/pages/guided-mode/data/GuidedStructure.js b/src/renderer/src/stories/pages/guided-mode/data/GuidedStructure.js index 93938a8ca..b4c24266d 100644 --- a/src/renderer/src/stories/pages/guided-mode/data/GuidedStructure.js +++ b/src/renderer/src/stories/pages/guided-mode/data/GuidedStructure.js @@ -75,9 +75,9 @@ export class GuidedStructurePage extends Page { }; async updated() { - const selected = this.info.globalState.interfaces; + const { interfaces = {} } = this.info.globalState; - if (Object.keys(selected).length > 0) this.list.emptyMessage = "Loading valid interfaces..."; + if (Object.keys(interfaces).length > 0) this.list.emptyMessage = "Loading valid interfaces..."; this.search.options = await fetch(`${baseUrl}/neuroconv`) .then((res) => res.json()) @@ -93,7 +93,7 @@ export class GuidedStructurePage extends Page { ) .catch((e) => console.error(e)); - for (const [key, name] of Object.entries(selected || {})) { + for (const [key, name] of Object.entries(interfaces)) { let found = this.search.options?.find((o) => o.value === name); // If not found, spoof based on the key and names provided previously From 3b06fd9c78ee52c2a2bc529e04a01abe6dfcc907 Mon Sep 17 00:00:00 2001 From: Garrett Date: Thu, 7 Sep 2023 09:32:05 -0700 Subject: [PATCH 17/36] Remove all internal files / folders related to a conversion --- src/renderer/src/dependencies/simple.js | 3 +++ src/renderer/src/progress/index.js | 20 ++++++++++++++----- .../stories/pages/guided-mode/ProgressCard.js | 2 +- 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/src/renderer/src/dependencies/simple.js b/src/renderer/src/dependencies/simple.js index 35c3adafd..95152a68f 100644 --- a/src/renderer/src/dependencies/simple.js +++ b/src/renderer/src/dependencies/simple.js @@ -12,4 +12,7 @@ export const homeDirectory = app?.getPath("home") ?? ""; export const appDirectory = homeDirectory ? joinPath(homeDirectory, paths.root) : ""; export const guidedProgressFilePath = homeDirectory ? joinPath(appDirectory, ...paths.subfolders.progress) : ""; +export const stubSaveFolderPath = homeDirectory ? joinPath(homeDirectory, paths["root"], ...paths.subfolders.stubs) : ""; +export const conversionSaveFolderPath = homeDirectory ? joinPath(homeDirectory, paths["root"], ...paths.subfolders.conversions) : ""; + export const isStorybook = window.location.href.includes("iframe.html"); diff --git a/src/renderer/src/progress/index.js b/src/renderer/src/progress/index.js index b5c7c2121..958c6da27 100644 --- a/src/renderer/src/progress/index.js +++ b/src/renderer/src/progress/index.js @@ -1,6 +1,6 @@ import Swal from "sweetalert2"; -import { guidedProgressFilePath, reloadPageToHome, isStorybook, appDirectory } from "../dependencies/simple.js"; +import { guidedProgressFilePath, reloadPageToHome, isStorybook, appDirectory, stubSaveFolderPath, conversionSaveFolderPath } from "../dependencies/simple.js"; import { fs } from "../electron/index.js"; import { joinPath, runOnLoad } from "../globals.js"; import { merge } from "../stories/pages/utils.js"; @@ -102,14 +102,14 @@ export function resume(name) { export const remove = async (name) => { const result = await Swal.fire({ - title: `Are you sure you would like to delete NWB GUIDE progress made on the dataset: ${name}?`, - text: "Your progress file will be deleted permanently, and all existing progress will be lost.", + title: `Are you sure you would like to delete this conversion pipeline?`, + html: `All related files will be deleted permanently, and existing progress will be lost.`, icon: "warning", heightAuto: false, showCancelButton: true, confirmButtonColor: "#3085d6", cancelButtonColor: "#d33", - confirmButtonText: "Delete progress file", + confirmButtonText: `Delete ${name}`, cancelButtonText: "Cancel", focusCancel: true, }); @@ -119,9 +119,19 @@ export const remove = async (name) => { const progressFilePathToDelete = joinPath(guidedProgressFilePath, name + ".json"); //delete the progress file - if (fs) fs.unlinkSync(progressFilePathToDelete, (err) => console.log(err)); + if (fs) fs.unlinkSync(progressFilePathToDelete); else localStorage.removeItem(progressFilePathToDelete); + if (fs) { + + // delete default stub location + fs.rmSync(joinPath(stubSaveFolderPath, name), { recursive: true, force: true }); + + // delete default conversion location + fs.rmSync(joinPath(conversionSaveFolderPath, name), { recursive: true, force: true }); + + } + return true; } diff --git a/src/renderer/src/stories/pages/guided-mode/ProgressCard.js b/src/renderer/src/stories/pages/guided-mode/ProgressCard.js index 062ae8fe3..d66b0e8de 100644 --- a/src/renderer/src/stories/pages/guided-mode/ProgressCard.js +++ b/src/renderer/src/stories/pages/guided-mode/ProgressCard.js @@ -126,7 +126,7 @@ export class ProgressCard extends LitElement { @click=${(ev) => progress.deleteProgressCard(ev.target)} > - Delete progress file + Delete pipeline From 888a9df455855d716f60a53d97261a46af121a06 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 7 Sep 2023 16:36:42 +0000 Subject: [PATCH 18/36] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/renderer/src/dependencies/simple.js | 8 ++++++-- src/renderer/src/progress/index.js | 11 ++++++++--- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/src/renderer/src/dependencies/simple.js b/src/renderer/src/dependencies/simple.js index 95152a68f..97f8d2148 100644 --- a/src/renderer/src/dependencies/simple.js +++ b/src/renderer/src/dependencies/simple.js @@ -12,7 +12,11 @@ export const homeDirectory = app?.getPath("home") ?? ""; export const appDirectory = homeDirectory ? joinPath(homeDirectory, paths.root) : ""; export const guidedProgressFilePath = homeDirectory ? joinPath(appDirectory, ...paths.subfolders.progress) : ""; -export const stubSaveFolderPath = homeDirectory ? joinPath(homeDirectory, paths["root"], ...paths.subfolders.stubs) : ""; -export const conversionSaveFolderPath = homeDirectory ? joinPath(homeDirectory, paths["root"], ...paths.subfolders.conversions) : ""; +export const stubSaveFolderPath = homeDirectory + ? joinPath(homeDirectory, paths["root"], ...paths.subfolders.stubs) + : ""; +export const conversionSaveFolderPath = homeDirectory + ? joinPath(homeDirectory, paths["root"], ...paths.subfolders.conversions) + : ""; export const isStorybook = window.location.href.includes("iframe.html"); diff --git a/src/renderer/src/progress/index.js b/src/renderer/src/progress/index.js index 958c6da27..b3dfb9ea8 100644 --- a/src/renderer/src/progress/index.js +++ b/src/renderer/src/progress/index.js @@ -1,6 +1,13 @@ import Swal from "sweetalert2"; -import { guidedProgressFilePath, reloadPageToHome, isStorybook, appDirectory, stubSaveFolderPath, conversionSaveFolderPath } from "../dependencies/simple.js"; +import { + guidedProgressFilePath, + reloadPageToHome, + isStorybook, + appDirectory, + stubSaveFolderPath, + conversionSaveFolderPath, +} from "../dependencies/simple.js"; import { fs } from "../electron/index.js"; import { joinPath, runOnLoad } from "../globals.js"; import { merge } from "../stories/pages/utils.js"; @@ -123,13 +130,11 @@ export const remove = async (name) => { else localStorage.removeItem(progressFilePathToDelete); if (fs) { - // delete default stub location fs.rmSync(joinPath(stubSaveFolderPath, name), { recursive: true, force: true }); // delete default conversion location fs.rmSync(joinPath(conversionSaveFolderPath, name), { recursive: true, force: true }); - } return true; From 25ac15ea3fdccacd0e3881eeb3e2d8bf53595add Mon Sep 17 00:00:00 2001 From: Garrett Date: Thu, 7 Sep 2023 10:05:35 -0700 Subject: [PATCH 19/36] Update package.json --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index e61702881..faf9ccdf7 100644 --- a/package.json +++ b/package.json @@ -24,7 +24,7 @@ "test": "npm run test:app && npm run test:server", "test:app": "vitest run", "test:server": "pytest pyflask/tests/ -s", - "test:executable": "concurrently -n EXE,TEST --kill-others \"node tests/testPyinstallerExecutable.js --port 3434 --forever\" \"pytest pyflask/tests/ -s --target http://localhost:3434\"", + "test:executable": "concurrently -n EXE,TEST --kill-others-on-fail --success last \"node tests/testPyinstallerExecutable.js --port 3434 --forever\" \"pytest pyflask/tests/ -s --target http://localhost:3434\"", "test:coverage": "npm run coverage:app && npm run coverage:server", "coverage:app": "vitest run --coverage", "coverage:server": "pytest pyflask/tests/ -s --cov=pyflask --cov-report=xml", From e8addf014413944bd794d9cc29d1ad376602768e Mon Sep 17 00:00:00 2001 From: Garrett Date: Thu, 7 Sep 2023 10:20:19 -0700 Subject: [PATCH 20/36] Update package.json --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index faf9ccdf7..5eb1fa7f2 100644 --- a/package.json +++ b/package.json @@ -24,7 +24,7 @@ "test": "npm run test:app && npm run test:server", "test:app": "vitest run", "test:server": "pytest pyflask/tests/ -s", - "test:executable": "concurrently -n EXE,TEST --kill-others-on-fail --success last \"node tests/testPyinstallerExecutable.js --port 3434 --forever\" \"pytest pyflask/tests/ -s --target http://localhost:3434\"", + "test:executable": "concurrently -n EXE,TEST --kill-others --success last \"node tests/testPyinstallerExecutable.js --port 3434 --forever\" \"pytest pyflask/tests/ -s --target http://localhost:3434\"", "test:coverage": "npm run coverage:app && npm run coverage:server", "coverage:app": "vitest run --coverage", "coverage:server": "pytest pyflask/tests/ -s --cov=pyflask --cov-report=xml", From 18280d486bf13da7663781af64829ccd6e5a7a64 Mon Sep 17 00:00:00 2001 From: Garrett Date: Thu, 7 Sep 2023 11:10:24 -0700 Subject: [PATCH 21/36] Update package.json --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 5eb1fa7f2..e69551d9b 100644 --- a/package.json +++ b/package.json @@ -24,7 +24,7 @@ "test": "npm run test:app && npm run test:server", "test:app": "vitest run", "test:server": "pytest pyflask/tests/ -s", - "test:executable": "concurrently -n EXE,TEST --kill-others --success last \"node tests/testPyinstallerExecutable.js --port 3434 --forever\" \"pytest pyflask/tests/ -s --target http://localhost:3434\"", + "test:executable": "concurrently -n EXE,TEST --kill-others --success all \"node tests/testPyinstallerExecutable.js --port 3434 --forever\" \"pytest pyflask/tests/ -s --target http://localhost:3434\"", "test:coverage": "npm run coverage:app && npm run coverage:server", "coverage:app": "vitest run --coverage", "coverage:server": "pytest pyflask/tests/ -s --cov=pyflask --cov-report=xml", From 40c3c17b0ce7dafdcc4ada4387af2b6c5892e8c8 Mon Sep 17 00:00:00 2001 From: Garrett Date: Thu, 7 Sep 2023 11:30:59 -0700 Subject: [PATCH 22/36] kill gracefully --- package.json | 2 +- tests/testPyinstallerExecutable.js | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index e69551d9b..e61702881 100644 --- a/package.json +++ b/package.json @@ -24,7 +24,7 @@ "test": "npm run test:app && npm run test:server", "test:app": "vitest run", "test:server": "pytest pyflask/tests/ -s", - "test:executable": "concurrently -n EXE,TEST --kill-others --success all \"node tests/testPyinstallerExecutable.js --port 3434 --forever\" \"pytest pyflask/tests/ -s --target http://localhost:3434\"", + "test:executable": "concurrently -n EXE,TEST --kill-others \"node tests/testPyinstallerExecutable.js --port 3434 --forever\" \"pytest pyflask/tests/ -s --target http://localhost:3434\"", "test:coverage": "npm run coverage:app && npm run coverage:server", "coverage:app": "vitest run --coverage", "coverage:server": "pytest pyflask/tests/ -s --cov=pyflask --cov-report=xml", diff --git a/tests/testPyinstallerExecutable.js b/tests/testPyinstallerExecutable.js index 4611dba7a..fffab8b7a 100644 --- a/tests/testPyinstallerExecutable.js +++ b/tests/testPyinstallerExecutable.js @@ -52,5 +52,11 @@ function handleProcess(proc, id = "process") { console.error(`[${id}] Exit: ${code}`); error(); }); + + process.once('SIGTERM', () => { + proc.kill(0) + process.exit(0) + }) // Exit gracefully if this is caught + } else console.error("child process failed to start on port" + port); } From fa3f445c100c7d3887873f74defd272ed407dcc9 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 7 Sep 2023 18:32:21 +0000 Subject: [PATCH 23/36] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/testPyinstallerExecutable.js | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tests/testPyinstallerExecutable.js b/tests/testPyinstallerExecutable.js index fffab8b7a..a8dfb910e 100644 --- a/tests/testPyinstallerExecutable.js +++ b/tests/testPyinstallerExecutable.js @@ -53,10 +53,9 @@ function handleProcess(proc, id = "process") { error(); }); - process.once('SIGTERM', () => { - proc.kill(0) - process.exit(0) - }) // Exit gracefully if this is caught - + process.once("SIGTERM", () => { + proc.kill(0); + process.exit(0); + }); // Exit gracefully if this is caught } else console.error("child process failed to start on port" + port); } From 6db9bcb37a6fcb84ad9eb06caefdbc8ab9c2fdb1 Mon Sep 17 00:00:00 2001 From: Garrett Date: Thu, 7 Sep 2023 12:38:45 -0700 Subject: [PATCH 24/36] Update testPyinstallerExecutable.js --- tests/testPyinstallerExecutable.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/testPyinstallerExecutable.js b/tests/testPyinstallerExecutable.js index a8dfb910e..d8ff46469 100644 --- a/tests/testPyinstallerExecutable.js +++ b/tests/testPyinstallerExecutable.js @@ -54,8 +54,10 @@ function handleProcess(proc, id = "process") { }); process.once("SIGTERM", () => { + console.log('GOT SIGTERM!!!!') proc.kill(0); process.exit(0); }); // Exit gracefully if this is caught + } else console.error("child process failed to start on port" + port); } From b77f2c6091e58254dd5131714cf144d129bea0fd Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 7 Sep 2023 19:39:01 +0000 Subject: [PATCH 25/36] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/testPyinstallerExecutable.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/testPyinstallerExecutable.js b/tests/testPyinstallerExecutable.js index d8ff46469..7a59e3582 100644 --- a/tests/testPyinstallerExecutable.js +++ b/tests/testPyinstallerExecutable.js @@ -54,10 +54,9 @@ function handleProcess(proc, id = "process") { }); process.once("SIGTERM", () => { - console.log('GOT SIGTERM!!!!') + console.log("GOT SIGTERM!!!!"); proc.kill(0); process.exit(0); }); // Exit gracefully if this is caught - } else console.error("child process failed to start on port" + port); } From ad7c74e6692903296cf10a41115e85c35c7d5561 Mon Sep 17 00:00:00 2001 From: Garrett Date: Thu, 7 Sep 2023 13:00:10 -0700 Subject: [PATCH 26/36] Update testPyinstallerExecutable.js --- tests/testPyinstallerExecutable.js | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/testPyinstallerExecutable.js b/tests/testPyinstallerExecutable.js index d8ff46469..c8fd00864 100644 --- a/tests/testPyinstallerExecutable.js +++ b/tests/testPyinstallerExecutable.js @@ -26,6 +26,13 @@ const regex = /.+Error: .+/; function handleProcess(proc, id = "process") { if (proc != null) { + + process.on("SIGTERM", () => { + console.log('GOT SIGTERM!!!!') + proc.kill(0); + process.exit(0); + }); // Exit gracefully if this is caught + // Listen for errors from Python process proc.stderr.on("data", function (data) { const message = data.toString(); @@ -53,11 +60,5 @@ function handleProcess(proc, id = "process") { error(); }); - process.once("SIGTERM", () => { - console.log('GOT SIGTERM!!!!') - proc.kill(0); - process.exit(0); - }); // Exit gracefully if this is caught - } else console.error("child process failed to start on port" + port); } From 496fdee5ac3807a845da846768d4390fd30b28fa Mon Sep 17 00:00:00 2001 From: Garrett Michael Flynn <46533749+GarrettMFlynn@users.noreply.github.com> Date: Thu, 7 Sep 2023 14:53:48 -0700 Subject: [PATCH 27/36] Wait to ensure server has started --- package.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index e61702881..de1b1d6b2 100644 --- a/package.json +++ b/package.json @@ -24,7 +24,8 @@ "test": "npm run test:app && npm run test:server", "test:app": "vitest run", "test:server": "pytest pyflask/tests/ -s", - "test:executable": "concurrently -n EXE,TEST --kill-others \"node tests/testPyinstallerExecutable.js --port 3434 --forever\" \"pytest pyflask/tests/ -s --target http://localhost:3434\"", + "wait3s": "node -e \"setTimeout(() => process.exit(0),3000)\"", + "test:executable": "concurrently -n EXE,TEST --kill-others \"node tests/testPyinstallerExecutable.js --port 3434 --forever\" \"npm run wait3s && pytest pyflask/tests/ -s --target http://localhost:3434\"", "test:coverage": "npm run coverage:app && npm run coverage:server", "coverage:app": "vitest run --coverage", "coverage:server": "pytest pyflask/tests/ -s --cov=pyflask --cov-report=xml", From 412c1a1ee66aab807ff1c46a4249d475f0fcb50e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 7 Sep 2023 21:56:15 +0000 Subject: [PATCH 28/36] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/testPyinstallerExecutable.js | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/testPyinstallerExecutable.js b/tests/testPyinstallerExecutable.js index c8fd00864..1135eaf99 100644 --- a/tests/testPyinstallerExecutable.js +++ b/tests/testPyinstallerExecutable.js @@ -26,13 +26,12 @@ const regex = /.+Error: .+/; function handleProcess(proc, id = "process") { if (proc != null) { - process.on("SIGTERM", () => { - console.log('GOT SIGTERM!!!!') + console.log("GOT SIGTERM!!!!"); proc.kill(0); process.exit(0); }); // Exit gracefully if this is caught - + // Listen for errors from Python process proc.stderr.on("data", function (data) { const message = data.toString(); @@ -59,6 +58,5 @@ function handleProcess(proc, id = "process") { console.error(`[${id}] Exit: ${code}`); error(); }); - } else console.error("child process failed to start on port" + port); } From 603fba6b64eb95c1b405fefaa8fce530f0b22ec8 Mon Sep 17 00:00:00 2001 From: Garrett Date: Thu, 7 Sep 2023 15:29:13 -0700 Subject: [PATCH 29/36] Test locally until it works --- package.json | 2 +- tests/testPyinstallerExecutable.js | 6 ------ 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/package.json b/package.json index de1b1d6b2..6d545d081 100644 --- a/package.json +++ b/package.json @@ -25,7 +25,7 @@ "test:app": "vitest run", "test:server": "pytest pyflask/tests/ -s", "wait3s": "node -e \"setTimeout(() => process.exit(0),3000)\"", - "test:executable": "concurrently -n EXE,TEST --kill-others \"node tests/testPyinstallerExecutable.js --port 3434 --forever\" \"npm run wait3s && pytest pyflask/tests/ -s --target http://localhost:3434\"", + "test:executable": "concurrently -n EXE,TEST --kill-others --success first \"node tests/testPyinstallerExecutable.js --port 3434 --forever\" \"npm run wait3s && pytest pyflask/tests/ -s --target http://localhost:3434\"", "test:coverage": "npm run coverage:app && npm run coverage:server", "coverage:app": "vitest run --coverage", "coverage:server": "pytest pyflask/tests/ -s --cov=pyflask --cov-report=xml", diff --git a/tests/testPyinstallerExecutable.js b/tests/testPyinstallerExecutable.js index c8fd00864..cf1deec14 100644 --- a/tests/testPyinstallerExecutable.js +++ b/tests/testPyinstallerExecutable.js @@ -27,12 +27,6 @@ const regex = /.+Error: .+/; function handleProcess(proc, id = "process") { if (proc != null) { - process.on("SIGTERM", () => { - console.log('GOT SIGTERM!!!!') - proc.kill(0); - process.exit(0); - }); // Exit gracefully if this is caught - // Listen for errors from Python process proc.stderr.on("data", function (data) { const message = data.toString(); From e6dc9684ea036199eea425012289682ce8be8837 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 7 Sep 2023 22:29:48 +0000 Subject: [PATCH 30/36] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/testPyinstallerExecutable.js | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/testPyinstallerExecutable.js b/tests/testPyinstallerExecutable.js index 2b44cb542..4611dba7a 100644 --- a/tests/testPyinstallerExecutable.js +++ b/tests/testPyinstallerExecutable.js @@ -26,7 +26,6 @@ const regex = /.+Error: .+/; function handleProcess(proc, id = "process") { if (proc != null) { - // Listen for errors from Python process proc.stderr.on("data", function (data) { const message = data.toString(); From 0a9a521299e988d3e3b8be3693e0ec949723405b Mon Sep 17 00:00:00 2001 From: rly Date: Fri, 8 Sep 2023 11:02:01 +0200 Subject: [PATCH 31/36] Try adjustment to mac env --- environments/environment-MAC.yml | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/environments/environment-MAC.yml b/environments/environment-MAC.yml index cd76d4596..b2333fa86 100644 --- a/environments/environment-MAC.yml +++ b/environments/environment-MAC.yml @@ -1,11 +1,11 @@ name: nwb-guide channels: - - defaults - conda-forge + - defaults dependencies: - - python = 3.9.17 - - nodejs = 18.16.1 + - python = 3.9.18 - PyInstaller = 5.13.0 + - nodejs = 18.16.1 - numcodecs = 0.11.0 - jsonschema = 4.18.0 # this installs jsonschema-specifications which is needed for validation - pydantic[email] = 2.0.2 # email validator is used by dandi @@ -14,9 +14,7 @@ dependencies: - chardet == 5.1.0 - configparser == 6.0.0 - flask == 2.3.2 - - flask-cors === 3.0.10 + - flask-cors == 4.0.0 - flask_restx == 1.1.0 - neuroconv @ git+https://github.com/catalystneuro/neuroconv.git@main#neuroconv[full] - - hdmf >= 3.7.0 - - pytest == 7.2.2 - - pytest-cov == 4.1.0 + - pytest == 7.4.0 From aa88b9f97258ba6d72332c360fbd8d84b4019c50 Mon Sep 17 00:00:00 2001 From: Ryan Ly Date: Fri, 8 Sep 2023 06:37:43 -0700 Subject: [PATCH 32/36] Discard changes to environments/environment-MAC.yml --- environments/environment-MAC.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/environments/environment-MAC.yml b/environments/environment-MAC.yml index b2333fa86..cd76d4596 100644 --- a/environments/environment-MAC.yml +++ b/environments/environment-MAC.yml @@ -1,11 +1,11 @@ name: nwb-guide channels: - - conda-forge - defaults + - conda-forge dependencies: - - python = 3.9.18 - - PyInstaller = 5.13.0 + - python = 3.9.17 - nodejs = 18.16.1 + - PyInstaller = 5.13.0 - numcodecs = 0.11.0 - jsonschema = 4.18.0 # this installs jsonschema-specifications which is needed for validation - pydantic[email] = 2.0.2 # email validator is used by dandi @@ -14,7 +14,9 @@ dependencies: - chardet == 5.1.0 - configparser == 6.0.0 - flask == 2.3.2 - - flask-cors == 4.0.0 + - flask-cors === 3.0.10 - flask_restx == 1.1.0 - neuroconv @ git+https://github.com/catalystneuro/neuroconv.git@main#neuroconv[full] - - pytest == 7.4.0 + - hdmf >= 3.7.0 + - pytest == 7.2.2 + - pytest-cov == 4.1.0 From b236bba9e2e04a24f665564450bccfca007e35a1 Mon Sep 17 00:00:00 2001 From: Garrett Date: Mon, 11 Sep 2023 10:37:45 -0700 Subject: [PATCH 33/36] Rename and change iframe src --- docs/{support.rst => format_support.rst} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename docs/{support.rst => format_support.rst} (59%) diff --git a/docs/support.rst b/docs/format_support.rst similarity index 59% rename from docs/support.rst rename to docs/format_support.rst index 38c26bad0..a6363ace6 100644 --- a/docs/support.rst +++ b/docs/format_support.rst @@ -4,4 +4,4 @@ The following Google Sheet is a live record of all the supported formats in the .. raw:: html - + From 778d8372e55f8e4703361f1091daa1264270fdc9 Mon Sep 17 00:00:00 2001 From: Garrett Date: Mon, 11 Sep 2023 10:39:54 -0700 Subject: [PATCH 34/36] Update text --- docs/format_support.rst | 2 +- docs/index.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/format_support.rst b/docs/format_support.rst index a6363ace6..34ff68a5e 100644 --- a/docs/format_support.rst +++ b/docs/format_support.rst @@ -1,6 +1,6 @@ Ecosystem Format Support ======================================= -The following Google Sheet is a live record of all the supported formats in the NWB GUIDE and underlying ecosystem. +The following is a live record of all the supported formats in the NWB GUIDE and underlying ecosystem. .. raw:: html diff --git a/docs/index.rst b/docs/index.rst index 895337c80..432034626 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -19,4 +19,4 @@ The resulting files are fully compliant with the best practices expected of the :maxdepth: 2 developer_guide - support + format_support From db3a46388c147981d2ca420b8ed97674c534f622 Mon Sep 17 00:00:00 2001 From: Garrett Date: Mon, 11 Sep 2023 10:47:30 -0700 Subject: [PATCH 35/36] Update format_support.rst --- docs/format_support.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/format_support.rst b/docs/format_support.rst index 34ff68a5e..831d5e462 100644 --- a/docs/format_support.rst +++ b/docs/format_support.rst @@ -4,4 +4,4 @@ The following is a live record of all the supported formats in the NWB GUIDE and .. raw:: html - + From 28627b4755d2d4ee98799f55016c95f93761a84a Mon Sep 17 00:00:00 2001 From: Garrett Date: Mon, 11 Sep 2023 11:17:29 -0700 Subject: [PATCH 36/36] Update format_support.rst --- docs/format_support.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/format_support.rst b/docs/format_support.rst index 831d5e462..d2f35083e 100644 --- a/docs/format_support.rst +++ b/docs/format_support.rst @@ -4,4 +4,4 @@ The following is a live record of all the supported formats in the NWB GUIDE and .. raw:: html - +