From ee76961fcfdbbb40a09b6d617ec23ef863f3bb71 Mon Sep 17 00:00:00 2001 From: Garrett Date: Wed, 6 Sep 2023 09:24:47 -0700 Subject: [PATCH 01/62] Update index.rst --- docs/index.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/index.rst b/docs/index.rst index 57fa8202f..166949267 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -19,3 +19,9 @@ The resulting files are fully compliant with the best practices expected of the :maxdepth: 2 developer_guide + +To see all the file types supported by the NWB GUIDE, visit the following website: + +.. raw:: html + + \ No newline at end of file From bb95024b4f4f67ddc69f4551d1c5097db490366c Mon Sep 17 00:00:00 2001 From: Garrett Date: Wed, 6 Sep 2023 09:33:35 -0700 Subject: [PATCH 02/62] Update index.rst --- docs/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index 166949267..c6a513a83 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -24,4 +24,4 @@ To see all the file types supported by the NWB GUIDE, visit the following websit .. raw:: html - \ No newline at end of file + \ No newline at end of file From 7e88bfffcda43ba2c6205c8b9ce17e7c3b17ab7f Mon Sep 17 00:00:00 2001 From: Garrett Date: Wed, 6 Sep 2023 09:41:12 -0700 Subject: [PATCH 03/62] Move to separate page --- docs/index.rst | 7 +------ docs/support.rst | 5 +++++ 2 files changed, 6 insertions(+), 6 deletions(-) create mode 100644 docs/support.rst diff --git a/docs/index.rst b/docs/index.rst index c6a513a83..952fdc2f3 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -19,9 +19,4 @@ The resulting files are fully compliant with the best practices expected of the :maxdepth: 2 developer_guide - -To see all the file types supported by the NWB GUIDE, visit the following website: - -.. raw:: html - - \ No newline at end of file + support \ No newline at end of file diff --git a/docs/support.rst b/docs/support.rst new file mode 100644 index 000000000..ddefced66 --- /dev/null +++ b/docs/support.rst @@ -0,0 +1,5 @@ +Ecosystem Format Support +======================================= +.. raw:: html + + \ No newline at end of file From df7cf71f0ca36fb6984901761c0e6761d7b0c062 Mon Sep 17 00:00:00 2001 From: Garrett Date: Wed, 6 Sep 2023 09:41:54 -0700 Subject: [PATCH 04/62] Update support.rst --- docs/support.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/support.rst b/docs/support.rst index ddefced66..130d02ec3 100644 --- a/docs/support.rst +++ b/docs/support.rst @@ -2,4 +2,4 @@ Ecosystem Format Support ======================================= .. raw:: html - \ No newline at end of file + \ No newline at end of file From eaa84774a4d8709f48a4b4a18bee148d7bd82b46 Mon Sep 17 00:00:00 2001 From: Garrett Date: Wed, 6 Sep 2023 09:51:55 -0700 Subject: [PATCH 05/62] Use the published sheet --- docs/support.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/support.rst b/docs/support.rst index 130d02ec3..38c26bad0 100644 --- a/docs/support.rst +++ b/docs/support.rst @@ -1,5 +1,7 @@ Ecosystem Format Support ======================================= +The following Google Sheet is a live record of all the supported formats in the NWB GUIDE and underlying ecosystem. + .. raw:: html - \ No newline at end of file + From d72d9fdd11264bd0605ba2e0eb9be04e3292d570 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 6 Sep 2023 17:25:09 +0000 Subject: [PATCH 06/62] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- docs/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index 952fdc2f3..895337c80 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -19,4 +19,4 @@ The resulting files are fully compliant with the best practices expected of the :maxdepth: 2 developer_guide - support \ No newline at end of file + support From 0aad5e9d67a84d31c7002b834b2ef07690e38b56 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Wed, 6 Sep 2023 13:25:42 -0400 Subject: [PATCH 07/62] Update index.ts --- src/renderer/src/index.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/renderer/src/index.ts b/src/renderer/src/index.ts index efbc4b093..54d841024 100644 --- a/src/renderer/src/index.ts +++ b/src/renderer/src/index.ts @@ -94,18 +94,25 @@ async function checkInternetConnection() { return hasInternet }; -// Check if the Pysoda server is live +// Check if the Flask server is live const serverIsLiveStartup = async () => { const echoResponse = await fetch(`${baseUrl}/startup/echo?arg=server ready`).then(res => res.json()).catch(e => e) return echoResponse === "server ready" ? true : false; }; +// Preload Flask imports for faster on-demand operations +const preloadFlaskImports = async () => { + const result = await fetch(`${baseUrl}/startup/preload-imports`).then(res => res.json()).catch(e => e) + return result +}; + let openPythonStatusNotyf: undefined | any; async function pythonServerOpened() { // Confirm requests are actually received by the server const isLive = await serverIsLiveStartup() + if (isLive) return preloadFlaskImports() // initiate preload of Flask imports if (!isLive) return pythonServerClosed() // Update server status and throw a notification From c78eff6910e3c8296931ae1d15fc21d23f9e8b7a Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Wed, 6 Sep 2023 13:31:57 -0400 Subject: [PATCH 08/62] Update startup.py --- pyflask/apis/startup.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/pyflask/apis/startup.py b/pyflask/apis/startup.py index 3824a9465..28e20d48b 100644 --- a/pyflask/apis/startup.py +++ b/pyflask/apis/startup.py @@ -1,6 +1,8 @@ """API endpoint definitions for startup operations.""" from flask_restx import Namespace, Resource +from errorHandlers import notBadRequestException + startup_api = Namespace("startup", description="API for startup commands related to the NWB GUIDE.") parser = startup_api.parser() @@ -19,3 +21,23 @@ class Echo(Resource): def get(self): args = parser.parse_args() return args["arg"] + +@startup_api.route("/preload-imports") +class PreloadImports(Resource): + """ + Preload various imports on startup instead of waiting for them later on. + + Python caches all modules that have been imported at least once in the same kernel, + even if their namespace is not always exposed to a given scope. This means that later imports + simply expose the cached namespaces to their scope instead of retriggering the entire import. + """ + @startup_api.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"}) + def get(self): + try: + import neuroconv + + return True + except Exception as exception: + if notBadRequestException(exception=exception): + startup_api.abort(500, str(exception)) + raise exception From 3fab5e5a6cd2d693cf603f5ee3bad813d85cbf2d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 6 Sep 2023 17:33:18 +0000 Subject: [PATCH 09/62] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pyflask/apis/startup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyflask/apis/startup.py b/pyflask/apis/startup.py index 28e20d48b..d72ae3cdc 100644 --- a/pyflask/apis/startup.py +++ b/pyflask/apis/startup.py @@ -22,6 +22,7 @@ def get(self): args = parser.parse_args() return args["arg"] + @startup_api.route("/preload-imports") class PreloadImports(Resource): """ @@ -31,6 +32,7 @@ class PreloadImports(Resource): even if their namespace is not always exposed to a given scope. This means that later imports simply expose the cached namespaces to their scope instead of retriggering the entire import. """ + @startup_api.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"}) def get(self): try: From 8d2a6ab24ee198c6553d0b50bcbdb8191496a040 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Wed, 6 Sep 2023 14:11:18 -0400 Subject: [PATCH 10/62] Update test_neuroconv.py --- pyflask/tests/test_neuroconv.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/pyflask/tests/test_neuroconv.py b/pyflask/tests/test_neuroconv.py index 8c363c004..4f0e793f5 100644 --- a/pyflask/tests/test_neuroconv.py +++ b/pyflask/tests/test_neuroconv.py @@ -2,9 +2,8 @@ from utils import get, post, get_converter_output_schema -# --------------------- Tests --------------------- -# Accesses the dictionary of all interfaces and their metadata def test_get_all_interfaces(client): + """Accesses the dictionary of all interfaces and their metadata.""" validate( get("neuroconv", client), schema={ @@ -23,14 +22,14 @@ def test_get_all_interfaces(client): ) -# Test single interface schema request def test_single_schema_request(client): + """Test single interface schema request.""" interfaces = {"myname": "SpikeGLXRecordingInterface"} validate(post("neuroconv/schema", interfaces, client), schema=get_converter_output_schema(interfaces)) -# Uses the NWBConverter Class to combine multiple interfaces def test_multiple_schema_request(client): + """Uses the NWBConverter Class to combine multiple interfaces.""" interfaces = {"myname": "SpikeGLXRecordingInterface", "myphyinterface": "PhySortingInterface"} data = post("/neuroconv/schema", interfaces, client) validate(data, schema=get_converter_output_schema(interfaces)) From aa3e202791e75b72531599f1ea3d0e40f4a2fcae Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Wed, 6 Sep 2023 14:12:34 -0400 Subject: [PATCH 11/62] Create test_startup.py --- pyflask/tests/test_startup.py | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 pyflask/tests/test_startup.py diff --git a/pyflask/tests/test_startup.py b/pyflask/tests/test_startup.py new file mode 100644 index 000000000..a99fa6e2a --- /dev/null +++ b/pyflask/tests/test_startup.py @@ -0,0 +1,7 @@ +from utils import get, post, get_converter_output_schema + + +def test_preload_imports(client): + """Verify that the preload import endpoint returned good status.""" + result = get("startup", client) + assert result == True From 1ca85232b4ec9c61e102849b5e31c5a84d6ae381 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Wed, 6 Sep 2023 14:13:04 -0400 Subject: [PATCH 12/62] Update test_startup.py --- pyflask/tests/test_startup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyflask/tests/test_startup.py b/pyflask/tests/test_startup.py index a99fa6e2a..42cead8c7 100644 --- a/pyflask/tests/test_startup.py +++ b/pyflask/tests/test_startup.py @@ -3,5 +3,5 @@ def test_preload_imports(client): """Verify that the preload import endpoint returned good status.""" - result = get("startup", client) + result = get("startup/preload-imports", client) assert result == True From 483c9f1819a616ac698595960dad9552b8940452 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Wed, 6 Sep 2023 15:29:51 -0400 Subject: [PATCH 13/62] Update environment-Windows.yml --- environments/environment-Windows.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/environments/environment-Windows.yml b/environments/environment-Windows.yml index 7faeba0ee..eb846dee2 100644 --- a/environments/environment-Windows.yml +++ b/environments/environment-Windows.yml @@ -20,3 +20,4 @@ dependencies: - hdmf >= 3.7.0 - pytest == 7.2.2 - pytest-cov == 4.1.0 + - email-validator == 2.0.0 From 19781793d74a6c0fd4e61756bf641e4713f03f13 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Wed, 6 Sep 2023 16:30:23 -0400 Subject: [PATCH 14/62] add hidden import --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index bad2cfe3e..45d4c17ae 100644 --- a/package.json +++ b/package.json @@ -15,7 +15,7 @@ "build:win": "npm run build && npm run build:flask:win && npm run build:electron:win", "build:mac": "npm run build && npm run build:flask:unix && npm run build:electron:mac", "build:linux": "npm run build && npm run build:flask:unix && npm run build:electron:linux", - "build:flask:base": "python -m PyInstaller --name nwb-guide --onedir --clean --noconfirm ./pyflask/app.py --distpath ./build/flask --collect-data jsonschema_specifications --collect-all nwbinspector --collect-all neuroconv --collect-all pynwb --collect-all hdmf --collect-all ci_info --hidden-import scipy._distributor_init --hidden-import scipy._lib.messagestream --hidden-import scipy._lib._ccallback --hidden-import scipy._lib._testutils", + "build:flask:base": "python -m PyInstaller --name nwb-guide --onedir --clean --noconfirm ./pyflask/app.py --distpath ./build/flask --collect-data jsonschema_specifications --collect-all nwbinspector --collect-all neuroconv --collect-all pynwb --collect-all hdmf --collect-all ci_info --hidden-import scipy._distributor_init --hidden-import scipy._lib.messagestream --hidden-import scipy._lib._ccallback --hidden-import scipy._lib._testutils --hidden-import email_validator", "build:flask:win": "npm run build:flask:base -- --add-data ./paths.config.json;. --add-data ./package.json;.", "build:flask:unix": "npm run build:flask:base -- --add-data ./paths.config.json:. --add-data ./package.json:. --collect-all ndx_dandi_icephys", "build:electron:win": "electron-builder build --win --publish never", From afa547e2630738530fe3e8f587890dfdcd17ca12 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Wed, 6 Sep 2023 16:48:32 -0400 Subject: [PATCH 15/62] past the pydantic problem; add schema files for ndx_dandi_icephys --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 45d4c17ae..e61702881 100644 --- a/package.json +++ b/package.json @@ -15,7 +15,7 @@ "build:win": "npm run build && npm run build:flask:win && npm run build:electron:win", "build:mac": "npm run build && npm run build:flask:unix && npm run build:electron:mac", "build:linux": "npm run build && npm run build:flask:unix && npm run build:electron:linux", - "build:flask:base": "python -m PyInstaller --name nwb-guide --onedir --clean --noconfirm ./pyflask/app.py --distpath ./build/flask --collect-data jsonschema_specifications --collect-all nwbinspector --collect-all neuroconv --collect-all pynwb --collect-all hdmf --collect-all ci_info --hidden-import scipy._distributor_init --hidden-import scipy._lib.messagestream --hidden-import scipy._lib._ccallback --hidden-import scipy._lib._testutils --hidden-import email_validator", + "build:flask:base": "python -m PyInstaller --name nwb-guide --onedir --clean --noconfirm ./pyflask/app.py --distpath ./build/flask --collect-data jsonschema_specifications --collect-all nwbinspector --collect-all neuroconv --collect-all pynwb --collect-all hdmf --collect-all ci_info --collect-all ndx_dandi_icephys --hidden-import scipy._distributor_init --hidden-import scipy._lib.messagestream --hidden-import scipy._lib._ccallback --hidden-import scipy._lib._testutils --hidden-import email_validator", "build:flask:win": "npm run build:flask:base -- --add-data ./paths.config.json;. --add-data ./package.json;.", "build:flask:unix": "npm run build:flask:base -- --add-data ./paths.config.json:. --add-data ./package.json:. --collect-all ndx_dandi_icephys", "build:electron:win": "electron-builder build --win --publish never", From e1f13248d790b8cfef0825e3eb7d74adb944a042 Mon Sep 17 00:00:00 2001 From: Garrett Michael Flynn <46533749+GarrettMFlynn@users.noreply.github.com> Date: Thu, 7 Sep 2023 08:36:16 -0700 Subject: [PATCH 16/62] Do not return after fetch request and ensure interfaces are defined --- src/renderer/src/index.ts | 7 ++----- .../src/stories/pages/guided-mode/data/GuidedStructure.js | 6 +++--- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/src/renderer/src/index.ts b/src/renderer/src/index.ts index 54d841024..c13b6f258 100644 --- a/src/renderer/src/index.ts +++ b/src/renderer/src/index.ts @@ -101,10 +101,7 @@ const serverIsLiveStartup = async () => { }; // Preload Flask imports for faster on-demand operations -const preloadFlaskImports = async () => { - const result = await fetch(`${baseUrl}/startup/preload-imports`).then(res => res.json()).catch(e => e) - return result -}; +const preloadFlaskImports = async () => await fetch(`${baseUrl}/startup/preload-imports`).then(res => res.json()).catch(e => e) let openPythonStatusNotyf: undefined | any; @@ -112,7 +109,7 @@ async function pythonServerOpened() { // Confirm requests are actually received by the server const isLive = await serverIsLiveStartup() - if (isLive) return preloadFlaskImports() // initiate preload of Flask imports + if (isLive) await preloadFlaskImports() // initiate preload of Flask imports if (!isLive) return pythonServerClosed() // Update server status and throw a notification diff --git a/src/renderer/src/stories/pages/guided-mode/data/GuidedStructure.js b/src/renderer/src/stories/pages/guided-mode/data/GuidedStructure.js index 93938a8ca..b4c24266d 100644 --- a/src/renderer/src/stories/pages/guided-mode/data/GuidedStructure.js +++ b/src/renderer/src/stories/pages/guided-mode/data/GuidedStructure.js @@ -75,9 +75,9 @@ export class GuidedStructurePage extends Page { }; async updated() { - const selected = this.info.globalState.interfaces; + const { interfaces = {} } = this.info.globalState; - if (Object.keys(selected).length > 0) this.list.emptyMessage = "Loading valid interfaces..."; + if (Object.keys(interfaces).length > 0) this.list.emptyMessage = "Loading valid interfaces..."; this.search.options = await fetch(`${baseUrl}/neuroconv`) .then((res) => res.json()) @@ -93,7 +93,7 @@ export class GuidedStructurePage extends Page { ) .catch((e) => console.error(e)); - for (const [key, name] of Object.entries(selected || {})) { + for (const [key, name] of Object.entries(interfaces)) { let found = this.search.options?.find((o) => o.value === name); // If not found, spoof based on the key and names provided previously From 3b06fd9c78ee52c2a2bc529e04a01abe6dfcc907 Mon Sep 17 00:00:00 2001 From: Garrett Date: Thu, 7 Sep 2023 09:32:05 -0700 Subject: [PATCH 17/62] Remove all internal files / folders related to a conversion --- src/renderer/src/dependencies/simple.js | 3 +++ src/renderer/src/progress/index.js | 20 ++++++++++++++----- .../stories/pages/guided-mode/ProgressCard.js | 2 +- 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/src/renderer/src/dependencies/simple.js b/src/renderer/src/dependencies/simple.js index 35c3adafd..95152a68f 100644 --- a/src/renderer/src/dependencies/simple.js +++ b/src/renderer/src/dependencies/simple.js @@ -12,4 +12,7 @@ export const homeDirectory = app?.getPath("home") ?? ""; export const appDirectory = homeDirectory ? joinPath(homeDirectory, paths.root) : ""; export const guidedProgressFilePath = homeDirectory ? joinPath(appDirectory, ...paths.subfolders.progress) : ""; +export const stubSaveFolderPath = homeDirectory ? joinPath(homeDirectory, paths["root"], ...paths.subfolders.stubs) : ""; +export const conversionSaveFolderPath = homeDirectory ? joinPath(homeDirectory, paths["root"], ...paths.subfolders.conversions) : ""; + export const isStorybook = window.location.href.includes("iframe.html"); diff --git a/src/renderer/src/progress/index.js b/src/renderer/src/progress/index.js index b5c7c2121..958c6da27 100644 --- a/src/renderer/src/progress/index.js +++ b/src/renderer/src/progress/index.js @@ -1,6 +1,6 @@ import Swal from "sweetalert2"; -import { guidedProgressFilePath, reloadPageToHome, isStorybook, appDirectory } from "../dependencies/simple.js"; +import { guidedProgressFilePath, reloadPageToHome, isStorybook, appDirectory, stubSaveFolderPath, conversionSaveFolderPath } from "../dependencies/simple.js"; import { fs } from "../electron/index.js"; import { joinPath, runOnLoad } from "../globals.js"; import { merge } from "../stories/pages/utils.js"; @@ -102,14 +102,14 @@ export function resume(name) { export const remove = async (name) => { const result = await Swal.fire({ - title: `Are you sure you would like to delete NWB GUIDE progress made on the dataset: ${name}?`, - text: "Your progress file will be deleted permanently, and all existing progress will be lost.", + title: `Are you sure you would like to delete this conversion pipeline?`, + html: `All related files will be deleted permanently, and existing progress will be lost.`, icon: "warning", heightAuto: false, showCancelButton: true, confirmButtonColor: "#3085d6", cancelButtonColor: "#d33", - confirmButtonText: "Delete progress file", + confirmButtonText: `Delete ${name}`, cancelButtonText: "Cancel", focusCancel: true, }); @@ -119,9 +119,19 @@ export const remove = async (name) => { const progressFilePathToDelete = joinPath(guidedProgressFilePath, name + ".json"); //delete the progress file - if (fs) fs.unlinkSync(progressFilePathToDelete, (err) => console.log(err)); + if (fs) fs.unlinkSync(progressFilePathToDelete); else localStorage.removeItem(progressFilePathToDelete); + if (fs) { + + // delete default stub location + fs.rmSync(joinPath(stubSaveFolderPath, name), { recursive: true, force: true }); + + // delete default conversion location + fs.rmSync(joinPath(conversionSaveFolderPath, name), { recursive: true, force: true }); + + } + return true; } diff --git a/src/renderer/src/stories/pages/guided-mode/ProgressCard.js b/src/renderer/src/stories/pages/guided-mode/ProgressCard.js index 062ae8fe3..d66b0e8de 100644 --- a/src/renderer/src/stories/pages/guided-mode/ProgressCard.js +++ b/src/renderer/src/stories/pages/guided-mode/ProgressCard.js @@ -126,7 +126,7 @@ export class ProgressCard extends LitElement { @click=${(ev) => progress.deleteProgressCard(ev.target)} > - Delete progress file + Delete pipeline From 888a9df455855d716f60a53d97261a46af121a06 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 7 Sep 2023 16:36:42 +0000 Subject: [PATCH 18/62] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/renderer/src/dependencies/simple.js | 8 ++++++-- src/renderer/src/progress/index.js | 11 ++++++++--- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/src/renderer/src/dependencies/simple.js b/src/renderer/src/dependencies/simple.js index 95152a68f..97f8d2148 100644 --- a/src/renderer/src/dependencies/simple.js +++ b/src/renderer/src/dependencies/simple.js @@ -12,7 +12,11 @@ export const homeDirectory = app?.getPath("home") ?? ""; export const appDirectory = homeDirectory ? joinPath(homeDirectory, paths.root) : ""; export const guidedProgressFilePath = homeDirectory ? joinPath(appDirectory, ...paths.subfolders.progress) : ""; -export const stubSaveFolderPath = homeDirectory ? joinPath(homeDirectory, paths["root"], ...paths.subfolders.stubs) : ""; -export const conversionSaveFolderPath = homeDirectory ? joinPath(homeDirectory, paths["root"], ...paths.subfolders.conversions) : ""; +export const stubSaveFolderPath = homeDirectory + ? joinPath(homeDirectory, paths["root"], ...paths.subfolders.stubs) + : ""; +export const conversionSaveFolderPath = homeDirectory + ? joinPath(homeDirectory, paths["root"], ...paths.subfolders.conversions) + : ""; export const isStorybook = window.location.href.includes("iframe.html"); diff --git a/src/renderer/src/progress/index.js b/src/renderer/src/progress/index.js index 958c6da27..b3dfb9ea8 100644 --- a/src/renderer/src/progress/index.js +++ b/src/renderer/src/progress/index.js @@ -1,6 +1,13 @@ import Swal from "sweetalert2"; -import { guidedProgressFilePath, reloadPageToHome, isStorybook, appDirectory, stubSaveFolderPath, conversionSaveFolderPath } from "../dependencies/simple.js"; +import { + guidedProgressFilePath, + reloadPageToHome, + isStorybook, + appDirectory, + stubSaveFolderPath, + conversionSaveFolderPath, +} from "../dependencies/simple.js"; import { fs } from "../electron/index.js"; import { joinPath, runOnLoad } from "../globals.js"; import { merge } from "../stories/pages/utils.js"; @@ -123,13 +130,11 @@ export const remove = async (name) => { else localStorage.removeItem(progressFilePathToDelete); if (fs) { - // delete default stub location fs.rmSync(joinPath(stubSaveFolderPath, name), { recursive: true, force: true }); // delete default conversion location fs.rmSync(joinPath(conversionSaveFolderPath, name), { recursive: true, force: true }); - } return true; From 25ac15ea3fdccacd0e3881eeb3e2d8bf53595add Mon Sep 17 00:00:00 2001 From: Garrett Date: Thu, 7 Sep 2023 10:05:35 -0700 Subject: [PATCH 19/62] Update package.json --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index e61702881..faf9ccdf7 100644 --- a/package.json +++ b/package.json @@ -24,7 +24,7 @@ "test": "npm run test:app && npm run test:server", "test:app": "vitest run", "test:server": "pytest pyflask/tests/ -s", - "test:executable": "concurrently -n EXE,TEST --kill-others \"node tests/testPyinstallerExecutable.js --port 3434 --forever\" \"pytest pyflask/tests/ -s --target http://localhost:3434\"", + "test:executable": "concurrently -n EXE,TEST --kill-others-on-fail --success last \"node tests/testPyinstallerExecutable.js --port 3434 --forever\" \"pytest pyflask/tests/ -s --target http://localhost:3434\"", "test:coverage": "npm run coverage:app && npm run coverage:server", "coverage:app": "vitest run --coverage", "coverage:server": "pytest pyflask/tests/ -s --cov=pyflask --cov-report=xml", From e8addf014413944bd794d9cc29d1ad376602768e Mon Sep 17 00:00:00 2001 From: Garrett Date: Thu, 7 Sep 2023 10:20:19 -0700 Subject: [PATCH 20/62] Update package.json --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index faf9ccdf7..5eb1fa7f2 100644 --- a/package.json +++ b/package.json @@ -24,7 +24,7 @@ "test": "npm run test:app && npm run test:server", "test:app": "vitest run", "test:server": "pytest pyflask/tests/ -s", - "test:executable": "concurrently -n EXE,TEST --kill-others-on-fail --success last \"node tests/testPyinstallerExecutable.js --port 3434 --forever\" \"pytest pyflask/tests/ -s --target http://localhost:3434\"", + "test:executable": "concurrently -n EXE,TEST --kill-others --success last \"node tests/testPyinstallerExecutable.js --port 3434 --forever\" \"pytest pyflask/tests/ -s --target http://localhost:3434\"", "test:coverage": "npm run coverage:app && npm run coverage:server", "coverage:app": "vitest run --coverage", "coverage:server": "pytest pyflask/tests/ -s --cov=pyflask --cov-report=xml", From 18280d486bf13da7663781af64829ccd6e5a7a64 Mon Sep 17 00:00:00 2001 From: Garrett Date: Thu, 7 Sep 2023 11:10:24 -0700 Subject: [PATCH 21/62] Update package.json --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 5eb1fa7f2..e69551d9b 100644 --- a/package.json +++ b/package.json @@ -24,7 +24,7 @@ "test": "npm run test:app && npm run test:server", "test:app": "vitest run", "test:server": "pytest pyflask/tests/ -s", - "test:executable": "concurrently -n EXE,TEST --kill-others --success last \"node tests/testPyinstallerExecutable.js --port 3434 --forever\" \"pytest pyflask/tests/ -s --target http://localhost:3434\"", + "test:executable": "concurrently -n EXE,TEST --kill-others --success all \"node tests/testPyinstallerExecutable.js --port 3434 --forever\" \"pytest pyflask/tests/ -s --target http://localhost:3434\"", "test:coverage": "npm run coverage:app && npm run coverage:server", "coverage:app": "vitest run --coverage", "coverage:server": "pytest pyflask/tests/ -s --cov=pyflask --cov-report=xml", From 40c3c17b0ce7dafdcc4ada4387af2b6c5892e8c8 Mon Sep 17 00:00:00 2001 From: Garrett Date: Thu, 7 Sep 2023 11:30:59 -0700 Subject: [PATCH 22/62] kill gracefully --- package.json | 2 +- tests/testPyinstallerExecutable.js | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index e69551d9b..e61702881 100644 --- a/package.json +++ b/package.json @@ -24,7 +24,7 @@ "test": "npm run test:app && npm run test:server", "test:app": "vitest run", "test:server": "pytest pyflask/tests/ -s", - "test:executable": "concurrently -n EXE,TEST --kill-others --success all \"node tests/testPyinstallerExecutable.js --port 3434 --forever\" \"pytest pyflask/tests/ -s --target http://localhost:3434\"", + "test:executable": "concurrently -n EXE,TEST --kill-others \"node tests/testPyinstallerExecutable.js --port 3434 --forever\" \"pytest pyflask/tests/ -s --target http://localhost:3434\"", "test:coverage": "npm run coverage:app && npm run coverage:server", "coverage:app": "vitest run --coverage", "coverage:server": "pytest pyflask/tests/ -s --cov=pyflask --cov-report=xml", diff --git a/tests/testPyinstallerExecutable.js b/tests/testPyinstallerExecutable.js index 4611dba7a..fffab8b7a 100644 --- a/tests/testPyinstallerExecutable.js +++ b/tests/testPyinstallerExecutable.js @@ -52,5 +52,11 @@ function handleProcess(proc, id = "process") { console.error(`[${id}] Exit: ${code}`); error(); }); + + process.once('SIGTERM', () => { + proc.kill(0) + process.exit(0) + }) // Exit gracefully if this is caught + } else console.error("child process failed to start on port" + port); } From fa3f445c100c7d3887873f74defd272ed407dcc9 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 7 Sep 2023 18:32:21 +0000 Subject: [PATCH 23/62] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/testPyinstallerExecutable.js | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tests/testPyinstallerExecutable.js b/tests/testPyinstallerExecutable.js index fffab8b7a..a8dfb910e 100644 --- a/tests/testPyinstallerExecutable.js +++ b/tests/testPyinstallerExecutable.js @@ -53,10 +53,9 @@ function handleProcess(proc, id = "process") { error(); }); - process.once('SIGTERM', () => { - proc.kill(0) - process.exit(0) - }) // Exit gracefully if this is caught - + process.once("SIGTERM", () => { + proc.kill(0); + process.exit(0); + }); // Exit gracefully if this is caught } else console.error("child process failed to start on port" + port); } From 6db9bcb37a6fcb84ad9eb06caefdbc8ab9c2fdb1 Mon Sep 17 00:00:00 2001 From: Garrett Date: Thu, 7 Sep 2023 12:38:45 -0700 Subject: [PATCH 24/62] Update testPyinstallerExecutable.js --- tests/testPyinstallerExecutable.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/testPyinstallerExecutable.js b/tests/testPyinstallerExecutable.js index a8dfb910e..d8ff46469 100644 --- a/tests/testPyinstallerExecutable.js +++ b/tests/testPyinstallerExecutable.js @@ -54,8 +54,10 @@ function handleProcess(proc, id = "process") { }); process.once("SIGTERM", () => { + console.log('GOT SIGTERM!!!!') proc.kill(0); process.exit(0); }); // Exit gracefully if this is caught + } else console.error("child process failed to start on port" + port); } From b77f2c6091e58254dd5131714cf144d129bea0fd Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 7 Sep 2023 19:39:01 +0000 Subject: [PATCH 25/62] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/testPyinstallerExecutable.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/testPyinstallerExecutable.js b/tests/testPyinstallerExecutable.js index d8ff46469..7a59e3582 100644 --- a/tests/testPyinstallerExecutable.js +++ b/tests/testPyinstallerExecutable.js @@ -54,10 +54,9 @@ function handleProcess(proc, id = "process") { }); process.once("SIGTERM", () => { - console.log('GOT SIGTERM!!!!') + console.log("GOT SIGTERM!!!!"); proc.kill(0); process.exit(0); }); // Exit gracefully if this is caught - } else console.error("child process failed to start on port" + port); } From ad7c74e6692903296cf10a41115e85c35c7d5561 Mon Sep 17 00:00:00 2001 From: Garrett Date: Thu, 7 Sep 2023 13:00:10 -0700 Subject: [PATCH 26/62] Update testPyinstallerExecutable.js --- tests/testPyinstallerExecutable.js | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/testPyinstallerExecutable.js b/tests/testPyinstallerExecutable.js index d8ff46469..c8fd00864 100644 --- a/tests/testPyinstallerExecutable.js +++ b/tests/testPyinstallerExecutable.js @@ -26,6 +26,13 @@ const regex = /.+Error: .+/; function handleProcess(proc, id = "process") { if (proc != null) { + + process.on("SIGTERM", () => { + console.log('GOT SIGTERM!!!!') + proc.kill(0); + process.exit(0); + }); // Exit gracefully if this is caught + // Listen for errors from Python process proc.stderr.on("data", function (data) { const message = data.toString(); @@ -53,11 +60,5 @@ function handleProcess(proc, id = "process") { error(); }); - process.once("SIGTERM", () => { - console.log('GOT SIGTERM!!!!') - proc.kill(0); - process.exit(0); - }); // Exit gracefully if this is caught - } else console.error("child process failed to start on port" + port); } From 496fdee5ac3807a845da846768d4390fd30b28fa Mon Sep 17 00:00:00 2001 From: Garrett Michael Flynn <46533749+GarrettMFlynn@users.noreply.github.com> Date: Thu, 7 Sep 2023 14:53:48 -0700 Subject: [PATCH 27/62] Wait to ensure server has started --- package.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index e61702881..de1b1d6b2 100644 --- a/package.json +++ b/package.json @@ -24,7 +24,8 @@ "test": "npm run test:app && npm run test:server", "test:app": "vitest run", "test:server": "pytest pyflask/tests/ -s", - "test:executable": "concurrently -n EXE,TEST --kill-others \"node tests/testPyinstallerExecutable.js --port 3434 --forever\" \"pytest pyflask/tests/ -s --target http://localhost:3434\"", + "wait3s": "node -e \"setTimeout(() => process.exit(0),3000)\"", + "test:executable": "concurrently -n EXE,TEST --kill-others \"node tests/testPyinstallerExecutable.js --port 3434 --forever\" \"npm run wait3s && pytest pyflask/tests/ -s --target http://localhost:3434\"", "test:coverage": "npm run coverage:app && npm run coverage:server", "coverage:app": "vitest run --coverage", "coverage:server": "pytest pyflask/tests/ -s --cov=pyflask --cov-report=xml", From 412c1a1ee66aab807ff1c46a4249d475f0fcb50e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 7 Sep 2023 21:56:15 +0000 Subject: [PATCH 28/62] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/testPyinstallerExecutable.js | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/testPyinstallerExecutable.js b/tests/testPyinstallerExecutable.js index c8fd00864..1135eaf99 100644 --- a/tests/testPyinstallerExecutable.js +++ b/tests/testPyinstallerExecutable.js @@ -26,13 +26,12 @@ const regex = /.+Error: .+/; function handleProcess(proc, id = "process") { if (proc != null) { - process.on("SIGTERM", () => { - console.log('GOT SIGTERM!!!!') + console.log("GOT SIGTERM!!!!"); proc.kill(0); process.exit(0); }); // Exit gracefully if this is caught - + // Listen for errors from Python process proc.stderr.on("data", function (data) { const message = data.toString(); @@ -59,6 +58,5 @@ function handleProcess(proc, id = "process") { console.error(`[${id}] Exit: ${code}`); error(); }); - } else console.error("child process failed to start on port" + port); } From 603fba6b64eb95c1b405fefaa8fce530f0b22ec8 Mon Sep 17 00:00:00 2001 From: Garrett Date: Thu, 7 Sep 2023 15:29:13 -0700 Subject: [PATCH 29/62] Test locally until it works --- package.json | 2 +- tests/testPyinstallerExecutable.js | 6 ------ 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/package.json b/package.json index de1b1d6b2..6d545d081 100644 --- a/package.json +++ b/package.json @@ -25,7 +25,7 @@ "test:app": "vitest run", "test:server": "pytest pyflask/tests/ -s", "wait3s": "node -e \"setTimeout(() => process.exit(0),3000)\"", - "test:executable": "concurrently -n EXE,TEST --kill-others \"node tests/testPyinstallerExecutable.js --port 3434 --forever\" \"npm run wait3s && pytest pyflask/tests/ -s --target http://localhost:3434\"", + "test:executable": "concurrently -n EXE,TEST --kill-others --success first \"node tests/testPyinstallerExecutable.js --port 3434 --forever\" \"npm run wait3s && pytest pyflask/tests/ -s --target http://localhost:3434\"", "test:coverage": "npm run coverage:app && npm run coverage:server", "coverage:app": "vitest run --coverage", "coverage:server": "pytest pyflask/tests/ -s --cov=pyflask --cov-report=xml", diff --git a/tests/testPyinstallerExecutable.js b/tests/testPyinstallerExecutable.js index c8fd00864..cf1deec14 100644 --- a/tests/testPyinstallerExecutable.js +++ b/tests/testPyinstallerExecutable.js @@ -27,12 +27,6 @@ const regex = /.+Error: .+/; function handleProcess(proc, id = "process") { if (proc != null) { - process.on("SIGTERM", () => { - console.log('GOT SIGTERM!!!!') - proc.kill(0); - process.exit(0); - }); // Exit gracefully if this is caught - // Listen for errors from Python process proc.stderr.on("data", function (data) { const message = data.toString(); From e6dc9684ea036199eea425012289682ce8be8837 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 7 Sep 2023 22:29:48 +0000 Subject: [PATCH 30/62] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/testPyinstallerExecutable.js | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/testPyinstallerExecutable.js b/tests/testPyinstallerExecutable.js index 2b44cb542..4611dba7a 100644 --- a/tests/testPyinstallerExecutable.js +++ b/tests/testPyinstallerExecutable.js @@ -26,7 +26,6 @@ const regex = /.+Error: .+/; function handleProcess(proc, id = "process") { if (proc != null) { - // Listen for errors from Python process proc.stderr.on("data", function (data) { const message = data.toString(); From 0a9a521299e988d3e3b8be3693e0ec949723405b Mon Sep 17 00:00:00 2001 From: rly Date: Fri, 8 Sep 2023 11:02:01 +0200 Subject: [PATCH 31/62] Try adjustment to mac env --- environments/environment-MAC.yml | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/environments/environment-MAC.yml b/environments/environment-MAC.yml index cd76d4596..b2333fa86 100644 --- a/environments/environment-MAC.yml +++ b/environments/environment-MAC.yml @@ -1,11 +1,11 @@ name: nwb-guide channels: - - defaults - conda-forge + - defaults dependencies: - - python = 3.9.17 - - nodejs = 18.16.1 + - python = 3.9.18 - PyInstaller = 5.13.0 + - nodejs = 18.16.1 - numcodecs = 0.11.0 - jsonschema = 4.18.0 # this installs jsonschema-specifications which is needed for validation - pydantic[email] = 2.0.2 # email validator is used by dandi @@ -14,9 +14,7 @@ dependencies: - chardet == 5.1.0 - configparser == 6.0.0 - flask == 2.3.2 - - flask-cors === 3.0.10 + - flask-cors == 4.0.0 - flask_restx == 1.1.0 - neuroconv @ git+https://github.com/catalystneuro/neuroconv.git@main#neuroconv[full] - - hdmf >= 3.7.0 - - pytest == 7.2.2 - - pytest-cov == 4.1.0 + - pytest == 7.4.0 From aa88b9f97258ba6d72332c360fbd8d84b4019c50 Mon Sep 17 00:00:00 2001 From: Ryan Ly Date: Fri, 8 Sep 2023 06:37:43 -0700 Subject: [PATCH 32/62] Discard changes to environments/environment-MAC.yml --- environments/environment-MAC.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/environments/environment-MAC.yml b/environments/environment-MAC.yml index b2333fa86..cd76d4596 100644 --- a/environments/environment-MAC.yml +++ b/environments/environment-MAC.yml @@ -1,11 +1,11 @@ name: nwb-guide channels: - - conda-forge - defaults + - conda-forge dependencies: - - python = 3.9.18 - - PyInstaller = 5.13.0 + - python = 3.9.17 - nodejs = 18.16.1 + - PyInstaller = 5.13.0 - numcodecs = 0.11.0 - jsonschema = 4.18.0 # this installs jsonschema-specifications which is needed for validation - pydantic[email] = 2.0.2 # email validator is used by dandi @@ -14,7 +14,9 @@ dependencies: - chardet == 5.1.0 - configparser == 6.0.0 - flask == 2.3.2 - - flask-cors == 4.0.0 + - flask-cors === 3.0.10 - flask_restx == 1.1.0 - neuroconv @ git+https://github.com/catalystneuro/neuroconv.git@main#neuroconv[full] - - pytest == 7.4.0 + - hdmf >= 3.7.0 + - pytest == 7.2.2 + - pytest-cov == 4.1.0 From 8f9f9f7e4a9fd5c43a97c0b3058d8ad87fa8c7b9 Mon Sep 17 00:00:00 2001 From: rly Date: Sat, 9 Sep 2023 12:19:09 +0200 Subject: [PATCH 33/62] Testing --- environments/environment-MAC.yml | 11 +++++------ package.json | 2 +- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/environments/environment-MAC.yml b/environments/environment-MAC.yml index cd76d4596..58c1a6855 100644 --- a/environments/environment-MAC.yml +++ b/environments/environment-MAC.yml @@ -1,11 +1,11 @@ name: nwb-guide channels: - - defaults - conda-forge + - defaults dependencies: - - python = 3.9.17 - - nodejs = 18.16.1 + - python = 3.9.18 - PyInstaller = 5.13.0 + - nodejs = 18.16.1 - numcodecs = 0.11.0 - jsonschema = 4.18.0 # this installs jsonschema-specifications which is needed for validation - pydantic[email] = 2.0.2 # email validator is used by dandi @@ -14,9 +14,8 @@ dependencies: - chardet == 5.1.0 - configparser == 6.0.0 - flask == 2.3.2 - - flask-cors === 3.0.10 + - flask-cors == 4.0.0 - flask_restx == 1.1.0 - neuroconv @ git+https://github.com/catalystneuro/neuroconv.git@main#neuroconv[full] - - hdmf >= 3.7.0 - - pytest == 7.2.2 + - pytest == 7.4.0 - pytest-cov == 4.1.0 diff --git a/package.json b/package.json index bad2cfe3e..90c229d6e 100644 --- a/package.json +++ b/package.json @@ -15,7 +15,7 @@ "build:win": "npm run build && npm run build:flask:win && npm run build:electron:win", "build:mac": "npm run build && npm run build:flask:unix && npm run build:electron:mac", "build:linux": "npm run build && npm run build:flask:unix && npm run build:electron:linux", - "build:flask:base": "python -m PyInstaller --name nwb-guide --onedir --clean --noconfirm ./pyflask/app.py --distpath ./build/flask --collect-data jsonschema_specifications --collect-all nwbinspector --collect-all neuroconv --collect-all pynwb --collect-all hdmf --collect-all ci_info --hidden-import scipy._distributor_init --hidden-import scipy._lib.messagestream --hidden-import scipy._lib._ccallback --hidden-import scipy._lib._testutils", + "build:flask:base": "python -m PyInstaller --name nwb-guide --onedir --clean --noconfirm ./pyflask/app.py --distpath ./build/flask --collect-data jsonschema_specifications --collect-all nwbinspector --collect-all neuroconv --collect-all pynwb --collect-all hdmf --collect-all ci_info --collect-all sonpy --hidden-import scipy._distributor_init --hidden-import scipy._lib.messagestream --hidden-import scipy._lib._ccallback --hidden-import scipy._lib._testutils", "build:flask:win": "npm run build:flask:base -- --add-data ./paths.config.json;. --add-data ./package.json;.", "build:flask:unix": "npm run build:flask:base -- --add-data ./paths.config.json:. --add-data ./package.json:. --collect-all ndx_dandi_icephys", "build:electron:win": "electron-builder build --win --publish never", From f49184dbf9584f2d98052df6adea4e41dd183805 Mon Sep 17 00:00:00 2001 From: rly Date: Sat, 9 Sep 2023 18:31:38 +0200 Subject: [PATCH 34/62] Set pyinstaller log level to debug --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 90c229d6e..3838c86fb 100644 --- a/package.json +++ b/package.json @@ -15,7 +15,7 @@ "build:win": "npm run build && npm run build:flask:win && npm run build:electron:win", "build:mac": "npm run build && npm run build:flask:unix && npm run build:electron:mac", "build:linux": "npm run build && npm run build:flask:unix && npm run build:electron:linux", - "build:flask:base": "python -m PyInstaller --name nwb-guide --onedir --clean --noconfirm ./pyflask/app.py --distpath ./build/flask --collect-data jsonschema_specifications --collect-all nwbinspector --collect-all neuroconv --collect-all pynwb --collect-all hdmf --collect-all ci_info --collect-all sonpy --hidden-import scipy._distributor_init --hidden-import scipy._lib.messagestream --hidden-import scipy._lib._ccallback --hidden-import scipy._lib._testutils", + "build:flask:base": "python -m PyInstaller --log-level DEBUG --name nwb-guide --onedir --clean --noconfirm ./pyflask/app.py --distpath ./build/flask --collect-data jsonschema_specifications --collect-all nwbinspector --collect-all neuroconv --collect-all pynwb --collect-all hdmf --collect-all ci_info --collect-all sonpy --hidden-import scipy._distributor_init --hidden-import scipy._lib.messagestream --hidden-import scipy._lib._ccallback --hidden-import scipy._lib._testutils", "build:flask:win": "npm run build:flask:base -- --add-data ./paths.config.json;. --add-data ./package.json;.", "build:flask:unix": "npm run build:flask:base -- --add-data ./paths.config.json:. --add-data ./package.json:. --collect-all ndx_dandi_icephys", "build:electron:win": "electron-builder build --win --publish never", From 7005c4bf3215c5926e03c3d73715476aa96e2185 Mon Sep 17 00:00:00 2001 From: rly Date: Sat, 9 Sep 2023 18:55:07 +0200 Subject: [PATCH 35/62] Add sonpy.so explicitly --- package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 3838c86fb..1564ef8bb 100644 --- a/package.json +++ b/package.json @@ -15,9 +15,9 @@ "build:win": "npm run build && npm run build:flask:win && npm run build:electron:win", "build:mac": "npm run build && npm run build:flask:unix && npm run build:electron:mac", "build:linux": "npm run build && npm run build:flask:unix && npm run build:electron:linux", - "build:flask:base": "python -m PyInstaller --log-level DEBUG --name nwb-guide --onedir --clean --noconfirm ./pyflask/app.py --distpath ./build/flask --collect-data jsonschema_specifications --collect-all nwbinspector --collect-all neuroconv --collect-all pynwb --collect-all hdmf --collect-all ci_info --collect-all sonpy --hidden-import scipy._distributor_init --hidden-import scipy._lib.messagestream --hidden-import scipy._lib._ccallback --hidden-import scipy._lib._testutils", + "build:flask:base": "python -m PyInstaller --log-level DEBUG --name nwb-guide --onedir --clean --noconfirm ./pyflask/app.py --distpath ./build/flask --collect-data jsonschema_specifications --collect-all nwbinspector --collect-all neuroconv --collect-all pynwb --collect-all hdmf --collect-all ci_info --hidden-import scipy._distributor_init --hidden-import scipy._lib.messagestream --hidden-import scipy._lib._ccallback --hidden-import scipy._lib._testutils", "build:flask:win": "npm run build:flask:base -- --add-data ./paths.config.json;. --add-data ./package.json;.", - "build:flask:unix": "npm run build:flask:base -- --add-data ./paths.config.json:. --add-data ./package.json:. --collect-all ndx_dandi_icephys", + "build:flask:unix": "npm run build:flask:base -- --add-data ./paths.config.json:. --add-data ./package.json:. --add-data /Users/rly/mambaforge/envs/nwb-guide/lib/python3.9/site-packages/sonpy/linux/sonpy.so:. --collect-all ndx_dandi_icephys", "build:electron:win": "electron-builder build --win --publish never", "build:electron:mac": "electron-builder build --mac --publish never", "build:electron:linux": "electron-builder build --linux --publish never", From 5ead0115b34ff768e35ab3ab42440243d716b634 Mon Sep 17 00:00:00 2001 From: rly Date: Sat, 9 Sep 2023 19:52:56 +0200 Subject: [PATCH 36/62] Testing --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 1564ef8bb..da33c69cb 100644 --- a/package.json +++ b/package.json @@ -17,7 +17,7 @@ "build:linux": "npm run build && npm run build:flask:unix && npm run build:electron:linux", "build:flask:base": "python -m PyInstaller --log-level DEBUG --name nwb-guide --onedir --clean --noconfirm ./pyflask/app.py --distpath ./build/flask --collect-data jsonschema_specifications --collect-all nwbinspector --collect-all neuroconv --collect-all pynwb --collect-all hdmf --collect-all ci_info --hidden-import scipy._distributor_init --hidden-import scipy._lib.messagestream --hidden-import scipy._lib._ccallback --hidden-import scipy._lib._testutils", "build:flask:win": "npm run build:flask:base -- --add-data ./paths.config.json;. --add-data ./package.json;.", - "build:flask:unix": "npm run build:flask:base -- --add-data ./paths.config.json:. --add-data ./package.json:. --add-data /Users/rly/mambaforge/envs/nwb-guide/lib/python3.9/site-packages/sonpy/linux/sonpy.so:. --collect-all ndx_dandi_icephys", + "build:flask:unix": "npm run build:flask:base -- --add-data ./paths.config.json:. --add-data ./package.json:. --add-data /Users/runner/miniconda3/envs/nwb-guide/lib/python3.9/site-packages/sonpy/linux/sonpy.so:. --collect-all ndx_dandi_icephys", "build:electron:win": "electron-builder build --win --publish never", "build:electron:mac": "electron-builder build --mac --publish never", "build:electron:linux": "electron-builder build --linux --publish never", From 0d346cfe8302177645df1200700fdc9d112b3704 Mon Sep 17 00:00:00 2001 From: rly Date: Sat, 9 Sep 2023 20:14:55 +0200 Subject: [PATCH 37/62] Testing --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index da33c69cb..5a54117b3 100644 --- a/package.json +++ b/package.json @@ -17,7 +17,7 @@ "build:linux": "npm run build && npm run build:flask:unix && npm run build:electron:linux", "build:flask:base": "python -m PyInstaller --log-level DEBUG --name nwb-guide --onedir --clean --noconfirm ./pyflask/app.py --distpath ./build/flask --collect-data jsonschema_specifications --collect-all nwbinspector --collect-all neuroconv --collect-all pynwb --collect-all hdmf --collect-all ci_info --hidden-import scipy._distributor_init --hidden-import scipy._lib.messagestream --hidden-import scipy._lib._ccallback --hidden-import scipy._lib._testutils", "build:flask:win": "npm run build:flask:base -- --add-data ./paths.config.json;. --add-data ./package.json;.", - "build:flask:unix": "npm run build:flask:base -- --add-data ./paths.config.json:. --add-data ./package.json:. --add-data /Users/runner/miniconda3/envs/nwb-guide/lib/python3.9/site-packages/sonpy/linux/sonpy.so:. --collect-all ndx_dandi_icephys", + "build:flask:unix": "npm run build:flask:base -- --add-data ./paths.config.json:. --add-data ./package.json:. --add-binary /Users/runner/miniconda3/envs/nwb-guide/lib/python3.9/site-packages/sonpy/linux/sonpy.so:. --collect-all ndx_dandi_icephys", "build:electron:win": "electron-builder build --win --publish never", "build:electron:mac": "electron-builder build --mac --publish never", "build:electron:linux": "electron-builder build --linux --publish never", From 53e62b26330af3e5a3fa5eefae1552b7e16a9717 Mon Sep 17 00:00:00 2001 From: rly Date: Sat, 9 Sep 2023 20:49:49 +0200 Subject: [PATCH 38/62] Testing --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 5a54117b3..c55b82cc3 100644 --- a/package.json +++ b/package.json @@ -17,7 +17,7 @@ "build:linux": "npm run build && npm run build:flask:unix && npm run build:electron:linux", "build:flask:base": "python -m PyInstaller --log-level DEBUG --name nwb-guide --onedir --clean --noconfirm ./pyflask/app.py --distpath ./build/flask --collect-data jsonschema_specifications --collect-all nwbinspector --collect-all neuroconv --collect-all pynwb --collect-all hdmf --collect-all ci_info --hidden-import scipy._distributor_init --hidden-import scipy._lib.messagestream --hidden-import scipy._lib._ccallback --hidden-import scipy._lib._testutils", "build:flask:win": "npm run build:flask:base -- --add-data ./paths.config.json;. --add-data ./package.json;.", - "build:flask:unix": "npm run build:flask:base -- --add-data ./paths.config.json:. --add-data ./package.json:. --add-binary /Users/runner/miniconda3/envs/nwb-guide/lib/python3.9/site-packages/sonpy/linux/sonpy.so:. --collect-all ndx_dandi_icephys", + "build:flask:unix": "rm /Users/runner/miniconda3/envs/nwb-guide/lib/python3.9/site-packages/sonpy/linux/sonpy.so && npm run build:flask:base -- --add-data ./paths.config.json:. --add-data ./package.json:. --add-binary /Users/runner/miniconda3/envs/nwb-guide/lib/python3.9/site-packages/sonpy/linux/sonpy.so:. --collect-all ndx_dandi_icephys", "build:electron:win": "electron-builder build --win --publish never", "build:electron:mac": "electron-builder build --mac --publish never", "build:electron:linux": "electron-builder build --linux --publish never", From e55b823a40c6902ba9f199e5cdf87087f327182d Mon Sep 17 00:00:00 2001 From: rly Date: Sat, 9 Sep 2023 20:50:13 +0200 Subject: [PATCH 39/62] Testing --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index c55b82cc3..6edc0071a 100644 --- a/package.json +++ b/package.json @@ -17,7 +17,7 @@ "build:linux": "npm run build && npm run build:flask:unix && npm run build:electron:linux", "build:flask:base": "python -m PyInstaller --log-level DEBUG --name nwb-guide --onedir --clean --noconfirm ./pyflask/app.py --distpath ./build/flask --collect-data jsonschema_specifications --collect-all nwbinspector --collect-all neuroconv --collect-all pynwb --collect-all hdmf --collect-all ci_info --hidden-import scipy._distributor_init --hidden-import scipy._lib.messagestream --hidden-import scipy._lib._ccallback --hidden-import scipy._lib._testutils", "build:flask:win": "npm run build:flask:base -- --add-data ./paths.config.json;. --add-data ./package.json;.", - "build:flask:unix": "rm /Users/runner/miniconda3/envs/nwb-guide/lib/python3.9/site-packages/sonpy/linux/sonpy.so && npm run build:flask:base -- --add-data ./paths.config.json:. --add-data ./package.json:. --add-binary /Users/runner/miniconda3/envs/nwb-guide/lib/python3.9/site-packages/sonpy/linux/sonpy.so:. --collect-all ndx_dandi_icephys", + "build:flask:unix": "rm /Users/runner/miniconda3/envs/nwb-guide/lib/python3.9/site-packages/sonpy/linux/sonpy.so && npm run build:flask:base -- --add-data ./paths.config.json:. --add-data ./package.json:. --collect-all ndx_dandi_icephys", "build:electron:win": "electron-builder build --win --publish never", "build:electron:mac": "electron-builder build --mac --publish never", "build:electron:linux": "electron-builder build --linux --publish never", From 499dc92f43c70551f205f9a0536ca9e850f7cff0 Mon Sep 17 00:00:00 2001 From: rly Date: Sat, 9 Sep 2023 21:16:26 +0200 Subject: [PATCH 40/62] Fix --- environments/environment-Windows.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/environments/environment-Windows.yml b/environments/environment-Windows.yml index 7faeba0ee..eb846dee2 100644 --- a/environments/environment-Windows.yml +++ b/environments/environment-Windows.yml @@ -20,3 +20,4 @@ dependencies: - hdmf >= 3.7.0 - pytest == 7.2.2 - pytest-cov == 4.1.0 + - email-validator == 2.0.0 From 074c2a6220855e99a08d42791796b62eae70a255 Mon Sep 17 00:00:00 2001 From: rly Date: Sat, 9 Sep 2023 21:28:55 +0200 Subject: [PATCH 41/62] Fix --- package.json | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/package.json b/package.json index 6edc0071a..08db3502b 100644 --- a/package.json +++ b/package.json @@ -13,18 +13,19 @@ "build": "npm run build:app", "build:app": "electron-vite build --outDir build", "build:win": "npm run build && npm run build:flask:win && npm run build:electron:win", - "build:mac": "npm run build && npm run build:flask:unix && npm run build:electron:mac", + "build:mac": "npm run build && rm /Users/runner/miniconda3/envs/nwb-guide/lib/python3.9/site-packages/sonpy/linux/sonpy.so && npm run build:flask:unix && npm run build:electron:mac", "build:linux": "npm run build && npm run build:flask:unix && npm run build:electron:linux", - "build:flask:base": "python -m PyInstaller --log-level DEBUG --name nwb-guide --onedir --clean --noconfirm ./pyflask/app.py --distpath ./build/flask --collect-data jsonschema_specifications --collect-all nwbinspector --collect-all neuroconv --collect-all pynwb --collect-all hdmf --collect-all ci_info --hidden-import scipy._distributor_init --hidden-import scipy._lib.messagestream --hidden-import scipy._lib._ccallback --hidden-import scipy._lib._testutils", + "build:flask:base": "python -m PyInstaller --log-level DEBUG --name nwb-guide --onedir --clean --noconfirm ./pyflask/app.py --distpath ./build/flask --collect-data jsonschema_specifications --collect-all nwbinspector --collect-all neuroconv --collect-all pynwb --collect-all hdmf --collect-all ci_info --collect-all ndx_dandi_icephys --hidden-import scipy._distributor_init --hidden-import scipy._lib.messagestream --hidden-import scipy._lib._ccallback --hidden-import scipy._lib._testutils", "build:flask:win": "npm run build:flask:base -- --add-data ./paths.config.json;. --add-data ./package.json;.", - "build:flask:unix": "rm /Users/runner/miniconda3/envs/nwb-guide/lib/python3.9/site-packages/sonpy/linux/sonpy.so && npm run build:flask:base -- --add-data ./paths.config.json:. --add-data ./package.json:. --collect-all ndx_dandi_icephys", + "build:flask:unix": "npm run build:flask:base -- --add-data ./paths.config.json:. --add-data ./package.json:.", "build:electron:win": "electron-builder build --win --publish never", "build:electron:mac": "electron-builder build --mac --publish never", "build:electron:linux": "electron-builder build --linux --publish never", "test": "npm run test:app && npm run test:server", "test:app": "vitest run", "test:server": "pytest pyflask/tests/ -s", - "test:executable": "concurrently -n EXE,TEST --kill-others \"node tests/testPyinstallerExecutable.js --port 3434 --forever\" \"pytest pyflask/tests/ -s --target http://localhost:3434\"", + "wait3s": "node -e \"setTimeout(() => process.exit(0),3000)\"", + "test:executable": "concurrently -n EXE,TEST --kill-others --success first \"node tests/testPyinstallerExecutable.js --port 3434 --forever\" \"npm run wait3s && pytest pyflask/tests/ -s --target http://localhost:3434\"", "test:coverage": "npm run coverage:app && npm run coverage:server", "coverage:app": "vitest run --coverage", "coverage:server": "pytest pyflask/tests/ -s --cov=pyflask --cov-report=xml", From e29e43e9c8203214d7a2dcf46e3433e4742ae610 Mon Sep 17 00:00:00 2001 From: rly Date: Sat, 9 Sep 2023 21:29:18 +0200 Subject: [PATCH 42/62] Update arm64 env --- environments/environment-MAC-arm64.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/environments/environment-MAC-arm64.yml b/environments/environment-MAC-arm64.yml index 4c0ca3ae3..20d6619d4 100644 --- a/environments/environment-MAC-arm64.yml +++ b/environments/environment-MAC-arm64.yml @@ -3,21 +3,22 @@ channels: - conda-forge - defaults dependencies: - - python = 3.9.17 + - python = 3.9.18 - PyInstaller = 5.13.0 - nodejs = 18.16.1 - numcodecs = 0.11.0 - - lxml = 4.9.3 # pypi build fails due to x64/arm64 mismatch so install from conda-forge - - pyedflib = 0.1.32 # pypi build fails due to x64/arm64 mismatch so install from conda-forge - jsonschema = 4.18.0 # this installs jsonschema-specifications which is needed for validation - pydantic[email] = 2.0.2 # email validator is used by dandi + - lxml = 4.9.3 # pypi build fails due to x64/arm64 mismatch so install from conda-forge + - pyedflib = 0.1.32 # pypi build fails due to x64/arm64 mismatch so install from conda-forge - numpy # may have x64/arm64 mismatch issues so install from conda-forge - pip - pip: - chardet == 5.1.0 - configparser == 6.0.0 - flask == 2.3.2 - - flask_restx == 1.1.0 - flask-cors == 4.0.0 + - flask_restx == 1.1.0 - neuroconv @ git+https://github.com/catalystneuro/neuroconv.git@main#neuroconv[ecephys] - pytest == 7.4.0 + - pytest-cov == 4.1.0 From d1be94ad225f4e16164021bbc5e85715fc08f320 Mon Sep 17 00:00:00 2001 From: rly Date: Sat, 9 Sep 2023 22:02:04 +0200 Subject: [PATCH 43/62] Adjust all envs --- .../pyflask-build-and-dist-tests.yml | 4 ++ environments/environment-Linux.yml | 37 +++++++------------ environments/environment-MAC-arm64.yml | 5 ++- environments/environment-MAC.yml | 5 ++- environments/environment-Windows.yml | 21 ++++++----- package.json | 2 +- 6 files changed, 36 insertions(+), 38 deletions(-) diff --git a/.github/workflows/pyflask-build-and-dist-tests.yml b/.github/workflows/pyflask-build-and-dist-tests.yml index e2e983010..8bf667130 100644 --- a/.github/workflows/pyflask-build-and-dist-tests.yml +++ b/.github/workflows/pyflask-build-and-dist-tests.yml @@ -83,6 +83,10 @@ jobs: - run: npm ci + # fix for macos build + - if: matrix.os == 'macos-latest' + run: rm /Users/runner/miniconda3/envs/nwb-guide/lib/python3.9/site-packages/sonpy/linux/sonpy.so + - name: Build PyFlask distribution run: npm run build:flask:${{ matrix.shorthand }} diff --git a/environments/environment-Linux.yml b/environments/environment-Linux.yml index 87a4fe4a1..d55ea40bf 100644 --- a/environments/environment-Linux.yml +++ b/environments/environment-Linux.yml @@ -1,31 +1,22 @@ name: nwb-guide channels: - - defaults - conda-forge + - defaults dependencies: - - python = 3.9.17 - - PyInstaller = 4.7 - - nodejs = 16.14.2 - - libgcc = 7.2.0 - - git = 2.20.1 + - python = 3.9.18 + - PyInstaller = 5.13.0 + - nodejs = 18.16.1 + - numcodecs = 0.11.0 + # install these from conda-forge so that dependent packages get included in the distributable + - jsonschema = 4.18.0 # installs jsonschema-specifications + - pydantic[email] = 2.0.2 # installs email-validator - pip - pip: - - chardet == 4.0.0 - - pandas == 1.4.2 - - openpyxl == 3.0.3 - - pennsieve == 6.1.2 - - configparser == 4.0.2 - - python-docx == 0.8.10 - - xlrd == 1.2.0 - - biopython == 1.79 - - flask == 2.0.2 - - flask_restx - - protobuf == 3.20.0 - - gevent == 21.12.0 - - werkzeug == 2.0.2 - - opencv-python == 4.5.3.56 + - chardet == 5.1.0 + - configparser == 6.0.0 + - flask == 2.3.2 + - flask-cors == 4.0.0 + - flask_restx == 1.1.0 - neuroconv @ git+https://github.com/catalystneuro/neuroconv.git@main#neuroconv[full] - - hdmf >= 3.7.0 - - pytest == 7.2.2 + - pytest == 7.4.0 - pytest-cov == 4.1.0 - - flask-cors === 3.0.10 diff --git a/environments/environment-MAC-arm64.yml b/environments/environment-MAC-arm64.yml index 20d6619d4..bb11633f9 100644 --- a/environments/environment-MAC-arm64.yml +++ b/environments/environment-MAC-arm64.yml @@ -7,11 +7,12 @@ dependencies: - PyInstaller = 5.13.0 - nodejs = 18.16.1 - numcodecs = 0.11.0 - - jsonschema = 4.18.0 # this installs jsonschema-specifications which is needed for validation - - pydantic[email] = 2.0.2 # email validator is used by dandi - lxml = 4.9.3 # pypi build fails due to x64/arm64 mismatch so install from conda-forge - pyedflib = 0.1.32 # pypi build fails due to x64/arm64 mismatch so install from conda-forge - numpy # may have x64/arm64 mismatch issues so install from conda-forge + # install these from conda-forge so that dependent packages get included in the distributable + - jsonschema = 4.18.0 # installs jsonschema-specifications + - pydantic[email] = 2.0.2 # installs email-validator - pip - pip: - chardet == 5.1.0 diff --git a/environments/environment-MAC.yml b/environments/environment-MAC.yml index 58c1a6855..d55ea40bf 100644 --- a/environments/environment-MAC.yml +++ b/environments/environment-MAC.yml @@ -7,8 +7,9 @@ dependencies: - PyInstaller = 5.13.0 - nodejs = 18.16.1 - numcodecs = 0.11.0 - - jsonschema = 4.18.0 # this installs jsonschema-specifications which is needed for validation - - pydantic[email] = 2.0.2 # email validator is used by dandi + # install these from conda-forge so that dependent packages get included in the distributable + - jsonschema = 4.18.0 # installs jsonschema-specifications + - pydantic[email] = 2.0.2 # installs email-validator - pip - pip: - chardet == 5.1.0 diff --git a/environments/environment-Windows.yml b/environments/environment-Windows.yml index eb846dee2..6a53b6567 100644 --- a/environments/environment-Windows.yml +++ b/environments/environment-Windows.yml @@ -1,23 +1,24 @@ name: nwb-guide channels: - - defaults - conda-forge + - defaults dependencies: - - python = 3.9.17 - - nodejs = 18.16.1 + - python = 3.9.18 - PyInstaller = 5.13.0 - - pywin32 = 303 - - git = 2.20.1 - - setuptools=58.0.4 + - nodejs = 18.16.1 + - numcodecs = 0.11.0 # not sure if this is needed + - pywin32 = 303 # not sure if this is needed + # install these from conda-forge so that dependent packages get included in the distributable + - jsonschema = 4.18.0 # installs jsonschema-specifications + - pydantic[email] = 2.0.2 # installs email-validator - pip - pip: - chardet == 5.1.0 - configparser == 6.0.0 - flask == 2.3.2 - - flask-cors === 3.0.10 + - flask-cors == 4.0.0 - flask_restx == 1.1.0 - neuroconv @ git+https://github.com/catalystneuro/neuroconv.git@main#neuroconv[full] - - hdmf >= 3.7.0 - - pytest == 7.2.2 + - pytest == 7.4.0 - pytest-cov == 4.1.0 - - email-validator == 2.0.0 + diff --git a/package.json b/package.json index 08db3502b..f9aff8eab 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,7 @@ "build": "npm run build:app", "build:app": "electron-vite build --outDir build", "build:win": "npm run build && npm run build:flask:win && npm run build:electron:win", - "build:mac": "npm run build && rm /Users/runner/miniconda3/envs/nwb-guide/lib/python3.9/site-packages/sonpy/linux/sonpy.so && npm run build:flask:unix && npm run build:electron:mac", + "build:mac": "npm run build && npm run build:flask:unix && npm run build:electron:mac", "build:linux": "npm run build && npm run build:flask:unix && npm run build:electron:linux", "build:flask:base": "python -m PyInstaller --log-level DEBUG --name nwb-guide --onedir --clean --noconfirm ./pyflask/app.py --distpath ./build/flask --collect-data jsonschema_specifications --collect-all nwbinspector --collect-all neuroconv --collect-all pynwb --collect-all hdmf --collect-all ci_info --collect-all ndx_dandi_icephys --hidden-import scipy._distributor_init --hidden-import scipy._lib.messagestream --hidden-import scipy._lib._ccallback --hidden-import scipy._lib._testutils", "build:flask:win": "npm run build:flask:base -- --add-data ./paths.config.json;. --add-data ./package.json;.", From 18c363ce6563d68b2ca5dfa1c905d54ef29cbec8 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 9 Sep 2023 20:02:22 +0000 Subject: [PATCH 44/62] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- environments/environment-Windows.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/environments/environment-Windows.yml b/environments/environment-Windows.yml index 6a53b6567..5783d3f9b 100644 --- a/environments/environment-Windows.yml +++ b/environments/environment-Windows.yml @@ -21,4 +21,3 @@ dependencies: - neuroconv @ git+https://github.com/catalystneuro/neuroconv.git@main#neuroconv[full] - pytest == 7.4.0 - pytest-cov == 4.1.0 - From 0760fc4d2ce8d9b3a515b90ac5d04f25f50899e2 Mon Sep 17 00:00:00 2001 From: rly Date: Sat, 9 Sep 2023 22:38:12 +0200 Subject: [PATCH 45/62] Fix windows? --- environments/environment-Windows.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/environments/environment-Windows.yml b/environments/environment-Windows.yml index 5783d3f9b..6e8bf2815 100644 --- a/environments/environment-Windows.yml +++ b/environments/environment-Windows.yml @@ -19,5 +19,6 @@ dependencies: - flask-cors == 4.0.0 - flask_restx == 1.1.0 - neuroconv @ git+https://github.com/catalystneuro/neuroconv.git@main#neuroconv[full] + - hdmf >= 3.7.0 - pytest == 7.4.0 - pytest-cov == 4.1.0 From cd9e512dc1ea172ff31b9171609151510b3aadc0 Mon Sep 17 00:00:00 2001 From: rly Date: Sat, 9 Sep 2023 22:56:10 +0200 Subject: [PATCH 46/62] Fix mac build from cache --- .github/workflows/pyflask-build-and-dist-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pyflask-build-and-dist-tests.yml b/.github/workflows/pyflask-build-and-dist-tests.yml index 8bf667130..5fbea858d 100644 --- a/.github/workflows/pyflask-build-and-dist-tests.yml +++ b/.github/workflows/pyflask-build-and-dist-tests.yml @@ -85,7 +85,7 @@ jobs: # fix for macos build - if: matrix.os == 'macos-latest' - run: rm /Users/runner/miniconda3/envs/nwb-guide/lib/python3.9/site-packages/sonpy/linux/sonpy.so + run: rm -f /Users/runner/miniconda3/envs/nwb-guide/lib/python3.9/site-packages/sonpy/linux/sonpy.so - name: Build PyFlask distribution run: npm run build:flask:${{ matrix.shorthand }} From 7f1d4d6474cfa068b1ed6523345e02ed2992e9a0 Mon Sep 17 00:00:00 2001 From: rly Date: Sat, 9 Sep 2023 22:56:14 +0200 Subject: [PATCH 47/62] reset windows --- environments/environment-Windows.yml | 43 ++++++++++++++++++++++------ 1 file changed, 34 insertions(+), 9 deletions(-) diff --git a/environments/environment-Windows.yml b/environments/environment-Windows.yml index 6e8bf2815..9030ad6b0 100644 --- a/environments/environment-Windows.yml +++ b/environments/environment-Windows.yml @@ -1,24 +1,49 @@ name: nwb-guide channels: - - conda-forge - defaults + - conda-forge dependencies: - - python = 3.9.18 - - PyInstaller = 5.13.0 + - python = 3.9.17 - nodejs = 18.16.1 - - numcodecs = 0.11.0 # not sure if this is needed - - pywin32 = 303 # not sure if this is needed - # install these from conda-forge so that dependent packages get included in the distributable - - jsonschema = 4.18.0 # installs jsonschema-specifications + - PyInstaller = 5.13.0 + - pywin32 = 303 + - git = 2.20.1 + - setuptools=58.0.4 - pydantic[email] = 2.0.2 # installs email-validator - pip - pip: - chardet == 5.1.0 - configparser == 6.0.0 - flask == 2.3.2 - - flask-cors == 4.0.0 + - flask-cors === 3.0.10 - flask_restx == 1.1.0 - neuroconv @ git+https://github.com/catalystneuro/neuroconv.git@main#neuroconv[full] - hdmf >= 3.7.0 - - pytest == 7.4.0 + - pytest == 7.2.2 - pytest-cov == 4.1.0 + # - email-validator == 2.0.0 + +# name: nwb-guide +# channels: +# - conda-forge +# - defaults +# dependencies: +# - python = 3.9.18 +# - PyInstaller = 5.13.0 +# - nodejs = 18.16.1 +# - numcodecs = 0.11.0 # not sure if this is needed +# - pywin32 = 303 # not sure if this is needed +# # install these from conda-forge so that dependent packages get included in the distributable +# - jsonschema = 4.18.0 # installs jsonschema-specifications +# - pydantic[email] = 2.0.2 # installs email-validator +# - pip +# - pip: +# - chardet == 5.1.0 +# - configparser == 6.0.0 +# - flask == 2.3.2 +# - flask-cors == 4.0.0 +# - flask_restx == 1.1.0 +# - neuroconv @ git+https://github.com/catalystneuro/neuroconv.git@main#neuroconv[full] +# - hdmf >= 3.7.0 +# - pytest == 7.4.0 +# - pytest-cov == 4.1.0 From 0a5d9d0db7eed16efc4d4a1c3fb65cad59687a41 Mon Sep 17 00:00:00 2001 From: rly Date: Sat, 9 Sep 2023 23:30:43 +0200 Subject: [PATCH 48/62] reset windows --- environments/environment-Windows.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/environments/environment-Windows.yml b/environments/environment-Windows.yml index 9030ad6b0..44ef46d0c 100644 --- a/environments/environment-Windows.yml +++ b/environments/environment-Windows.yml @@ -9,7 +9,6 @@ dependencies: - pywin32 = 303 - git = 2.20.1 - setuptools=58.0.4 - - pydantic[email] = 2.0.2 # installs email-validator - pip - pip: - chardet == 5.1.0 @@ -21,7 +20,7 @@ dependencies: - hdmf >= 3.7.0 - pytest == 7.2.2 - pytest-cov == 4.1.0 - # - email-validator == 2.0.0 + - email-validator == 2.0.0 # name: nwb-guide # channels: From 92e1f4653ce298542e62b2139024c7a9a31ea032 Mon Sep 17 00:00:00 2001 From: rly Date: Sat, 9 Sep 2023 23:47:22 +0200 Subject: [PATCH 49/62] Fix missing import --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index f9aff8eab..7bfa42d93 100644 --- a/package.json +++ b/package.json @@ -15,7 +15,7 @@ "build:win": "npm run build && npm run build:flask:win && npm run build:electron:win", "build:mac": "npm run build && npm run build:flask:unix && npm run build:electron:mac", "build:linux": "npm run build && npm run build:flask:unix && npm run build:electron:linux", - "build:flask:base": "python -m PyInstaller --log-level DEBUG --name nwb-guide --onedir --clean --noconfirm ./pyflask/app.py --distpath ./build/flask --collect-data jsonschema_specifications --collect-all nwbinspector --collect-all neuroconv --collect-all pynwb --collect-all hdmf --collect-all ci_info --collect-all ndx_dandi_icephys --hidden-import scipy._distributor_init --hidden-import scipy._lib.messagestream --hidden-import scipy._lib._ccallback --hidden-import scipy._lib._testutils", + "build:flask:base": "python -m PyInstaller --log-level DEBUG --name nwb-guide --onedir --clean --noconfirm ./pyflask/app.py --distpath ./build/flask --collect-data jsonschema_specifications --collect-all nwbinspector --collect-all neuroconv --collect-all pynwb --collect-all hdmf --collect-all ci_info --collect-all ndx_dandi_icephys --hidden-import scipy._distributor_init --hidden-import scipy._lib.messagestream --hidden-import scipy._lib._ccallback --hidden-import scipy._lib._testutils --hidden-import email_validator", "build:flask:win": "npm run build:flask:base -- --add-data ./paths.config.json;. --add-data ./package.json;.", "build:flask:unix": "npm run build:flask:base -- --add-data ./paths.config.json:. --add-data ./package.json:.", "build:electron:win": "electron-builder build --win --publish never", From b236bba9e2e04a24f665564450bccfca007e35a1 Mon Sep 17 00:00:00 2001 From: Garrett Date: Mon, 11 Sep 2023 10:37:45 -0700 Subject: [PATCH 50/62] Rename and change iframe src --- docs/{support.rst => format_support.rst} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename docs/{support.rst => format_support.rst} (59%) diff --git a/docs/support.rst b/docs/format_support.rst similarity index 59% rename from docs/support.rst rename to docs/format_support.rst index 38c26bad0..a6363ace6 100644 --- a/docs/support.rst +++ b/docs/format_support.rst @@ -4,4 +4,4 @@ The following Google Sheet is a live record of all the supported formats in the .. raw:: html - + From 778d8372e55f8e4703361f1091daa1264270fdc9 Mon Sep 17 00:00:00 2001 From: Garrett Date: Mon, 11 Sep 2023 10:39:54 -0700 Subject: [PATCH 51/62] Update text --- docs/format_support.rst | 2 +- docs/index.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/format_support.rst b/docs/format_support.rst index a6363ace6..34ff68a5e 100644 --- a/docs/format_support.rst +++ b/docs/format_support.rst @@ -1,6 +1,6 @@ Ecosystem Format Support ======================================= -The following Google Sheet is a live record of all the supported formats in the NWB GUIDE and underlying ecosystem. +The following is a live record of all the supported formats in the NWB GUIDE and underlying ecosystem. .. raw:: html diff --git a/docs/index.rst b/docs/index.rst index 895337c80..432034626 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -19,4 +19,4 @@ The resulting files are fully compliant with the best practices expected of the :maxdepth: 2 developer_guide - support + format_support From db3a46388c147981d2ca420b8ed97674c534f622 Mon Sep 17 00:00:00 2001 From: Garrett Date: Mon, 11 Sep 2023 10:47:30 -0700 Subject: [PATCH 52/62] Update format_support.rst --- docs/format_support.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/format_support.rst b/docs/format_support.rst index 34ff68a5e..831d5e462 100644 --- a/docs/format_support.rst +++ b/docs/format_support.rst @@ -4,4 +4,4 @@ The following is a live record of all the supported formats in the NWB GUIDE and .. raw:: html - + From 28627b4755d2d4ee98799f55016c95f93761a84a Mon Sep 17 00:00:00 2001 From: Garrett Date: Mon, 11 Sep 2023 11:17:29 -0700 Subject: [PATCH 53/62] Update format_support.rst --- docs/format_support.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/format_support.rst b/docs/format_support.rst index 831d5e462..d2f35083e 100644 --- a/docs/format_support.rst +++ b/docs/format_support.rst @@ -4,4 +4,4 @@ The following is a live record of all the supported formats in the NWB GUIDE and .. raw:: html - + From 4c5d756e227b1e271faf0bb4c72a9e211b7c351f Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 11 Sep 2023 21:46:18 +0000 Subject: [PATCH 54/62] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/psf/black: 23.7.0 → 23.9.1](https://github.com/psf/black/compare/23.7.0...23.9.1) --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d3f9b3e7f..5cbf5fcc8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -6,7 +6,7 @@ repos: - id: end-of-file-fixer - id: trailing-whitespace - repo: https://github.com/psf/black - rev: 23.7.0 + rev: 23.9.1 hooks: - id: black exclude: ^docs/ From 9b84832a524e99a6b38b2a7a18eb110046f5f0be Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Tue, 12 Sep 2023 12:30:08 -0400 Subject: [PATCH 55/62] try neuroconv specific debug --- environments/environment-MAC-arm64.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environments/environment-MAC-arm64.yml b/environments/environment-MAC-arm64.yml index bb11633f9..93bd9ef53 100644 --- a/environments/environment-MAC-arm64.yml +++ b/environments/environment-MAC-arm64.yml @@ -20,6 +20,6 @@ dependencies: - flask == 2.3.2 - flask-cors == 4.0.0 - flask_restx == 1.1.0 - - neuroconv @ git+https://github.com/catalystneuro/neuroconv.git@main#neuroconv[ecephys] + - neuroconv @ git+https://github.com/catalystneuro/neuroconv.git@improve_sonpy_installation_m1#neuroconv[full] - pytest == 7.4.0 - pytest-cov == 4.1.0 From c3658e7bf71d7a2ca6545a11161c39f9cd23bedd Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Tue, 12 Sep 2023 13:45:44 -0400 Subject: [PATCH 56/62] Update pyflask-build-and-dist-tests.yml --- .github/workflows/pyflask-build-and-dist-tests.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pyflask-build-and-dist-tests.yml b/.github/workflows/pyflask-build-and-dist-tests.yml index 5fbea858d..a58c77944 100644 --- a/.github/workflows/pyflask-build-and-dist-tests.yml +++ b/.github/workflows/pyflask-build-and-dist-tests.yml @@ -85,7 +85,9 @@ jobs: # fix for macos build - if: matrix.os == 'macos-latest' - run: rm -f /Users/runner/miniconda3/envs/nwb-guide/lib/python3.9/site-packages/sonpy/linux/sonpy.so + run: | + rm -f /Users/runner/miniconda3/envs/nwb-guide/lib/python3.9/site-packages/sonpy/linux/sonpy.so + rm -f /usr/local/miniconda/envs/nwb-guide/lib/python3.9/site-packages/sonpy/linux/sonpy.so - name: Build PyFlask distribution run: npm run build:flask:${{ matrix.shorthand }} From e3d9eff1acc6f20a561d4d6c6e74ef3401e614c3 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Tue, 12 Sep 2023 13:46:59 -0400 Subject: [PATCH 57/62] Update Build-and-deploy-mac.yml --- .github/workflows/Build-and-deploy-mac.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/Build-and-deploy-mac.yml b/.github/workflows/Build-and-deploy-mac.yml index e4ee051bf..3411f2fed 100644 --- a/.github/workflows/Build-and-deploy-mac.yml +++ b/.github/workflows/Build-and-deploy-mac.yml @@ -33,6 +33,9 @@ jobs: - name: Install package.json modules and their dependencies run: npm install --ignore-scripts + - if: matrix.os == 'macos-latest' + run: rm -f /usr/local/miniconda/envs/nwb-guide/lib/python3.9/site-packages/sonpy/linux/sonpy.so + - uses: apple-actions/import-codesign-certs@v1 with: # https://developer.apple.com/account/resources/certificates/add @@ -56,5 +59,4 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} appleId: ${{ secrets.APPLE_ID }} # currently this is set to Ryan's Apple ID and password appleIdPassword: ${{ secrets.APPLE_PASSWORD }} - run: | - npm run deploy:mac + run: npm run deploy:mac From 97b1caa6eee66f4e1d398ea7d97e59e9c6897bc0 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Tue, 12 Sep 2023 13:47:13 -0400 Subject: [PATCH 58/62] Update pyflask-build-and-dist-tests.yml --- .github/workflows/pyflask-build-and-dist-tests.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/pyflask-build-and-dist-tests.yml b/.github/workflows/pyflask-build-and-dist-tests.yml index a58c77944..5fbea858d 100644 --- a/.github/workflows/pyflask-build-and-dist-tests.yml +++ b/.github/workflows/pyflask-build-and-dist-tests.yml @@ -85,9 +85,7 @@ jobs: # fix for macos build - if: matrix.os == 'macos-latest' - run: | - rm -f /Users/runner/miniconda3/envs/nwb-guide/lib/python3.9/site-packages/sonpy/linux/sonpy.so - rm -f /usr/local/miniconda/envs/nwb-guide/lib/python3.9/site-packages/sonpy/linux/sonpy.so + run: rm -f /Users/runner/miniconda3/envs/nwb-guide/lib/python3.9/site-packages/sonpy/linux/sonpy.so - name: Build PyFlask distribution run: npm run build:flask:${{ matrix.shorthand }} From 97b8d8f86cc26e140272f564479372960c070aca Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Tue, 12 Sep 2023 14:08:21 -0400 Subject: [PATCH 59/62] Update Build-and-deploy-mac.yml --- .github/workflows/Build-and-deploy-mac.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/Build-and-deploy-mac.yml b/.github/workflows/Build-and-deploy-mac.yml index 3411f2fed..dd7a77988 100644 --- a/.github/workflows/Build-and-deploy-mac.yml +++ b/.github/workflows/Build-and-deploy-mac.yml @@ -33,7 +33,7 @@ jobs: - name: Install package.json modules and their dependencies run: npm install --ignore-scripts - - if: matrix.os == 'macos-latest' + - name: Remove bad sonpy file (might make Spike2 format unusable on Mac - should exclude from selection) run: rm -f /usr/local/miniconda/envs/nwb-guide/lib/python3.9/site-packages/sonpy/linux/sonpy.so - uses: apple-actions/import-codesign-certs@v1 From 942edbdbbf4ab441faebf59372cccfdf7abecf13 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Tue, 12 Sep 2023 14:09:15 -0400 Subject: [PATCH 60/62] update to main post merge --- environments/environment-MAC-arm64.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environments/environment-MAC-arm64.yml b/environments/environment-MAC-arm64.yml index 93bd9ef53..6e4508a61 100644 --- a/environments/environment-MAC-arm64.yml +++ b/environments/environment-MAC-arm64.yml @@ -20,6 +20,6 @@ dependencies: - flask == 2.3.2 - flask-cors == 4.0.0 - flask_restx == 1.1.0 - - neuroconv @ git+https://github.com/catalystneuro/neuroconv.git@improve_sonpy_installation_m1#neuroconv[full] + - neuroconv @ git+https://github.com/catalystneuro/neuroconv.git@main#neuroconv[full] - pytest == 7.4.0 - pytest-cov == 4.1.0 From ddcbc4b255789e7a454ee432bd3fa73243c69809 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Tue, 12 Sep 2023 15:09:56 -0400 Subject: [PATCH 61/62] Update package.json --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 7bfa42d93..3ea6741de 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "nwb-guide", "productName": "NWB GUIDE", - "version": "0.0.4", + "version": "0.0.5", "description": "", "main": "./build/main/main.js", "scripts": { From 00cf557b0374dbba5ddeb037e535ba590aab3aec Mon Sep 17 00:00:00 2001 From: Garrett Date: Tue, 12 Sep 2023 17:25:43 -0700 Subject: [PATCH 62/62] Initialize without activate call --- src/main/main.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/main.ts b/src/main/main.ts index d00c1219b..da68ea02b 100755 --- a/src/main/main.ts +++ b/src/main/main.ts @@ -383,7 +383,7 @@ function makeSingleInstance() { else app.on("second-instance", () => restoreWindow()); } -if (process.platform === 'darwin') initialize(); +initialize(); app.on('activate', () => { if (BrowserWindow.getAllWindows().length === 0) initialize()