diff --git a/pyflask/app.py b/pyflask/app.py
index 319dda6c4..977dae0f3 100644
--- a/pyflask/app.py
+++ b/pyflask/app.py
@@ -95,6 +95,16 @@ def send_preview(path):
return send_from_directory(STUB_SAVE_FOLDER_PATH, path)
+@app.route("/cpus")
+def get_cpu_count():
+ from psutil import cpu_count
+
+ physical = cpu_count(logical=False)
+ logical = cpu_count()
+
+ return dict(physical=physical, logical=logical)
+
+
@api.route("/server_shutdown", endpoint="shutdown")
class Shutdown(Resource):
def get(self):
diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py
index ff07cb8b6..65585c00f 100644
--- a/pyflask/manageNeuroconv/manage_neuroconv.py
+++ b/pyflask/manageNeuroconv/manage_neuroconv.py
@@ -485,6 +485,8 @@ def upload_folder_to_dandi(
nwb_folder_path: Optional[str] = None,
staging: Optional[bool] = None, # Override default staging=True
cleanup: Optional[bool] = None,
+ number_of_jobs: Optional[int] = None,
+ number_of_threads: Optional[int] = None,
):
from neuroconv.tools.data_transfers import automatic_dandi_upload
@@ -495,6 +497,8 @@ def upload_folder_to_dandi(
nwb_folder_path=Path(nwb_folder_path),
staging=staging,
cleanup=cleanup,
+ number_of_jobs=number_of_jobs,
+ number_of_threads=number_of_threads,
)
@@ -504,6 +508,8 @@ def upload_to_dandi(
project: Optional[str] = None,
staging: Optional[bool] = None, # Override default staging=True
cleanup: Optional[bool] = None,
+ number_of_jobs: Optional[int] = None,
+ number_of_threads: Optional[int] = None,
):
from neuroconv.tools.data_transfers import automatic_dandi_upload
@@ -516,6 +522,8 @@ def upload_to_dandi(
nwb_folder_path=CONVERSION_SAVE_FOLDER_PATH / project, # Scope valid DANDI upload paths to GUIDE projects
staging=staging,
cleanup=cleanup,
+ number_of_jobs=number_of_jobs,
+ number_of_threads=number_of_threads,
)
diff --git a/schemas/dandi-upload.schema.ts b/schemas/dandi-upload.schema.ts
new file mode 100644
index 000000000..278121143
--- /dev/null
+++ b/schemas/dandi-upload.schema.ts
@@ -0,0 +1,13 @@
+import upload from './json/dandi/upload.json' assert { type: "json" }
+
+
+upload.properties.number_of_jobs.transform = upload.properties.number_of_threads.transform = (value, prevValue) => {
+ if (value === 0){
+ if (prevValue === -1) return 1
+ else return -1
+ }
+
+ return value
+}
+
+export default upload
diff --git a/schemas/json/dandi/upload.json b/schemas/json/dandi/upload.json
index c8507690b..163d75066 100644
--- a/schemas/json/dandi/upload.json
+++ b/schemas/json/dandi/upload.json
@@ -4,6 +4,20 @@
"type": "string",
"description": "The unique identifier for your Dandiset, manually created on the main archive or staging server."
},
+ "number_of_jobs": {
+ "type": "integer",
+ "title": "Job Count",
+ "description": "The number of files to upload in parallel. A value of -1
uses all available processes",
+ "default": 1,
+ "min": -1
+ },
+ "number_of_threads": {
+ "type": "integer",
+ "title": "Threads per Job",
+ "description": "The number of threads to handle each file. A value of -1
uses all available threads per process.",
+ "default": 1,
+ "min": -1
+ },
"cleanup": {
"type": "boolean",
"title": "Delete Local Files After Upload",
diff --git a/src/renderer/src/index.ts b/src/renderer/src/index.ts
index 1ba3787db..7d25747a0 100644
--- a/src/renderer/src/index.ts
+++ b/src/renderer/src/index.ts
@@ -1,5 +1,5 @@
import "./pages.js"
-import { isElectron, electron, app } from './electron/index.js'
+import { isElectron, electron } from './electron/index.js'
const { ipcRenderer } = electron;
import { Dashboard } from './stories/Dashboard.js'
@@ -9,31 +9,14 @@ import {
notify
} from './dependencies/globals.js'
-import { baseUrl } from './globals.js'
-
import Swal from 'sweetalert2'
-
-import { StatusBar } from "./stories/status/StatusBar.js";
-import { unsafeSVG } from "lit/directives/unsafe-svg.js";
-import serverSVG from "./stories/assets/server.svg?raw";
-import webAssetSVG from "./stories/assets/web_asset.svg?raw";
-import wifiSVG from "./stories/assets/wifi.svg?raw";
+import { loadServerEvents, pythonServerOpened, statusBar } from "./server.js";
// Set the sidebar subtitle to the current app version
const dashboard = document.querySelector('nwb-dashboard') as Dashboard
-const appVersion = app?.getVersion();
-
-const statusBar = new StatusBar({
- items: [
- { label: unsafeSVG(webAssetSVG), value: isElectron ? appVersion ?? 'ERROR' : 'Web' },
- { label: unsafeSVG(wifiSVG) },
- { label: unsafeSVG(serverSVG) }
- ]
-})
dashboard.subtitle = statusBar
-
//////////////////////////////////
// Connect to Python back-end
//////////////////////////////////
@@ -94,94 +77,10 @@ async function checkInternetConnection() {
return hasInternet
};
-// Check if the Flask server is live
-const serverIsLiveStartup = async () => {
- const echoResponse = await fetch(`${baseUrl}/startup/echo?arg=server ready`).then(res => res.json()).catch(e => e)
- return echoResponse === "server ready" ? true : false;
-};
-
-// Preload Flask imports for faster on-demand operations
-const preloadFlaskImports = () => fetch(`${baseUrl}/startup/preload-imports`).then(res => {
- if (res.ok) return res.json()
- else throw new Error('Error preloading Flask imports')
-})
-
-let openPythonStatusNotyf: undefined | any;
-
-async function pythonServerOpened() {
-
-
- // Confirm requests are actually received by the server
- const isLive = await serverIsLiveStartup()
-
- // initiate preload of Flask imports
- if (isLive) await preloadFlaskImports()
- .then(() => {
-
- // Update server status and throw a notification
- statusBar.items[2].status = true
-
- if (openPythonStatusNotyf) notyf.dismiss(openPythonStatusNotyf)
- openPythonStatusNotyf = notyf.open({
- type: "success",
- message: "Backend services are available",
- });
-
- })
- .catch(e =>{
-
- statusBar.items[2].status = 'issue'
-
- notyf.open({
- type: "warning",
- message: e.message,
- });
-
- })
-
- // If the server is not live, throw an error
- else return pythonServerClosed()
-
-}
-
-
-async function pythonServerClosed(message?: string) {
-
- if (message) console.error(message)
- statusBar.items[2].status = false
- await Swal.fire({
- icon: "error",
- html: `
Something went wrong while initializing the application's background services.
Please restart NWB GUIDE and try again. If this issue occurs multiple times, please open an issue on the NWB GUIDE Issue Tracker.`, - heightAuto: false, - backdrop: "rgba(0,0,0, 0.4)", - confirmButtonText: "Exit Now", - allowOutsideClick: false, - allowEscapeKey: false, - }); - - if (isElectron) app.exit(); - else location.reload() - - Swal.close(); - - -} +loadServerEvents() if (isElectron) { - ipcRenderer.send("python.status"); // Trigger status check - - ipcRenderer.on("python.open", pythonServerOpened); - - ipcRenderer.on("python.closed", (_, message) => pythonServerClosed(message)); - ipcRenderer.on("python.restart", () => { - statusBar.items[2].status = undefined - if (openPythonStatusNotyf) notyf.dismiss(openPythonStatusNotyf) - openPythonStatusNotyf = notyf.open({ - type: "warning", - message: "Backend services are restarting...", - }) - }); // Check for update and show the pop up box ipcRenderer.on("update_available", () => { @@ -205,11 +104,12 @@ if (isElectron) { } else { update_downloaded_notification = notyf.open({ type: "app_update_warning", - message: + message: "Update downloaded. It will be installed on the restart of the app. Click here to restart NWB GUIDE now.", }); } - update_downloaded_notification.on("click", async ({ target, event }) => { + + update_downloaded_notification.on("click", async () => { restartApp(); }); }); diff --git a/src/renderer/src/server.ts b/src/renderer/src/server.ts new file mode 100644 index 000000000..1485676b7 --- /dev/null +++ b/src/renderer/src/server.ts @@ -0,0 +1,143 @@ +import { isElectron, electron, app } from './electron/index.js' +const { ipcRenderer } = electron; + +import { + notyf, +} from './dependencies/globals.js' + +import { baseUrl } from './globals.js' + +import Swal from 'sweetalert2' + + +import { StatusBar } from "./stories/status/StatusBar.js"; +import { unsafeSVG } from "lit/directives/unsafe-svg.js"; +import serverSVG from "./stories/assets/server.svg?raw"; +import webAssetSVG from "./stories/assets/web_asset.svg?raw"; +import wifiSVG from "./stories/assets/wifi.svg?raw"; + +const appVersion = app?.getVersion(); + +export const statusBar = new StatusBar({ + items: [ + { label: unsafeSVG(webAssetSVG), value: isElectron ? appVersion ?? 'ERROR' : 'Web' }, + { label: unsafeSVG(wifiSVG) }, + { label: unsafeSVG(serverSVG) } + ] +}) + + +// Check if the Flask server is live +const serverIsLiveStartup = async () => { + const echoResponse = await fetch(`${baseUrl}/startup/echo?arg=server ready`).then(res => res.json()).catch(e => e) + return echoResponse === "server ready" ? true : false; + }; + + // Preload Flask imports for faster on-demand operations + const preloadFlaskImports = () => fetch(`${baseUrl}/startup/preload-imports`).then(res => { + if (res.ok) return res.json() + else throw new Error('Error preloading Flask imports') + }) + + +let serverCallbacks: Function[] = [] +export const onServerOpen = (callback:Function) => { + if (statusBar.items[2].status === true) return callback() + else { + return new Promise(res => { + serverCallbacks.push(() => { + res(callback()) + }) + + }) + } +} + +export const activateServer = () => { + console.log('GO!') + statusBar.items[2].status = true + + serverCallbacks.forEach(cb => cb()) + serverCallbacks = [] +} + +export async function pythonServerOpened() { + + // Confirm requests are actually received by the server + const isLive = await serverIsLiveStartup() + + // initiate preload of Flask imports + if (isLive) await preloadFlaskImports() + .then(() => { + + // Update server status and throw a notification + activateServer() + + if (openPythonStatusNotyf) notyf.dismiss(openPythonStatusNotyf) + openPythonStatusNotyf = notyf.open({ + type: "success", + message: "Backend services are available", + }); + + }) + .catch(e =>{ + + statusBar.items[2].status = 'issue' + + notyf.open({ + type: "warning", + message: e.message, + }); + + }) + + // If the server is not live, throw an error + else return pythonServerClosed() + +} + + +export async function pythonServerClosed(message?: string) { + + if (message) console.error(message) + statusBar.items[2].status = false + + await Swal.fire({ + icon: "error", + html: `Something went wrong while initializing the application's background services.
Please restart NWB GUIDE and try again. If this issue occurs multiple times, please open an issue on the NWB GUIDE Issue Tracker.`, + heightAuto: false, + backdrop: "rgba(0,0,0, 0.4)", + confirmButtonText: "Exit Now", + allowOutsideClick: false, + allowEscapeKey: false, + }); + + if (isElectron) app.exit(); + else location.reload() + + Swal.close(); + + +} + +let openPythonStatusNotyf: undefined | any; + +export const loadServerEvents = () => { + if (isElectron) { + ipcRenderer.send("python.status"); // Trigger status check + + ipcRenderer.on("python.open", pythonServerOpened); + + ipcRenderer.on("python.closed", (_, message) => pythonServerClosed(message)); + ipcRenderer.on("python.restart", () => { + statusBar.items[2].status = undefined + if (openPythonStatusNotyf) notyf.dismiss(openPythonStatusNotyf) + openPythonStatusNotyf = notyf.open({ + type: "warning", + message: "Backend services are restarting...", + }) + }); + } + + else activateServer() // Just mock-activate the server if we're in the browser +} diff --git a/src/renderer/src/stories/JSONSchemaInput.js b/src/renderer/src/stories/JSONSchemaInput.js index 76b105a68..6029bff6f 100644 --- a/src/renderer/src/stories/JSONSchemaInput.js +++ b/src/renderer/src/stories/JSONSchemaInput.js @@ -348,7 +348,11 @@ export class JSONSchemaInput extends LitElement { ?checked=${this.value ?? false} @change=${(ev) => validateOnChange && this.#triggerValidation(name, path)} />`; - } else if (info.type === "string" || info.type === "number") { + } else if (info.type === "string" || info.type === "number" || info.type === "integer") { + const isInteger = info.type === "integer"; + if (isInteger) info.type = "number"; + const isNumber = info.type === "number"; + const fileSystemFormat = isFilesystemSelector(name, info.format); if (fileSystemFormat) return createFilesystemSelector(fileSystemFormat); // Handle long string formats @@ -375,14 +379,38 @@ export class JSONSchemaInput extends LitElement { - this.#updateData( - fullPath, - info.type === "number" ? parseFloat(ev.target.value) : ev.target.value - )} + @input=${(ev) => { + let value = ev.target.value; + let newValue = value; + + // const isBlank = value === ''; + + if (isInteger) newValue = parseInt(value); + else if (isNumber) newValue = parseFloat(value); + + if (isNumber) { + if ("min" in info && value < info.min) newValue = info.min; + else if ("max" in info && value > info.max) newValue = info.max; + } + + if (info.transform) newValue = info.transform(newValue, this.value, info); + + // // Do not check patter if value is empty + // if (info.pattern && !isBlank) { + // const regex = new RegExp(info.pattern) + // if (!regex.test(isNaN(newValue) ? value : newValue)) newValue = this.value // revert to last value + // } + + if (!isNaN(newValue) && newValue !== value) { + ev.target.value = newValue; + value = newValue; + } + + this.#updateData(fullPath, value); + }} @change=${(ev) => validateOnChange && this.#triggerValidation(name, path)} /> `; diff --git a/src/renderer/src/stories/pages/guided-mode/data/GuidedSourceData.js b/src/renderer/src/stories/pages/guided-mode/data/GuidedSourceData.js index 70fa3bf03..f0c98fc80 100644 --- a/src/renderer/src/stories/pages/guided-mode/data/GuidedSourceData.js +++ b/src/renderer/src/stories/pages/guided-mode/data/GuidedSourceData.js @@ -124,6 +124,8 @@ export class GuidedSourceDataPage extends ManagedPage { }) ); + await this.save(); + this.to(1); }, }; diff --git a/src/renderer/src/stories/pages/guided-mode/options/GuidedUpload.js b/src/renderer/src/stories/pages/guided-mode/options/GuidedUpload.js index ca4a980c4..f3647e701 100644 --- a/src/renderer/src/stories/pages/guided-mode/options/GuidedUpload.js +++ b/src/renderer/src/stories/pages/guided-mode/options/GuidedUpload.js @@ -5,9 +5,12 @@ import { run } from "./utils.js"; import { onThrow } from "../../../../errors"; import { merge } from "../../utils.js"; import Swal from "sweetalert2"; -import dandiUploadSchema from "../../../../../../../schemas/json/dandi/upload.json"; +import dandiUploadSchema from "../../../../../../../schemas/dandi-upload.schema"; import { dandisetInfoContent, uploadToDandi } from "../../uploads/UploadsPage.js"; import { InfoBox } from "../../../InfoBox.js"; +import { until } from "lit/directives/until.js"; +import { onServerOpen } from "../../../../server"; +import { baseUrl } from "../../../../globals.js"; export class GuidedUploadPage extends Page { constructor(...args) { @@ -64,17 +67,27 @@ export class GuidedUploadPage extends Page { render() { const state = (this.localState = merge(this.info.globalState.upload ?? { info: {} }, {})); - this.form = new JSONSchemaForm({ - schema: dandiUploadSchema, - results: state.info, - onUpdate: () => (this.unsavedUpdates = true), - onThrow, + const promise = onServerOpen(async () => { + await fetch(new URL("cpus", baseUrl)) + .then((res) => res.json()) + .then(({ physical, logical }) => { + dandiUploadSchema.properties.number_of_jobs.max = physical; + dandiUploadSchema.properties.number_of_threads.max = logical / physical; + }) + .catch(() => {}); + + return (this.form = new JSONSchemaForm({ + schema: dandiUploadSchema, + results: state.info, + onUpdate: () => (this.unsavedUpdates = true), + onThrow, + })); }); return html`${new InfoBox({ header: "How do I create a Dandiset?", content: dandisetInfoContent, - })}Waiting to connect to the Flask server...
+ ` + )} +