From eacd75bca52e88484b53c186d1f4f795c47e815c Mon Sep 17 00:00:00 2001 From: Garrett Date: Thu, 2 Nov 2023 11:49:23 -0700 Subject: [PATCH] Properly restrict job count --- pyflask/app.py | 5 + schemas/dandi-upload.schema.ts | 13 ++ schemas/json/dandi/upload.json | 12 +- src/renderer/src/index.ts | 111 +------------- src/renderer/src/server.ts | 139 ++++++++++++++++++ src/renderer/src/stories/JSONSchemaInput.js | 45 +++++- .../guided-mode/data/GuidedSourceData.js | 2 + .../pages/guided-mode/options/GuidedUpload.js | 32 +++- .../src/stories/pages/uploads/UploadsPage.js | 67 ++++++--- 9 files changed, 281 insertions(+), 145 deletions(-) create mode 100644 schemas/dandi-upload.schema.ts create mode 100644 src/renderer/src/server.ts diff --git a/pyflask/app.py b/pyflask/app.py index 319dda6c4..95038ca15 100644 --- a/pyflask/app.py +++ b/pyflask/app.py @@ -95,6 +95,11 @@ def send_preview(path): return send_from_directory(STUB_SAVE_FOLDER_PATH, path) +@app.route("/cpus") +def get_cpu_count(): + from psutil import cpu_count + return {"physical": cpu_count(logical=False) } + @api.route("/server_shutdown", endpoint="shutdown") class Shutdown(Resource): def get(self): diff --git a/schemas/dandi-upload.schema.ts b/schemas/dandi-upload.schema.ts new file mode 100644 index 000000000..244acaa2e --- /dev/null +++ b/schemas/dandi-upload.schema.ts @@ -0,0 +1,13 @@ +import upload from './json/dandi/upload.json' assert { type: "json" } + + +upload.properties.number_of_jobs.transform = upload.properties.number_of_threads.transform = (value, prevValue) => { + if (value === 0){ + if (prevValue === -1) return 1 + else return -1 + } + + return value +} + +export default upload \ No newline at end of file diff --git a/schemas/json/dandi/upload.json b/schemas/json/dandi/upload.json index a46148d3c..35b055c16 100644 --- a/schemas/json/dandi/upload.json +++ b/schemas/json/dandi/upload.json @@ -5,16 +5,18 @@ "description": "The unique identifier for your Dandiset, manually created on the main archive or staging server." }, "number_of_jobs": { - "type": "number", + "type": "integer", "title": "Job Count", - "description": "The number of files to upload in parallel.", - "default": 1 + "description": "The number of files to upload in parallel. A value of -1 uses all available processes", + "default": 1, + "min": -1 }, "number_of_threads": { - "type": "number", + "type": "integer", "title": "Threads per Job", "description": "The number of threads to handle each file.", - "default": 1 + "default": 1, + "min": -1 }, "cleanup": { "type": "boolean", diff --git a/src/renderer/src/index.ts b/src/renderer/src/index.ts index 1ba3787db..f84f36604 100644 --- a/src/renderer/src/index.ts +++ b/src/renderer/src/index.ts @@ -1,5 +1,5 @@ import "./pages.js" -import { isElectron, electron, app } from './electron/index.js' +import { isElectron, electron } from './electron/index.js' const { ipcRenderer } = electron; import { Dashboard } from './stories/Dashboard.js' @@ -9,27 +9,11 @@ import { notify } from './dependencies/globals.js' -import { baseUrl } from './globals.js' - import Swal from 'sweetalert2' - -import { StatusBar } from "./stories/status/StatusBar.js"; -import { unsafeSVG } from "lit/directives/unsafe-svg.js"; -import serverSVG from "./stories/assets/server.svg?raw"; -import webAssetSVG from "./stories/assets/web_asset.svg?raw"; -import wifiSVG from "./stories/assets/wifi.svg?raw"; +import { loadServerEvents, pythonServerOpened, statusBar } from "./server.js"; // Set the sidebar subtitle to the current app version const dashboard = document.querySelector('nwb-dashboard') as Dashboard -const appVersion = app?.getVersion(); - -const statusBar = new StatusBar({ - items: [ - { label: unsafeSVG(webAssetSVG), value: isElectron ? appVersion ?? 'ERROR' : 'Web' }, - { label: unsafeSVG(wifiSVG) }, - { label: unsafeSVG(serverSVG) } - ] -}) dashboard.subtitle = statusBar @@ -94,94 +78,10 @@ async function checkInternetConnection() { return hasInternet }; -// Check if the Flask server is live -const serverIsLiveStartup = async () => { - const echoResponse = await fetch(`${baseUrl}/startup/echo?arg=server ready`).then(res => res.json()).catch(e => e) - return echoResponse === "server ready" ? true : false; -}; - -// Preload Flask imports for faster on-demand operations -const preloadFlaskImports = () => fetch(`${baseUrl}/startup/preload-imports`).then(res => { - if (res.ok) return res.json() - else throw new Error('Error preloading Flask imports') -}) - -let openPythonStatusNotyf: undefined | any; - -async function pythonServerOpened() { - - - // Confirm requests are actually received by the server - const isLive = await serverIsLiveStartup() - // initiate preload of Flask imports - if (isLive) await preloadFlaskImports() - .then(() => { - - // Update server status and throw a notification - statusBar.items[2].status = true - - if (openPythonStatusNotyf) notyf.dismiss(openPythonStatusNotyf) - openPythonStatusNotyf = notyf.open({ - type: "success", - message: "Backend services are available", - }); - - }) - .catch(e =>{ - - statusBar.items[2].status = 'issue' - - notyf.open({ - type: "warning", - message: e.message, - }); - - }) - - // If the server is not live, throw an error - else return pythonServerClosed() - -} - - -async function pythonServerClosed(message?: string) { - - if (message) console.error(message) - statusBar.items[2].status = false - - await Swal.fire({ - icon: "error", - html: `

Something went wrong while initializing the application's background services.

Please restart NWB GUIDE and try again. If this issue occurs multiple times, please open an issue on the NWB GUIDE Issue Tracker.`, - heightAuto: false, - backdrop: "rgba(0,0,0, 0.4)", - confirmButtonText: "Exit Now", - allowOutsideClick: false, - allowEscapeKey: false, - }); - - if (isElectron) app.exit(); - else location.reload() - - Swal.close(); - - -} +loadServerEvents() if (isElectron) { - ipcRenderer.send("python.status"); // Trigger status check - - ipcRenderer.on("python.open", pythonServerOpened); - - ipcRenderer.on("python.closed", (_, message) => pythonServerClosed(message)); - ipcRenderer.on("python.restart", () => { - statusBar.items[2].status = undefined - if (openPythonStatusNotyf) notyf.dismiss(openPythonStatusNotyf) - openPythonStatusNotyf = notyf.open({ - type: "warning", - message: "Backend services are restarting...", - }) - }); // Check for update and show the pop up box ipcRenderer.on("update_available", () => { @@ -205,11 +105,12 @@ if (isElectron) { } else { update_downloaded_notification = notyf.open({ type: "app_update_warning", - message: + message: "Update downloaded. It will be installed on the restart of the app. Click here to restart NWB GUIDE now.", }); } - update_downloaded_notification.on("click", async ({ target, event }) => { + + update_downloaded_notification.on("click", async () => { restartApp(); }); }); diff --git a/src/renderer/src/server.ts b/src/renderer/src/server.ts new file mode 100644 index 000000000..50d33894d --- /dev/null +++ b/src/renderer/src/server.ts @@ -0,0 +1,139 @@ +import { isElectron, electron, app } from './electron/index.js' +const { ipcRenderer } = electron; + +import { + notyf, +} from './dependencies/globals.js' + +import { baseUrl } from './globals.js' + +import Swal from 'sweetalert2' + + +import { StatusBar } from "./stories/status/StatusBar.js"; +import { unsafeSVG } from "lit/directives/unsafe-svg.js"; +import serverSVG from "./stories/assets/server.svg?raw"; +import webAssetSVG from "./stories/assets/web_asset.svg?raw"; +import wifiSVG from "./stories/assets/wifi.svg?raw"; + +const appVersion = app?.getVersion(); + +export const statusBar = new StatusBar({ + items: [ + { label: unsafeSVG(webAssetSVG), value: isElectron ? appVersion ?? 'ERROR' : 'Web' }, + { label: unsafeSVG(wifiSVG) }, + { label: unsafeSVG(serverSVG) } + ] +}) + + +// Check if the Flask server is live +const serverIsLiveStartup = async () => { + const echoResponse = await fetch(`${baseUrl}/startup/echo?arg=server ready`).then(res => res.json()).catch(e => e) + return echoResponse === "server ready" ? true : false; + }; + + // Preload Flask imports for faster on-demand operations + const preloadFlaskImports = () => fetch(`${baseUrl}/startup/preload-imports`).then(res => { + if (res.ok) return res.json() + else throw new Error('Error preloading Flask imports') + }) + + +let serverCallbacks: Function[] = [] +export const onServerOpen = (callback:Function) => { + if (statusBar.items[2].status === true) return callback() + else { + return new Promise(res => { + serverCallbacks.push(() => { + res(callback()) + }) + + }) + } +} + +export async function pythonServerOpened() { + + + console.error('Server open') + + // Confirm requests are actually received by the server + const isLive = await serverIsLiveStartup() + + // initiate preload of Flask imports + if (isLive) await preloadFlaskImports() + .then(() => { + + // Update server status and throw a notification + statusBar.items[2].status = true + + serverCallbacks.forEach(cb => cb()) + serverCallbacks = [] + + if (openPythonStatusNotyf) notyf.dismiss(openPythonStatusNotyf) + openPythonStatusNotyf = notyf.open({ + type: "success", + message: "Backend services are available", + }); + + }) + .catch(e =>{ + + statusBar.items[2].status = 'issue' + + notyf.open({ + type: "warning", + message: e.message, + }); + + }) + + // If the server is not live, throw an error + else return pythonServerClosed() + +} + + +export async function pythonServerClosed(message?: string) { + + if (message) console.error(message) + statusBar.items[2].status = false + + await Swal.fire({ + icon: "error", + html: `

Something went wrong while initializing the application's background services.

Please restart NWB GUIDE and try again. If this issue occurs multiple times, please open an issue on the NWB GUIDE Issue Tracker.`, + heightAuto: false, + backdrop: "rgba(0,0,0, 0.4)", + confirmButtonText: "Exit Now", + allowOutsideClick: false, + allowEscapeKey: false, + }); + + if (isElectron) app.exit(); + else location.reload() + + Swal.close(); + + +} + +let openPythonStatusNotyf: undefined | any; + +export const loadServerEvents = () => { + if (isElectron) { + ipcRenderer.send("python.status"); // Trigger status check + + ipcRenderer.on("python.open", pythonServerOpened); + + ipcRenderer.on("python.closed", (_, message) => pythonServerClosed(message)); + ipcRenderer.on("python.restart", () => { + statusBar.items[2].status = undefined + if (openPythonStatusNotyf) notyf.dismiss(openPythonStatusNotyf) + openPythonStatusNotyf = notyf.open({ + type: "warning", + message: "Backend services are restarting...", + }) + }); + } +} \ No newline at end of file diff --git a/src/renderer/src/stories/JSONSchemaInput.js b/src/renderer/src/stories/JSONSchemaInput.js index f99643d45..35f87a2b7 100644 --- a/src/renderer/src/stories/JSONSchemaInput.js +++ b/src/renderer/src/stories/JSONSchemaInput.js @@ -345,7 +345,12 @@ export class JSONSchemaInput extends LitElement { ?checked=${this.value ?? false} @change=${(ev) => validateOnChange && this.#triggerValidation(name, path)} />`; - } else if (info.type === "string" || info.type === "number") { + } else if (info.type === "string" || info.type === "number" || info.type === 'integer') { + + const isInteger = info.type === 'integer'; + if (isInteger) info.type = 'number'; + const isNumber = info.type === 'number'; + const fileSystemFormat = isFilesystemSelector(name, info.format); if (fileSystemFormat) return createFilesystemSelector(fileSystemFormat); // Handle long string formats @@ -372,14 +377,46 @@ export class JSONSchemaInput extends LitElement { + @input=${(ev) => { + + + let value = ev.target.value; + let newValue = value + + // const isBlank = value === ''; + + if (isInteger) newValue = parseInt(value); + + else if (isNumber) newValue = parseFloat(value) + + if (isNumber) { + if ('min' in info && value < info.min) newValue = info.min + else if ('max' in info && value > info.max) newValue = info.max + } + + if (info.transform) newValue = info.transform(newValue, this.value, info) + + // // Do not check patter if value is empty + // if (info.pattern && !isBlank) { + // const regex = new RegExp(info.pattern) + // if (!regex.test(isNaN(newValue) ? value : newValue)) newValue = this.value // revert to last value + // } + + if (!isNaN(newValue) && newValue !== value) { + ev.target.value = newValue + value = newValue + } + + this.#updateData( fullPath, - info.type === "number" ? parseFloat(ev.target.value) : ev.target.value + value )} + } @change=${(ev) => validateOnChange && this.#triggerValidation(name, path)} /> `; diff --git a/src/renderer/src/stories/pages/guided-mode/data/GuidedSourceData.js b/src/renderer/src/stories/pages/guided-mode/data/GuidedSourceData.js index 70fa3bf03..fdbc85b5a 100644 --- a/src/renderer/src/stories/pages/guided-mode/data/GuidedSourceData.js +++ b/src/renderer/src/stories/pages/guided-mode/data/GuidedSourceData.js @@ -124,6 +124,8 @@ export class GuidedSourceDataPage extends ManagedPage { }) ); + await this.save() + this.to(1); }, }; diff --git a/src/renderer/src/stories/pages/guided-mode/options/GuidedUpload.js b/src/renderer/src/stories/pages/guided-mode/options/GuidedUpload.js index ca4a980c4..c742184db 100644 --- a/src/renderer/src/stories/pages/guided-mode/options/GuidedUpload.js +++ b/src/renderer/src/stories/pages/guided-mode/options/GuidedUpload.js @@ -5,9 +5,12 @@ import { run } from "./utils.js"; import { onThrow } from "../../../../errors"; import { merge } from "../../utils.js"; import Swal from "sweetalert2"; -import dandiUploadSchema from "../../../../../../../schemas/json/dandi/upload.json"; +import dandiUploadSchema from "../../../../../../../schemas/dandi-upload.schema"; import { dandisetInfoContent, uploadToDandi } from "../../uploads/UploadsPage.js"; import { InfoBox } from "../../../InfoBox.js"; +import { until } from "lit/directives/until.js"; +import { onServerOpen } from "../../../../server"; +import { baseUrl } from "../../../../globals.js"; export class GuidedUploadPage extends Page { constructor(...args) { @@ -64,17 +67,30 @@ export class GuidedUploadPage extends Page { render() { const state = (this.localState = merge(this.info.globalState.upload ?? { info: {} }, {})); - this.form = new JSONSchemaForm({ - schema: dandiUploadSchema, - results: state.info, - onUpdate: () => (this.unsavedUpdates = true), - onThrow, - }); + + const promise = onServerOpen(() => { + return fetch(new URL("cpus", baseUrl)) + .then((res) => res.json()).then(({ physical }) => { + + dandiUploadSchema.properties.number_of_jobs.max = physical; + + return this.form = new JSONSchemaForm({ + schema: dandiUploadSchema, + results: state.info, + onUpdate: () => (this.unsavedUpdates = true), + onThrow, + }); + + }) + }) return html`${new InfoBox({ header: "How do I create a Dandiset?", content: dandisetInfoContent, - })}

${this.form} `; + })}

${until( + promise, + html`Loading form contents...` + )} `; } } diff --git a/src/renderer/src/stories/pages/uploads/UploadsPage.js b/src/renderer/src/stories/pages/uploads/UploadsPage.js index f039d0eef..734565690 100644 --- a/src/renderer/src/stories/pages/uploads/UploadsPage.js +++ b/src/renderer/src/stories/pages/uploads/UploadsPage.js @@ -1,10 +1,12 @@ import { html } from "lit"; +import { until } from 'lit/directives/until.js'; + import { JSONSchemaForm } from "../../JSONSchemaForm.js"; import { Page } from "../Page.js"; import { onThrow } from "../../../errors"; const folderPathKey = "filesystem_paths"; -import dandiUploadSchema from "../../../../../../schemas/json/dandi/upload.json"; +import dandiUploadSchema from "../../../../../../schemas/dandi-upload.schema"; import dandiStandaloneSchema from "../../../../../../schemas/json/dandi/standalone.json"; const dandiSchema = merge(dandiStandaloneSchema, merge(dandiUploadSchema, {}), { arrays: true }); @@ -23,6 +25,9 @@ import { header } from "../../forms/utils"; import { validateDANDIApiKey } from "../../../validation/dandi"; import { InfoBox } from "../../InfoBox.js"; +import { onServerOpen } from '../../../server' +import { baseUrl } from "../../../globals.js"; + export const isStaging = (id) => parseInt(id) >= 100000; export const dandisetInfoContent = html` @@ -162,25 +167,34 @@ export class UploadsPage extends Page { }, }); - // NOTE: API Keys and Dandiset IDs persist across selected project - this.form = new JSONSchemaForm({ - results: globalState, - schema: dandiSchema, - sort: ([k1]) => { - if (k1 === folderPathKey) return -1; - }, - onUpdate: ([id]) => { - if (id === folderPathKey) { - for (let key in dandiSchema.properties) { - const input = this.form.getInput([key]); - if (key !== folderPathKey && input.value) input.updateData(""); // Clear the results of the form - } - } + const promise = onServerOpen(() => { + return fetch(new URL("cpus", baseUrl)) + .then((res) => res.json()).then(({ physical }) => { + console.log(physical) + + dandiSchema.properties.number_of_jobs.max = physical; + + // NOTE: API Keys and Dandiset IDs persist across selected project + return this.form = new JSONSchemaForm({ + results: globalState, + schema: dandiSchema, + sort: ([k1]) => { + if (k1 === folderPathKey) return -1; + }, + onUpdate: ([id]) => { + if (id === folderPathKey) { + for (let key in dandiSchema.properties) { + const input = this.form.getInput([key]); + if (key !== folderPathKey && input.value) input.updateData(""); // Clear the results of the form + } + } - global.save(); - }, - onThrow, - }); + global.save(); + }, + onThrow, + }); + }) + }) return html` ${new InfoBox({ @@ -189,12 +203,19 @@ export class UploadsPage extends Page { })}

- ${this.form} -
- ${button} + ${until( + promise.then(form => { + return html` + ${form} +
+ ${button} + ` + }), + html`

Waiting to connect to the Flask server...

`, + )}

- `; + ` } }