Control DANDI Jobs and Threads

pyflask/app.py

@@ -95,6 +95,16 @@ def send_preview(path):
     return send_from_directory(STUB_SAVE_FOLDER_PATH, path)
 
 
+@app.route("/cpus")
+def get_cpu_count():
+    from psutil import cpu_count
+
+    physical = cpu_count(logical=False)
+    logical = cpu_count()
+
+    return dict(physical=physical, logical=logical)
+
+
 @api.route("/server_shutdown", endpoint="shutdown")
 class Shutdown(Resource):
     def get(self):

pyflask/manageNeuroconv/manage_neuroconv.py

@@ -485,6 +485,8 @@ def upload_folder_to_dandi(
     nwb_folder_path: Optional[str] = None,
     staging: Optional[bool] = None,  # Override default staging=True
     cleanup: Optional[bool] = None,
+    number_of_jobs: Optional[int] = None,
+    number_of_threads: Optional[int] = None,
 ):
     from neuroconv.tools.data_transfers import automatic_dandi_upload
 
@@ -495,6 +497,8 @@ def upload_folder_to_dandi(
         nwb_folder_path=Path(nwb_folder_path),
         staging=staging,
         cleanup=cleanup,
+        number_of_jobs=number_of_jobs,
+        number_of_threads=number_of_threads,
     )
 
 
@@ -504,6 +508,8 @@ def upload_to_dandi(
     project: Optional[str] = None,
     staging: Optional[bool] = None,  # Override default staging=True
     cleanup: Optional[bool] = None,
+    number_of_jobs: Optional[int] = None,
+    number_of_threads: Optional[int] = None,
 ):
     from neuroconv.tools.data_transfers import automatic_dandi_upload
 
@@ -516,6 +522,8 @@ def upload_to_dandi(
         nwb_folder_path=CONVERSION_SAVE_FOLDER_PATH / project,  # Scope valid DANDI upload paths to GUIDE projects
         staging=staging,
         cleanup=cleanup,
+        number_of_jobs=number_of_jobs,
+        number_of_threads=number_of_threads,
     )
 
 

schemas/dandi-upload.schema.ts

@@ -0,0 +1,13 @@
+import upload from './json/dandi/upload.json' assert { type: "json" }
+
+
+upload.properties.number_of_jobs.transform = upload.properties.number_of_threads.transform = (value, prevValue) => {
+    if (value === 0){
+        if (prevValue === -1) return 1
+        else return -1
+    }
+
+    return value
+}
+
+export default upload

schemas/json/dandi/upload.json

@@ -4,6 +4,20 @@
             "type": "string",
             "description": "The unique identifier for your Dandiset, manually created on the <a href='https://dandiarchive.org' target='_blank'>main archive</a> or <a href='https://gui-staging.dandiarchive.org' target='_blank'>staging server</a>."
         },
+        "number_of_jobs": {
+            "type": "integer",
+            "title": "Job Count",
+            "description": "The number of files to upload in parallel. A value of <code>-1</code> uses all available processes",
+            "default": 1,
+            "min": -1
+        },
+        "number_of_threads": {
+            "type": "integer",
+            "title": "Threads per Job",
+            "description": "The number of threads to handle each file. A value of <code>-1</code> uses all available threads per process.",
+            "default": 1,
+            "min": -1
+        },
         "cleanup": {
             "type": "boolean",
             "title": "Delete Local Files After Upload",

src/renderer/src/index.ts

@@ -1,5 +1,5 @@
 import "./pages.js"
-import { isElectron, electron, app } from './electron/index.js'
+import { isElectron, electron } from './electron/index.js'
 const { ipcRenderer } = electron;
 
 import { Dashboard } from './stories/Dashboard.js'
@@ -9,27 +9,11 @@ import {
   notify
 } from './dependencies/globals.js'
 
-import { baseUrl } from './globals.js'
-
 import Swal from 'sweetalert2'
-
-import { StatusBar } from "./stories/status/StatusBar.js";
-import { unsafeSVG } from "lit/directives/unsafe-svg.js";
-import serverSVG from "./stories/assets/server.svg?raw";
-import webAssetSVG from "./stories/assets/web_asset.svg?raw";
-import wifiSVG from "./stories/assets/wifi.svg?raw";
+import { loadServerEvents, pythonServerOpened, statusBar } from "./server.js";
 
 // Set the sidebar subtitle to the current app version
 const dashboard = document.querySelector('nwb-dashboard') as Dashboard
-const appVersion = app?.getVersion();
-
-const statusBar = new StatusBar({
-  items: [
-    { label: unsafeSVG(webAssetSVG), value: isElectron ? appVersion ?? 'ERROR' : 'Web' },
-    { label: unsafeSVG(wifiSVG) },
-    { label: unsafeSVG(serverSVG) }
-  ]
-})
 
 dashboard.subtitle = statusBar
 
@@ -94,94 +78,10 @@ async function checkInternetConnection() {
     return hasInternet
 };
 
-// Check if the Flask server is live
-const serverIsLiveStartup = async () => {
-  const echoResponse = await fetch(`${baseUrl}/startup/echo?arg=server ready`).then(res => res.json()).catch(e => e)
-  return echoResponse === "server ready" ? true : false;
-};
-
-// Preload Flask imports for faster on-demand operations
-const preloadFlaskImports = () => fetch(`${baseUrl}/startup/preload-imports`).then(res => {
-  if (res.ok) return res.json()
-  else throw new Error('Error preloading Flask imports')
-})
-
-let openPythonStatusNotyf: undefined | any;
-
-async function pythonServerOpened() {
-
-
-  // Confirm requests are actually received by the server
-  const isLive = await serverIsLiveStartup()
 
-  // initiate preload of Flask imports
-  if (isLive) await preloadFlaskImports()
-  .then(() => {
-
-      // Update server status and throw a notification
-    statusBar.items[2].status = true
-
-    if (openPythonStatusNotyf) notyf.dismiss(openPythonStatusNotyf)
-    openPythonStatusNotyf = notyf.open({
-      type: "success",
-      message: "Backend services are available",
-    });
-
-  })
-  .catch(e =>{
-
-    statusBar.items[2].status = 'issue'
-
-    notyf.open({
-      type: "warning",
-      message: e.message,
-    });
-
-  })
-
-  // If the server is not live, throw an error
-  else return pythonServerClosed()
-
-}
-
-
-async function pythonServerClosed(message?: string) {
-
-  if (message) console.error(message)
-  statusBar.items[2].status = false
-
-    await Swal.fire({
-      icon: "error",
-      html: `<p>Something went wrong while initializing the application's background services.</p><small>Please restart NWB GUIDE and try again. If this issue occurs multiple times, please open an issue on the <a href='https://github.com/catalystneuro/nwb-guide/issues' target='_blank'>NWB GUIDE Issue Tracker</a>.</small>`,
-      heightAuto: false,
-      backdrop: "rgba(0,0,0, 0.4)",
-      confirmButtonText: "Exit Now",
-      allowOutsideClick: false,
-      allowEscapeKey: false,
-    });
-
-    if (isElectron) app.exit();
-    else location.reload()
-
-    Swal.close();
-
-
-}
+loadServerEvents()
 
 if (isElectron) {
-    ipcRenderer.send("python.status"); // Trigger status check
-
-    ipcRenderer.on("python.open", pythonServerOpened);
-
-    ipcRenderer.on("python.closed", (_, message) => pythonServerClosed(message));
-    ipcRenderer.on("python.restart", () => {
-      statusBar.items[2].status = undefined
-      if (openPythonStatusNotyf) notyf.dismiss(openPythonStatusNotyf)
-      openPythonStatusNotyf = notyf.open({
-        type: "warning",
-        message: "Backend services are restarting...",
-      })
-    });
 
     // Check for update and show the pop up box
     ipcRenderer.on("update_available", () => {
@@ -205,11 +105,12 @@ if (isElectron) {
       } else {
         update_downloaded_notification = notyf.open({
           type: "app_update_warning",
-          message:
+      message:
             "Update downloaded. It will be installed on the restart of the app. Click here to restart NWB GUIDE now.",
         });
       }
-      update_downloaded_notification.on("click", async ({ target, event }) => {
+
+      update_downloaded_notification.on("click", async () => {
         restartApp();
       });
     });

src/renderer/src/server.ts

@@ -0,0 +1,139 @@
+import { isElectron, electron, app } from './electron/index.js'
+const { ipcRenderer } = electron;
+
+import {
+  notyf,
+} from './dependencies/globals.js'
+
+import { baseUrl } from './globals.js'
+
+import Swal from 'sweetalert2'
+
+
+import { StatusBar } from "./stories/status/StatusBar.js";
+import { unsafeSVG } from "lit/directives/unsafe-svg.js";
+import serverSVG from "./stories/assets/server.svg?raw";
+import webAssetSVG from "./stories/assets/web_asset.svg?raw";
+import wifiSVG from "./stories/assets/wifi.svg?raw";
+
+const appVersion = app?.getVersion();
+
+export const statusBar = new StatusBar({
+  items: [
+    { label: unsafeSVG(webAssetSVG), value: isElectron ? appVersion ?? 'ERROR' : 'Web' },
+    { label: unsafeSVG(wifiSVG) },
+    { label: unsafeSVG(serverSVG) }
+  ]
+})
+
+
+// Check if the Flask server is live
+const serverIsLiveStartup = async () => {
+    const echoResponse = await fetch(`${baseUrl}/startup/echo?arg=server ready`).then(res => res.json()).catch(e => e)
+    return echoResponse === "server ready" ? true : false;
+  };
+
+  // Preload Flask imports for faster on-demand operations
+  const preloadFlaskImports = () => fetch(`${baseUrl}/startup/preload-imports`).then(res => {
+    if (res.ok) return res.json()
+    else throw new Error('Error preloading Flask imports')
+  })
+
+
+let serverCallbacks: Function[] = []
+export const onServerOpen = (callback:Function) => {
+  if (statusBar.items[2].status === true) return callback()
+  else {
+    return new Promise(res => {
+        serverCallbacks.push(() => {
+            res(callback())
+        })
+
+    })
+ }
+}
+
+export async function pythonServerOpened() {
+
+
+  console.error('Server open')
+
+  // Confirm requests are actually received by the server
+  const isLive = await serverIsLiveStartup()
+
+  // initiate preload of Flask imports
+  if (isLive) await preloadFlaskImports()
+  .then(() => {
+
+      // Update server status and throw a notification
+    statusBar.items[2].status = true
+
+    serverCallbacks.forEach(cb => cb())
+    serverCallbacks = []
+
+    if (openPythonStatusNotyf) notyf.dismiss(openPythonStatusNotyf)
+    openPythonStatusNotyf = notyf.open({
+      type: "success",
+      message: "Backend services are available",
+    });
+
+  })
+  .catch(e =>{
+
+    statusBar.items[2].status = 'issue'
+
+    notyf.open({
+      type: "warning",
+      message: e.message,
+    });
+
+  })
+
+  // If the server is not live, throw an error
+  else return pythonServerClosed()
+
+}
+
+
+export async function pythonServerClosed(message?: string) {
+
+  if (message) console.error(message)
+  statusBar.items[2].status = false
+
+    await Swal.fire({
+      icon: "error",
+      html: `<p>Something went wrong while initializing the application's background services.</p><small>Please restart NWB GUIDE and try again. If this issue occurs multiple times, please open an issue on the <a href='https://github.com/catalystneuro/nwb-guide/issues' target='_blank'>NWB GUIDE Issue Tracker</a>.</small>`,
+      heightAuto: false,
+      backdrop: "rgba(0,0,0, 0.4)",
+      confirmButtonText: "Exit Now",
+      allowOutsideClick: false,
+      allowEscapeKey: false,
+    });
+
+    if (isElectron) app.exit();
+    else location.reload()
+
+    Swal.close();
+
+
+}
+
+let openPythonStatusNotyf: undefined | any;
+
+export const loadServerEvents = () => {
+    if (isElectron) {
+        ipcRenderer.send("python.status"); // Trigger status check
+
+        ipcRenderer.on("python.open", pythonServerOpened);
+
+        ipcRenderer.on("python.closed", (_, message) => pythonServerClosed(message));
+        ipcRenderer.on("python.restart", () => {
+        statusBar.items[2].status = undefined
+        if (openPythonStatusNotyf) notyf.dismiss(openPythonStatusNotyf)
+        openPythonStatusNotyf = notyf.open({
+            type: "warning",
+            message: "Backend services are restarting...",
+        })
+        });
+    }
+}