Merge pull request #778 from NeurodataWithoutBorders/hdmf-progress-bars

Add Conversion Progress Bars

src/electron/frontend/core/components/ProgressBar.ts

@@ -126,8 +126,10 @@ export class ProgressBar extends LitElement {
 
     render() {
 
-        const percent = this.format.total ? 100 * (this.format.n / this.format.total) : 0;
-        const remaining = this.format.rate && this.format.total ? (this.format.total - this.format.n) / this.format.rate : 0; // Seconds
+        const n = this.format.n > this.format.total ? this.format.total : this.format.n
+        const ratio = n/ this.format.total;
+        const percent = this.format.total ? 100 * ratio : 0;
+        const remaining = this.format.rate && this.format.total ? (this.format.total - n) / this.format.rate : 0; // Seconds
 
         const numerator = this.isBytes ? humanReadableBytes(this.format.n) : this.format.n
         const denominator = this.isBytes ? humanReadableBytes(this.format.total) : this.format.total

src/electron/frontend/core/components/pages/Page.js

@@ -1,5 +1,5 @@
 import { LitElement, html } from "lit";
-import { runConversion } from "./guided-mode/options/utils.js";
+import { run } from "./guided-mode/options/utils.js";
 import { get, save } from "../../progress/index.js";
 
 import { dismissNotification, notify } from "../../dependencies.js";
@@ -136,13 +136,10 @@ export class Page extends LitElement {
         this.unsavedUpdates = true;
 
         // Indicate conversion has run successfully
-        const { desyncedData } = this.info.globalState;
-        if (!desyncedData) this.info.globalState.desyncedData = {};
-
-        if (desyncedData) {
-            desyncedData[key] = false;
-            await this.save({}, false);
-        }
+        let { desyncedData } = this.info.globalState;
+        if (!desyncedData) desyncedData = this.info.globalState.desyncedData = {};
+        desyncedData[key] = false;
+        await this.save({}, false);
     }
 
     async runConversions(conversionOptions = {}, toRun, options = {}) {
@@ -157,18 +154,10 @@ export class Page extends LitElement {
 
         const results = {};
 
-        const isMultiple = toRun.length > 1;
-
         const swalOpts = await createProgressPopup({ title: `Running conversion`, ...options });
-        const { close: closeProgressPopup, elements } = swalOpts;
-
-        elements.container.insertAdjacentHTML(
-            "beforeend",
-            `<small><small><b>Note:</b> This may take a while to complete...</small></small><hr style="margin-bottom: 0;">`
-        );
 
-        let completed = 0;
-        elements.progress.format = { n: completed, total: toRun.length };
+        const { close: closeProgressPopup } = swalOpts;
+        const fileConfiguration = [];
 
         for (let info of toRun) {
             const { subject, session, globalState = this.info.globalState } = info;
@@ -187,44 +176,54 @@ export class Page extends LitElement {
                 source_data: merge(SourceData, sourceDataCopy),
             };
 
-            const result = await runConversion(
-                {
-                    output_folder: conversionOptions.stub_test ? undefined : conversion_output_folder,
-                    project_name: name,
-                    nwbfile_path: file,
-                    overwrite: true, // We assume override is true because the native NWB file dialog will not allow the user to select an existing file (unless they approve the overwrite)
-                    ...sessionInfo, // source_data and metadata are passed in here
-                    ...conversionOptions, // Any additional conversion options override the defaults
-
-                    interfaces: globalState.interfaces,
-                    alignment,
-                },
-                swalOpts
-            ).catch((error) => {
-                let message = error.message;
-
-                if (message.includes("The user aborted a request.")) {
-                    this.notify("Conversion was cancelled.", "warning");
-                    throw error;
-                }
+            const payload = {
+                output_folder: conversionOptions.stub_test ? undefined : conversion_output_folder,
+                project_name: name,
+                nwbfile_path: file,
+                overwrite: true, // We assume override is true because the native NWB file dialog will not allow the user to select an existing file (unless they approve the overwrite)
+                ...sessionInfo, // source_data and metadata are passed in here
+                ...conversionOptions, // Any additional conversion options override the defaults
 
-                this.notify(message, "error");
-                closeProgressPopup();
-                throw error;
-            });
+                interfaces: globalState.interfaces,
+            };
+
+            fileConfiguration.push(payload);
+        }
+
+        const conversionResults = await run(
+            `neuroconv/convert`,
+            {
+                files: fileConfiguration,
+                max_workers: 2, // TODO: Make this configurable and confirm default value
+                request_id: swalOpts.id,
+            },
+            {
+                title: "Running the conversion",
+                onError: () => "Conversion failed with current metadata. Please try again.",
+                ...swalOpts,
+            }
+        ).catch(async (error) => {
+            let message = error.message;
 
-            completed++;
-            if (isMultiple) {
-                const progressInfo = { n: completed, total: toRun.length };
-                elements.progress.format = progressInfo;
+            if (message.includes("The user aborted a request.")) {
+                this.notify("Conversion was cancelled.", "warning");
+                throw error;
             }
 
+            this.notify(message, "error");
+            await closeProgressPopup();
+            throw error;
+        });
+
+        conversionResults.forEach((info) => {
+            const { file } = info;
+            const fileName = file.split("/").pop();
+            const [subject, session] = fileName.match(/sub-(.+)_ses-(.+)\.nwb/).slice(1);
             const subRef = results[subject] ?? (results[subject] = {});
-            subRef[session] = result;
-        }
+            subRef[session] = info;
+        });
 
-        closeProgressPopup();
-        elements.container.style.textAlign = ""; // Clear style update
+        await closeProgressPopup();
 
         return results;
     }

src/electron/frontend/core/components/pages/guided-mode/options/GuidedInspectorPage.js

@@ -8,13 +8,13 @@ import { electron } from "../../../../../utils/electron.js";
 import { getSharedPath, removeFilePaths, truncateFilePaths } from "../../../preview/NWBFilePreview.js";
 const { ipcRenderer } = electron;
 import { until } from "lit/directives/until.js";
-import { run } from "./utils";
+import { run } from "./utils.js";
 import { InspectorList, InspectorLegend } from "../../../preview/inspector/InspectorList.js";
 import { getStubArray } from "./GuidedStubPreview.js";
 import { InstanceManager } from "../../../InstanceManager.js";
 import { getMessageType } from "../../../../validation/index.js";
 
-import { Button } from "../../../Button.js";
+import { Button } from "../../../Button";
 
 import { download } from "../../inspect/utils.js";
 import { createProgressPopup } from "../../../utils/progress.js";
@@ -184,16 +184,15 @@ export class GuidedInspectorPage extends Page {
                             "neuroconv/inspect_folder",
                             { path, ...options, request_id: swalOpts.id },
                             swalOpts
-                        ).catch((error) => {
-                            this.notify(error.message, "error");
-                            closeProgressPopup();
-                            return null;
-                        });
+                        )
+                            .catch(async (error) => {
+                                this.notify(error.message, "error");
+                                return null;
+                            })
+                            .finally(() => closeProgressPopup());
 
                         if (!result) return "Failed to generate inspector report.";
 
-                        closeProgressPopup();
-
                         this.report = globalState.preview.inspector = {
                             ...result,
                             messages: truncateFilePaths(result.messages, path),

src/electron/frontend/core/components/pages/inspect/InspectPage.js

@@ -4,7 +4,6 @@ import { onThrow } from "../../../errors";
 import { Button } from "../../Button.js";
 
 import { run } from "../guided-mode/options/utils.js";
-import { JSONSchemaInput } from "../../JSONSchemaInput.js";
 import { Modal } from "../../Modal";
 import { getSharedPath, truncateFilePaths } from "../../preview/NWBFilePreview.js";
 import { InspectorList, InspectorLegend } from "../../preview/inspector/InspectorList.js";
@@ -31,16 +30,15 @@ export class InspectPage extends Page {
         });
 
         const { close: closeProgressPopup } = swalOpts;
-
         const result = await run("neuroconv/inspect", { request_id: swalOpts.id, paths, ...kwargs }, swalOpts).catch(
-            (error) => {
+            async (error) => {
                 this.notify(error.message, "error");
-                closeProgressPopup();
+                await closeProgressPopup();
                 throw error;
             }
         );
 
-        closeProgressPopup();
+        await closeProgressPopup();
 
         if (typeof result === "string") return result;
 

src/electron/frontend/core/components/pages/utils.js

@@ -35,7 +35,7 @@ export const sanitize = (item, condition = isPrivate) => {
             if (condition(k, value)) delete item[k];
             else sanitize(value, condition);
         }
-    }
+    } else if (Array.isArray(item)) item.forEach((value) => sanitize(value, condition));
 
     return item;
 };

src/electron/frontend/core/components/utils/progress.js

@@ -29,6 +29,8 @@ export const createProgressPopup = async (options, tqdmCallback) => {
         textAlign: "left",
         display: "flex",
         flexDirection: "column",
+        overflow: "hidden",
+        width: "100%",
         gap: "5px",
     });
     element.append(container);
@@ -74,13 +76,16 @@ export const createProgressPopup = async (options, tqdmCallback) => {
 
     progressHandler.addEventListener("message", onProgressMessage);
 
-    const close = () => {
+    const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+
+    const close = async () => {
         if (lastUpdate) {
             // const timeSinceLastUpdate = now - lastUpdate;
             const animationLeft = 1000; // ProgressBar.animationDuration - timeSinceLastUpdate; // Add 100ms to ensure the animation has time to complete
-            if (animationLeft) setTimeout(() => popup.close(), animationLeft);
-            else popup.close();
-        } else popup.close();
+            if (animationLeft) await sleep(animationLeft);
+        }
+
+        popup.close();
 
         progressHandler.removeEventListener("message", onProgressMessage);
     };

src/pyflask/manageNeuroconv/__init__.py

@@ -1,6 +1,7 @@
 from .info import CONVERSION_SAVE_FOLDER_PATH, STUB_SAVE_FOLDER_PATH
 from .manage_neuroconv import (
     autocomplete_format_string,
+    convert_all_to_nwb,
     convert_to_nwb,
     generate_dataset,
     generate_test_data,