Merge pull request #862 from NeurodataWithoutBorders/fix-standalone-v…

…alidation

Fix standalone validation + progress bars

.github/workflows/daily_tests.yml

@@ -49,7 +49,7 @@ jobs:
           echo "ExampleDataCache: ${{ needs.ExampleDataCache.result }}"
           echo "ExampleDataTests: ${{ needs.ExampleDataTests.result }}"
       - name: debugging
-        run: export test=(${{ needs.DevTests.result }} == 'failure' || ${{ needs.LiveServices.result }} == 'failure' || ${{ needs.BuildTests.result }} == 'failure' || ${{ needs.ExampleDataCache.result }} == 'failure' || ${{ needs.ExampleDataTests.result }} == 'failure') 
+        run: export test=(${{ needs.DevTests.result }} == 'failure' || ${{ needs.LiveServices.result }} == 'failure' || ${{ needs.BuildTests.result }} == 'failure' || ${{ needs.ExampleDataCache.result }} == 'failure' || ${{ needs.ExampleDataTests.result }} == 'failure')
       - name: debugging print
         run: echo $test
       #- name: Printout logic trigger

src/electron/frontend/core/components/FileSystemSelector.js

@@ -53,6 +53,7 @@ const componentCSS = css`
     }
 
     button {
+        position: relative;
         background: WhiteSmoke;
         border: 1px solid #c3c3c3;
         border-radius: 4px;
@@ -78,7 +79,13 @@ const componentCSS = css`
         right: 0;
         bottom: 0;
         cursor: pointer;
-        padding: 0px 5px;
+        padding: 2px 5px;
+    }
+
+    nwb-list {
+        display: block;
+        width: 100%;
+        margin-top: 10px;
     }
 `;
 
@@ -272,6 +279,12 @@ export class FilesystemSelector extends LitElement {
                                             >Multiple directory support only available using drag-and-drop.</small
                                         >`
                                   : ""}`}
+
+                    <div class="controls">
+                        ${this.value
+                            ? html`<div @click=${() => this.#handleFiles()}>${unsafeSVG(restartSVG)}</div>`
+                            : ""}
+                    </div>
                 </button>
                 ${this.multiple && isArray && this.value.length > 1
                     ? new List({
@@ -282,10 +295,6 @@ export class FilesystemSelector extends LitElement {
                           },
                       })
                     : ""}
-
-                <div class="controls">
-                    ${this.value ? html`<div @click=${() => this.#handleFiles()}>${unsafeSVG(restartSVG)}</div>` : ""}
-                </div>
             </div>
             ${isMultipleTypes
                 ? html`<div id="button-div">

src/electron/frontend/core/components/InspectorList.js

@@ -4,29 +4,43 @@ import { getMessageType, isErrorImportance } from "../validation";
 
 import { unsafeHTML } from "lit/directives/unsafe-html.js";
 
+import { header } from "../../utils/text";
+
+const sortAlphabeticallyWithNumberedStrings = (a, b) => {
+    if (a === b) return 0;
+    return a.localeCompare(b, undefined, { numeric: true, sensitivity: "base" });
+};
+
 const sortList = (items) => {
     return items
-        .sort((a, b) => {
-            const aCritical = isErrorImportance.includes(a.importance);
-            const bCritical = isErrorImportance.includes(a.importance);
-            if (aCritical && bCritical) return 0;
-            else if (aCritical) return -1;
-            else return 1;
-        })
+        .sort((a, b) => sortAlphabeticallyWithNumberedStrings(a.object_name, b.object_name))
+        .sort((a, b) => sortAlphabeticallyWithNumberedStrings(a.object_type, b.object_type))
         .sort((a, b) => {
             const lowA = a.severity == "LOW";
             const lowB = b.severity === "LOW";
             if (lowA && lowB) return 0;
             else if (lowA) return 1;
             else return -1;
+        })
+
+        .sort((a, b) => {
+            const aCritical = isErrorImportance.includes(a.importance);
+            const bCritical = isErrorImportance.includes(b.importance);
+            if (aCritical && bCritical) return 0;
+            else if (aCritical) return -1;
+            else return 1;
         });
 };
 
 const aggregateMessages = (items) => {
     let messages = {};
+    console.log(items);
     items.forEach((item) => {
-        if (!messages[item.message]) messages[item.message] = [];
-        messages[item.message].push(item);
+        const copy = { ...item };
+        delete copy.file_path;
+        const encoded = JSON.stringify(copy);
+        if (!messages[encoded]) messages[encoded] = [];
+        messages[encoded].push(item);
     });
     return messages;
 };
@@ -55,7 +69,7 @@ export class InspectorList extends List {
             editable: false,
             unordered: true,
             ...props,
-            items: sortList(aggregatedItems).map((itemProps) => {
+            items: sortList(aggregatedItems).map((itemProps, i) => {
                 const item = new InspectorListItem(itemProps);
                 item.style.flexGrow = "1";
                 return { content: item };
@@ -94,7 +108,7 @@ export class InspectorListItem extends LitElement {
                 font-size: 10px;
             }
 
-            #objectType {
+            #header {
                 font-size: 10px;
             }
 
@@ -115,6 +129,10 @@ export class InspectorListItem extends LitElement {
                 border: 1px solid #ffeeba;
                 border-radius: 4px;
             }
+
+            small {
+                font-size: 10px;
+            }
         `;
     }
 
@@ -142,13 +160,14 @@ export class InspectorListItem extends LitElement {
         const isString = typeof this.message === "string";
         if (isString) this.setAttribute("title", this.message);
 
-        const hasObjectType = "object_type" in this;
-        const hasMetadata = hasObjectType && "object_name" in this;
+        const hasMetadata = "object_type" in this && "object_name" in this;
 
         const message = isString ? unsafeHTML(this.message) : this.message;
 
+        const headerText = this.object_name ? `${this.object_type} — ${header(this.object_name)}` : this.object_type;
+
         return html`
-            ${hasMetadata ? html`<span id="objectType">${hasObjectType ? `${this.object_type}` : ""} </span>` : ""}
+            ${hasMetadata ? html`<span id="header">${headerText}</span>` : ""}
             ${hasMetadata ? html`<span id="message">${message}</span>` : html`<p>${message}</p>`}
             ${this.file_path
                 ? html`<span id="filepath"

src/electron/frontend/core/components/NWBFilePreview.js

@@ -155,15 +155,11 @@ export class NWBFilePreview extends LitElement {
 
                                   const report = onlyFirstFile
                                       ? await run(
-                                            "neuroconv/inspect_file",
-                                            { nwbfile_path: fileArr[0].info.file, ...options },
+                                            "neuroconv/inspect",
+                                            { path: fileArr[0].info.file, ...options },
                                             { title }
                                         ) // Inspect the first file
-                                      : await run(
-                                            "neuroconv/inspect_folder",
-                                            { path, ...options },
-                                            { title: title + "s" }
-                                        ); // Inspect the folder
+                                      : await run("neuroconv/inspect", { path, ...options }, { title: title + "s" }); // Inspect the folder
 
                                   const result = onlyFirstFile
                                       ? {

src/electron/frontend/core/components/pages/guided-mode/options/GuidedInspectorPage.js

@@ -134,18 +134,27 @@ export class GuidedInspectorPage extends Page {
         return html`
             ${until(
                 (async () => {
-                    if (fileArr.length <= 1) {
-                        this.report = inspector;
+                    this.report = inspector;
+
+                    const oneFile = fileArr.length <= 1;
+
+                    const willRun = !this.report;
+                    const swalOpts = willRun ? await createProgressPopup({ title: oneFile ? title : `${title}s` }) : {};
 
+                    const { close: closeProgressPopup } = swalOpts;
+
+                    if (oneFile) {
                         if (!this.report) {
                             const result = await run(
-                                "neuroconv/inspect_file",
-                                { nwbfile_path: fileArr[0].info.file, ...options },
-                                { title }
-                            ).catch((error) => {
-                                this.notify(error.message, "error");
-                                return null;
-                            });
+                                "neuroconv/inspect",
+                                { path: fileArr[0].info.file, ...options, request_id: swalOpts.id },
+                                swalOpts
+                            )
+                                .catch((error) => {
+                                    this.notify(error.message, "error");
+                                    return null;
+                                })
+                                .finally(() => closeProgressPopup());
 
                             if (!result) return "Failed to generate inspector report.";
 
@@ -170,14 +179,9 @@ export class GuidedInspectorPage extends Page {
 
                     const path = getSharedPath(fileArr.map(({ info }) => info.file));
 
-                    this.report = inspector;
                     if (!this.report) {
-                        const swalOpts = await createProgressPopup({ title: `${title}s` });
-
-                        const { close: closeProgressPopup } = swalOpts;
-
                         const result = await run(
-                            "neuroconv/inspect_folder",
+                            "neuroconv/inspect",
                             { path, ...options, request_id: swalOpts.id },
                             swalOpts
                         )

src/electron/frontend/core/components/pages/inspect/InspectPage.js

@@ -39,7 +39,9 @@ export class InspectPage extends Page {
             }
         );
 
-        await closeProgressPopup();
+        console.log(result);
+
+        closeProgressPopup();
 
         if (typeof result === "string") return result;
 

src/electron/frontend/utils/popups.ts

@@ -33,14 +33,15 @@ export const createProgressPopup = async (
     options: SweetAlertOptions,
     tqdmCallback: (update: any) => void
 ) => {
+
     const cancelController = new AbortController();
 
     if (!("showCancelButton" in options)) {
         options.showCancelButton = true;
         options.customClass = { actions: "swal-conversion-actions" };
     }
 
-    const popup = await openProgressSwal(options, (result) => {
+    await openProgressSwal(options, (result) => {
         if (!result.isConfirmed) cancelController.abort();
     });
 
@@ -62,6 +63,7 @@ export const createProgressPopup = async (
         width: "100%",
         gap: "5px",
     });
+
     element.append(container);
 
     const bars: Record<string | symbol, ProgressBar> = {};
@@ -84,7 +86,7 @@ export const createProgressPopup = async (
 
     elements.bars = bars;
 
-    const commonReturnValue = { swal: popup, fetch: { signal: cancelController.signal }, elements, ...options };
+    const commonReturnValue = { swal: false, fetch: { signal: cancelController.signal }, elements, ...options };
 
     // Provide a default callback
     let lastUpdate: number;

src/pyflask/manageNeuroconv/__init__.py

@@ -11,9 +11,7 @@
     get_interface_alignment,
     get_metadata_schema,
     get_source_schema,
-    inspect_multiple_filesystem_objects,
-    inspect_nwb_file,
-    inspect_nwb_folder,
+    inspect_all,
     listen_to_neuroconv_progress_events,
     locate_data,
     progress_handler,

src/pyflask/manageNeuroconv/manage_neuroconv.py

@@ -1391,32 +1391,6 @@ def generate_dataset(input_path: str, output_path: str) -> dict:
     return {"output_path": str(output_path)}
 
 
-def inspect_nwb_file(payload) -> dict:
-    from nwbinspector import inspect_nwbfile, load_config
-    from nwbinspector.inspector_tools import format_messages, get_report_header
-    from nwbinspector.nwbinspector import InspectorOutputJSONEncoder
-
-    messages = list(
-        inspect_nwbfile(
-            ignore=[
-                "check_description",
-                "check_data_orientation",
-            ],  # TODO: remove when metadata control is exposed
-            config=load_config(filepath_or_keyword="dandi"),
-            **payload,
-        )
-    )
-
-    if payload.get("format") == "text":
-        return "\n".join(format_messages(messages=messages))
-
-    header = get_report_header()
-    header["NWBInspector_version"] = str(header["NWBInspector_version"])
-    json_report = dict(header=header, messages=messages, text="\n".join(format_messages(messages=messages)))
-
-    return json.loads(json.dumps(obj=json_report, cls=InspectorOutputJSONEncoder))
-
-
 def _inspect_file_per_job(
     nwbfile_path: str,
     url,
@@ -1457,17 +1431,24 @@ def _inspect_file_per_job(
         return messages
 
 
-def inspect_all(url, config):
+def _inspect_all(url, config):
 
     from concurrent.futures import ProcessPoolExecutor, as_completed
 
     from nwbinspector.utils import calculate_number_of_cpu
     from tqdm_publisher import TQDMProgressSubscriber
 
-    path = config["path"]
-    config.pop("path")
+    path = config.pop("path", None)
+
+    paths = [path] if path else config.pop("paths", [])
 
-    nwbfile_paths = list(Path(path).rglob("*.nwb"))
+    nwbfile_paths = []
+    for path in paths:
+        posix_path = Path(path)
+        if posix_path.is_file():
+            nwbfile_paths.append(posix_path)
+        else:
+            nwbfile_paths.extend(list(posix_path.rglob("*.nwb")))
 
     request_id = config.pop("request_id", None)
 
@@ -1518,18 +1499,18 @@ def on_progress_update(message):
     return messages
 
 
-def inspect_nwb_folder(url, payload) -> dict:
+def inspect_all(url, payload) -> dict:
     from pickle import PicklingError
 
     from nwbinspector.inspector_tools import format_messages, get_report_header
     from nwbinspector.nwbinspector import InspectorOutputJSONEncoder
 
     try:
-        messages = inspect_all(url, payload)
+        messages = _inspect_all(url, payload)
     except PicklingError as exception:
         if "attribute lookup auto_parse_some_output on nwbinspector.register_checks failed" in str(exception):
             del payload["n_jobs"]
-            messages = inspect_all(url, payload)
+            messages = _inspect_all(url, payload)
         else:
             raise exception
     except Exception as exception:
@@ -1561,13 +1542,6 @@ def _aggregate_symlinks_in_new_directory(paths, reason="", folder_path=None) ->
     return folder_path
 
 
-def inspect_multiple_filesystem_objects(url, paths, **kwargs) -> dict:
-    tmp_folder_path = _aggregate_symlinks_in_new_directory(paths, "inspect")
-    result = inspect_nwb_folder(url, {"path": tmp_folder_path, **kwargs})
-    rmtree(tmp_folder_path)
-    return result
-
-
 def _format_spikeglx_meta_file(bin_file_path: str) -> str:
     bin_file_path = Path(bin_file_path)