diff --git a/.github/workflows/daily_tests.yml b/.github/workflows/daily_tests.yml index f3e2d5596..4bc2c2c4c 100644 --- a/.github/workflows/daily_tests.yml +++ b/.github/workflows/daily_tests.yml @@ -39,21 +39,8 @@ jobs: NotifyOnAnyFailure: runs-on: ubuntu-latest needs: [DevTests, LiveServices, BuildTests, ExampleDataCache, ExampleDataTests] - if: always() && (${{ needs.DevTests.result }} == 'failure' || ${{ needs.LiveServices.result }} == 'failure' || ${{ needs.BuildTests.result }} == 'failure' || ${{ needs.ExampleDataCache.result }} == 'failure' || ${{ needs.ExampleDataTests.result }} == 'failure') + if: failure() steps: - - name: Printout which failed - run: | - echo "DevTests: ${{ needs.DevTests.result }}" - echo "LiveServices: ${{ needs.LiveServices.result }}" - echo "BuildTests: ${{ needs.BuildTests.result }}" - echo "ExampleDataCache: ${{ needs.ExampleDataCache.result }}" - echo "ExampleDataTests: ${{ needs.ExampleDataTests.result }}" - - name: debugging - run: export test=(${{ needs.DevTests.result }} == 'failure' || ${{ needs.LiveServices.result }} == 'failure' || ${{ needs.BuildTests.result }} == 'failure' || ${{ needs.ExampleDataCache.result }} == 'failure' || ${{ needs.ExampleDataTests.result }} == 'failure') - - name: debugging print - run: echo $test - #- name: Printout logic trigger - # run: ${{ needs.DevTests.result }} == 'failure' || ${{ needs.LiveServices.result }} == 'failure' || ${{ needs.BuildTests.result }} == 'failure' || ${{ needs.ExampleDataCache.result }} == 'failure' || ${{ needs.ExampleDataTests.result }} == 'failure' - uses: dawidd6/action-send-mail@v3 with: server_address: smtp.gmail.com diff --git a/src/electron/frontend/core/components/FileSystemSelector.js b/src/electron/frontend/core/components/FileSystemSelector.js index da0c10140..d9d444d8e 100644 --- a/src/electron/frontend/core/components/FileSystemSelector.js +++ b/src/electron/frontend/core/components/FileSystemSelector.js @@ -53,6 +53,7 @@ const componentCSS = css` } button { + position: relative; background: WhiteSmoke; border: 1px solid #c3c3c3; border-radius: 4px; @@ -78,7 +79,13 @@ const componentCSS = css` right: 0; bottom: 0; cursor: pointer; - padding: 0px 5px; + padding: 2px 5px; + } + + nwb-list { + display: block; + width: 100%; + margin-top: 10px; } `; @@ -272,6 +279,12 @@ export class FilesystemSelector extends LitElement { >Multiple directory support only available using drag-and-drop.` : ""}`} + +
+ ${this.value + ? html`
this.#handleFiles()}>${unsafeSVG(restartSVG)}
` + : ""} +
${this.multiple && isArray && this.value.length > 1 ? new List({ @@ -282,10 +295,6 @@ export class FilesystemSelector extends LitElement { }, }) : ""} - -
- ${this.value ? html`
this.#handleFiles()}>${unsafeSVG(restartSVG)}
` : ""} -
${isMultipleTypes ? html`
diff --git a/src/electron/frontend/core/components/InspectorList.js b/src/electron/frontend/core/components/InspectorList.js index 4e5dc56bc..5af3ac696 100644 --- a/src/electron/frontend/core/components/InspectorList.js +++ b/src/electron/frontend/core/components/InspectorList.js @@ -4,29 +4,43 @@ import { getMessageType, isErrorImportance } from "../validation"; import { unsafeHTML } from "lit/directives/unsafe-html.js"; +import { header } from "../../utils/text"; + +const sortAlphabeticallyWithNumberedStrings = (a, b) => { + if (a === b) return 0; + return a.localeCompare(b, undefined, { numeric: true, sensitivity: "base" }); +}; + const sortList = (items) => { return items - .sort((a, b) => { - const aCritical = isErrorImportance.includes(a.importance); - const bCritical = isErrorImportance.includes(a.importance); - if (aCritical && bCritical) return 0; - else if (aCritical) return -1; - else return 1; - }) + .sort((a, b) => sortAlphabeticallyWithNumberedStrings(a.object_name, b.object_name)) + .sort((a, b) => sortAlphabeticallyWithNumberedStrings(a.object_type, b.object_type)) .sort((a, b) => { const lowA = a.severity == "LOW"; const lowB = b.severity === "LOW"; if (lowA && lowB) return 0; else if (lowA) return 1; else return -1; + }) + + .sort((a, b) => { + const aCritical = isErrorImportance.includes(a.importance); + const bCritical = isErrorImportance.includes(b.importance); + if (aCritical && bCritical) return 0; + else if (aCritical) return -1; + else return 1; }); }; const aggregateMessages = (items) => { let messages = {}; + console.log(items); items.forEach((item) => { - if (!messages[item.message]) messages[item.message] = []; - messages[item.message].push(item); + const copy = { ...item }; + delete copy.file_path; + const encoded = JSON.stringify(copy); + if (!messages[encoded]) messages[encoded] = []; + messages[encoded].push(item); }); return messages; }; @@ -55,7 +69,7 @@ export class InspectorList extends List { editable: false, unordered: true, ...props, - items: sortList(aggregatedItems).map((itemProps) => { + items: sortList(aggregatedItems).map((itemProps, i) => { const item = new InspectorListItem(itemProps); item.style.flexGrow = "1"; return { content: item }; @@ -94,7 +108,7 @@ export class InspectorListItem extends LitElement { font-size: 10px; } - #objectType { + #header { font-size: 10px; } @@ -115,6 +129,10 @@ export class InspectorListItem extends LitElement { border: 1px solid #ffeeba; border-radius: 4px; } + + small { + font-size: 10px; + } `; } @@ -142,13 +160,14 @@ export class InspectorListItem extends LitElement { const isString = typeof this.message === "string"; if (isString) this.setAttribute("title", this.message); - const hasObjectType = "object_type" in this; - const hasMetadata = hasObjectType && "object_name" in this; + const hasMetadata = "object_type" in this && "object_name" in this; const message = isString ? unsafeHTML(this.message) : this.message; + const headerText = this.object_name ? `${this.object_type} — ${header(this.object_name)}` : this.object_type; + return html` - ${hasMetadata ? html`${hasObjectType ? `${this.object_type}` : ""} ` : ""} + ${hasMetadata ? html`${headerText}` : ""} ${hasMetadata ? html`${message}` : html`

${message}

`} ${this.file_path ? html` { - if (fileArr.length <= 1) { - this.report = inspector; + this.report = inspector; + + const oneFile = fileArr.length <= 1; + + const willRun = !this.report; + const swalOpts = willRun ? await createProgressPopup({ title: oneFile ? title : `${title}s` }) : {}; + const { close: closeProgressPopup } = swalOpts; + + if (oneFile) { if (!this.report) { const result = await run( - "neuroconv/inspect_file", - { nwbfile_path: fileArr[0].info.file, ...options }, - { title } - ).catch((error) => { - this.notify(error.message, "error"); - return null; - }); + "neuroconv/inspect", + { path: fileArr[0].info.file, ...options, request_id: swalOpts.id }, + swalOpts + ) + .catch((error) => { + this.notify(error.message, "error"); + return null; + }) + .finally(() => closeProgressPopup()); if (!result) return "Failed to generate inspector report."; @@ -170,14 +179,9 @@ export class GuidedInspectorPage extends Page { const path = getSharedPath(fileArr.map(({ info }) => info.file)); - this.report = inspector; if (!this.report) { - const swalOpts = await createProgressPopup({ title: `${title}s` }); - - const { close: closeProgressPopup } = swalOpts; - const result = await run( - "neuroconv/inspect_folder", + "neuroconv/inspect", { path, ...options, request_id: swalOpts.id }, swalOpts ) diff --git a/src/electron/frontend/core/components/pages/inspect/InspectPage.js b/src/electron/frontend/core/components/pages/inspect/InspectPage.js index 0b645209f..e711673b3 100644 --- a/src/electron/frontend/core/components/pages/inspect/InspectPage.js +++ b/src/electron/frontend/core/components/pages/inspect/InspectPage.js @@ -39,7 +39,9 @@ export class InspectPage extends Page { } ); - await closeProgressPopup(); + console.log(result); + + closeProgressPopup(); if (typeof result === "string") return result; diff --git a/src/electron/frontend/utils/popups.ts b/src/electron/frontend/utils/popups.ts index 396e2ccc8..d7584704c 100644 --- a/src/electron/frontend/utils/popups.ts +++ b/src/electron/frontend/utils/popups.ts @@ -33,6 +33,7 @@ export const createProgressPopup = async ( options: SweetAlertOptions, tqdmCallback: (update: any) => void ) => { + const cancelController = new AbortController(); if (!("showCancelButton" in options)) { @@ -40,7 +41,7 @@ export const createProgressPopup = async ( options.customClass = { actions: "swal-conversion-actions" }; } - const popup = await openProgressSwal(options, (result) => { + await openProgressSwal(options, (result) => { if (!result.isConfirmed) cancelController.abort(); }); @@ -62,6 +63,7 @@ export const createProgressPopup = async ( width: "100%", gap: "5px", }); + element.append(container); const bars: Record = {}; @@ -84,7 +86,7 @@ export const createProgressPopup = async ( elements.bars = bars; - const commonReturnValue = { swal: popup, fetch: { signal: cancelController.signal }, elements, ...options }; + const commonReturnValue = { swal: false, fetch: { signal: cancelController.signal }, elements, ...options }; // Provide a default callback let lastUpdate: number; diff --git a/src/pyflask/manageNeuroconv/__init__.py b/src/pyflask/manageNeuroconv/__init__.py index 8401b0f37..52de88e2e 100644 --- a/src/pyflask/manageNeuroconv/__init__.py +++ b/src/pyflask/manageNeuroconv/__init__.py @@ -11,9 +11,7 @@ get_interface_alignment, get_metadata_schema, get_source_schema, - inspect_multiple_filesystem_objects, - inspect_nwb_file, - inspect_nwb_folder, + inspect_all, listen_to_neuroconv_progress_events, locate_data, progress_handler, diff --git a/src/pyflask/manageNeuroconv/manage_neuroconv.py b/src/pyflask/manageNeuroconv/manage_neuroconv.py index 073164535..007491060 100644 --- a/src/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/src/pyflask/manageNeuroconv/manage_neuroconv.py @@ -1391,32 +1391,6 @@ def generate_dataset(input_path: str, output_path: str) -> dict: return {"output_path": str(output_path)} -def inspect_nwb_file(payload) -> dict: - from nwbinspector import inspect_nwbfile, load_config - from nwbinspector.inspector_tools import format_messages, get_report_header - from nwbinspector.nwbinspector import InspectorOutputJSONEncoder - - messages = list( - inspect_nwbfile( - ignore=[ - "check_description", - "check_data_orientation", - ], # TODO: remove when metadata control is exposed - config=load_config(filepath_or_keyword="dandi"), - **payload, - ) - ) - - if payload.get("format") == "text": - return "\n".join(format_messages(messages=messages)) - - header = get_report_header() - header["NWBInspector_version"] = str(header["NWBInspector_version"]) - json_report = dict(header=header, messages=messages, text="\n".join(format_messages(messages=messages))) - - return json.loads(json.dumps(obj=json_report, cls=InspectorOutputJSONEncoder)) - - def _inspect_file_per_job( nwbfile_path: str, url, @@ -1457,17 +1431,24 @@ def _inspect_file_per_job( return messages -def inspect_all(url, config): +def _inspect_all(url, config): from concurrent.futures import ProcessPoolExecutor, as_completed from nwbinspector.utils import calculate_number_of_cpu from tqdm_publisher import TQDMProgressSubscriber - path = config["path"] - config.pop("path") + path = config.pop("path", None) + + paths = [path] if path else config.pop("paths", []) - nwbfile_paths = list(Path(path).rglob("*.nwb")) + nwbfile_paths = [] + for path in paths: + posix_path = Path(path) + if posix_path.is_file(): + nwbfile_paths.append(posix_path) + else: + nwbfile_paths.extend(list(posix_path.rglob("*.nwb"))) request_id = config.pop("request_id", None) @@ -1518,18 +1499,18 @@ def on_progress_update(message): return messages -def inspect_nwb_folder(url, payload) -> dict: +def inspect_all(url, payload) -> dict: from pickle import PicklingError from nwbinspector.inspector_tools import format_messages, get_report_header from nwbinspector.nwbinspector import InspectorOutputJSONEncoder try: - messages = inspect_all(url, payload) + messages = _inspect_all(url, payload) except PicklingError as exception: if "attribute lookup auto_parse_some_output on nwbinspector.register_checks failed" in str(exception): del payload["n_jobs"] - messages = inspect_all(url, payload) + messages = _inspect_all(url, payload) else: raise exception except Exception as exception: @@ -1561,13 +1542,6 @@ def _aggregate_symlinks_in_new_directory(paths, reason="", folder_path=None) -> return folder_path -def inspect_multiple_filesystem_objects(url, paths, **kwargs) -> dict: - tmp_folder_path = _aggregate_symlinks_in_new_directory(paths, "inspect") - result = inspect_nwb_folder(url, {"path": tmp_folder_path, **kwargs}) - rmtree(tmp_folder_path) - return result - - def _format_spikeglx_meta_file(bin_file_path: str) -> str: bin_file_path = Path(bin_file_path) diff --git a/src/pyflask/namespaces/neuroconv.py b/src/pyflask/namespaces/neuroconv.py index fc3fba981..4312d427b 100644 --- a/src/pyflask/namespaces/neuroconv.py +++ b/src/pyflask/namespaces/neuroconv.py @@ -11,9 +11,7 @@ get_interface_alignment, get_metadata_schema, get_source_schema, - inspect_multiple_filesystem_objects, - inspect_nwb_file, - inspect_nwb_folder, + inspect_all, listen_to_neuroconv_progress_events, locate_data, progress_handler, @@ -164,21 +162,6 @@ def post(self): return upload_multiple_filesystem_objects_to_dandi(**neuroconv_namespace.payload) -@neuroconv_namespace.route("/inspect_file") -class InspectNWBFile(Resource): - @neuroconv_namespace.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"}) - def post(self): - return inspect_nwb_file(neuroconv_namespace.payload) - - -@neuroconv_namespace.route("/inspect_folder") -class InspectNWBFolder(Resource): - @neuroconv_namespace.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"}) - def post(self): - url = f"{request.url_root}neuroconv/announce/progress" - return inspect_nwb_folder(url, neuroconv_namespace.payload) - - @neuroconv_namespace.route("/announce/progress") class InspectNWBFolder(Resource): @neuroconv_namespace.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"}) @@ -192,23 +175,8 @@ def post(self): class InspectNWBFolder(Resource): @neuroconv_namespace.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"}) def post(self): - from os.path import isfile - url = f"{request.url_root}neuroconv/announce/progress" - - paths = neuroconv_namespace.payload["paths"] - - kwargs = {**neuroconv_namespace.payload} - del kwargs["paths"] - - if len(paths) == 1: - if isfile(paths[0]): - return inspect_nwb_file({"nwbfile_path": paths[0], **kwargs}) - else: - return inspect_nwb_folder(url, {"path": paths[0], **kwargs}) - - else: - return inspect_multiple_filesystem_objects(url, paths, **kwargs) + return inspect_all(url, neuroconv_namespace.payload) @neuroconv_namespace.route("/html")