From af178c5089e729fca8a7f7386eb23de99962b2ce Mon Sep 17 00:00:00 2001 From: Garrett Michael Flynn Date: Thu, 2 May 2024 15:08:41 -0700 Subject: [PATCH 01/25] Correctly handle converter electrodes --- pyflask/manageNeuroconv/manage_neuroconv.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index a0aec323c..28a8b6763 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -364,13 +364,19 @@ def get_source_schema(interface_class_dict: dict) -> dict: return CustomNWBConverter.get_source_schema() -def map_interfaces(BaseRecordingExtractorInterface, callback, converter): +def map_interfaces(callback, converter, to_match=None, parent_name=None): + from neuroconv import NWBConverter output = [] for name, interface in converter.data_interface_objects.items(): - if isinstance(interface, BaseRecordingExtractorInterface): - result = callback(name, interface) + + associated_name = f"{parent_name} — {name}" if parent_name else name + if isinstance(interface, NWBConverter): + result = map_interfaces(callback, interface, to_match, associated_name) + output.extend(result) + elif to_match == None or isinstance(interface, to_match): + result = callback(associated_name, interface) output.append(result) return output @@ -513,10 +519,10 @@ def on_recording_interface(name, recording_interface): from neuroconv.datainterfaces.ecephys.basesortingextractorinterface import BaseSortingExtractorInterface # Map recording interfaces to metadata - map_interfaces(BaseRecordingExtractorInterface, on_recording_interface, converter) + map_interfaces(on_recording_interface, converter, BaseRecordingExtractorInterface) # Map sorting interfaces to metadata - map_interfaces(BaseSortingExtractorInterface, on_sorting_interface, converter) + map_interfaces(on_sorting_interface, converter, BaseSortingExtractorInterface) # Delete Ecephys metadata if no interfaces processed if has_ecephys: @@ -749,7 +755,10 @@ def update_conversion_progress(**kwargs): shared_electrode_columns = ecephys_metadata["ElectrodeColumns"] for interface_name, interface_electrode_results in ecephys_metadata["Electrodes"].items(): - interface = converter.data_interface_objects[interface_name] + + interface = converter + for sub_interface in interface_name.split(" — "): + interface = interface.data_interface_objects[sub_interface] update_recording_properties_from_table_as_json( interface, From 58724f46ddc9870c37c50b42256812f5e89290f3 Mon Sep 17 00:00:00 2001 From: Garrett Michael Flynn Date: Mon, 13 May 2024 13:37:31 -0700 Subject: [PATCH 02/25] Update guide_testing_suite.yml --- guide_testing_suite.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/guide_testing_suite.yml b/guide_testing_suite.yml index 977185a70..c725faff6 100644 --- a/guide_testing_suite.yml +++ b/guide_testing_suite.yml @@ -13,11 +13,20 @@ pipelines: file_path: ephy_testing_data/spikeglx/Noise4Sam_g0/Noise4Sam_g0_imec0/Noise4Sam_g0_t0.imec0.ap.bin PhySortingInterface: folder_path: ephy_testing_data/phy/phy_example_0 + SpikeGLX_v1_SingleProbe_AP: SpikeGLXRecordingInterface: file_path: ephy_testing_data/spikeglx/Noise4Sam_g0/Noise4Sam_g0_imec0/Noise4Sam_g0_t0.imec0.ap.bin + + SpikeGLXConverter-Phy: + SpikeGLXConverter: + file_path: ephy_testing_data/spikeglx/Noise4Sam_g0 + PhySortingInterface: + folder_path: ephy_testing_data/phy/phy_example_0 + + Suite2P_SinglePlane_SingleChannel: Suite2pSegmentationInterface: folder_path: ophys_testing_data/segmentation_datasets/suite2p From 21cee64e2bd23135a558a614523622a2d42fe7f7 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 13 May 2024 20:37:48 +0000 Subject: [PATCH 03/25] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- guide_testing_suite.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/guide_testing_suite.yml b/guide_testing_suite.yml index c725faff6..8fbaffb39 100644 --- a/guide_testing_suite.yml +++ b/guide_testing_suite.yml @@ -13,7 +13,7 @@ pipelines: file_path: ephy_testing_data/spikeglx/Noise4Sam_g0/Noise4Sam_g0_imec0/Noise4Sam_g0_t0.imec0.ap.bin PhySortingInterface: folder_path: ephy_testing_data/phy/phy_example_0 - + SpikeGLX_v1_SingleProbe_AP: SpikeGLXRecordingInterface: From 6dc6b4c296d7f4e799e6fcfd06822593c72d14e6 Mon Sep 17 00:00:00 2001 From: Garrett Michael Flynn Date: Mon, 13 May 2024 13:41:08 -0700 Subject: [PATCH 04/25] Update guide_testing_suite.yml --- guide_testing_suite.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/guide_testing_suite.yml b/guide_testing_suite.yml index c725faff6..631f438bf 100644 --- a/guide_testing_suite.yml +++ b/guide_testing_suite.yml @@ -21,8 +21,8 @@ pipelines: SpikeGLXConverter-Phy: - SpikeGLXConverter: - file_path: ephy_testing_data/spikeglx/Noise4Sam_g0 + SpikeGLXConverterPipe: + folder_path: ephy_testing_data/spikeglx/Noise4Sam_g0 PhySortingInterface: folder_path: ephy_testing_data/phy/phy_example_0 From a2ddeb99a9b70ffbcd4162940837883706c207a3 Mon Sep 17 00:00:00 2001 From: Garrett Michael Flynn Date: Wed, 15 May 2024 09:07:08 -0700 Subject: [PATCH 05/25] Limit file extension selections --- schemas/source-data.schema.ts | 6 ++++++ src/renderer/src/stories/FileSystemSelector.js | 7 +++++++ src/renderer/src/stories/JSONSchemaInput.js | 1 + 3 files changed, 14 insertions(+) diff --git a/schemas/source-data.schema.ts b/schemas/source-data.schema.ts index ef189b856..785d26910 100644 --- a/schemas/source-data.schema.ts +++ b/schemas/source-data.schema.ts @@ -13,6 +13,7 @@ export default function preprocessSourceDataSchema (schema) { const info = interfaces[key] ?? {} + const files = schema.properties.file_paths ?? schema.properties.file_path const singleLocationInfo = schema.properties.file_path ?? schema.properties.folder_path if (schema.properties.file_paths) { @@ -29,6 +30,11 @@ export default function preprocessSourceDataSchema (schema) { } + if (files) { + const base = singleLocationInfo ? files : files.items + if (!base.accept && info.suffixes) base.accept = info.suffixes.join(',') + } + // Do not show steps if (schema.properties.gain) schema.properties.gain.step = null diff --git a/src/renderer/src/stories/FileSystemSelector.js b/src/renderer/src/stories/FileSystemSelector.js index 7a38d1f5e..6ce05f340 100644 --- a/src/renderer/src/stories/FileSystemSelector.js +++ b/src/renderer/src/stories/FileSystemSelector.js @@ -103,6 +103,8 @@ export class FilesystemSelector extends LitElement { if (props.onSelect) this.onSelect = props.onSelect; if (props.onChange) this.onChange = props.onChange; if (props.onThrow) this.onThrow = props.onThrow; + + this.accept = props.accept; this.multiple = props.multiple; this.type = props.type ?? "file"; this.value = props.value ?? ""; @@ -125,6 +127,11 @@ export class FilesystemSelector extends LitElement { #useElectronDialog = async (type) => { const options = { ...this.dialogOptions }; + + if (!options.filters && this.accept) { + options.filters = [{ name: "Selected Files", extensions: this.accept.split(",") }]; + } + options.properties = [ type === "file" ? "openFile" : "openDirectory", "noResolveAliases", diff --git a/src/renderer/src/stories/JSONSchemaInput.js b/src/renderer/src/stories/JSONSchemaInput.js index 3ce99b0aa..f68fb0f72 100644 --- a/src/renderer/src/stories/JSONSchemaInput.js +++ b/src/renderer/src/stories/JSONSchemaInput.js @@ -893,6 +893,7 @@ export class JSONSchemaInput extends LitElement { const filesystemSelectorElement = new FilesystemSelector({ type: format, value: this.value, + accept: schema.accept, onSelect: (paths = []) => { const value = paths.length ? paths : undefined; this.#updateData(fullPath, value); From 94ba625d813b50486d75f93215bfca94f364e86b Mon Sep 17 00:00:00 2001 From: Garrett Michael Flynn Date: Wed, 15 May 2024 09:43:00 -0700 Subject: [PATCH 06/25] Use array directly. Block for drag too --- schemas/source-data.schema.ts | 2 +- .../src/stories/FileSystemSelector.js | 19 +++++++++++++++---- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/schemas/source-data.schema.ts b/schemas/source-data.schema.ts index 785d26910..b09c88abf 100644 --- a/schemas/source-data.schema.ts +++ b/schemas/source-data.schema.ts @@ -32,7 +32,7 @@ export default function preprocessSourceDataSchema (schema) { if (files) { const base = singleLocationInfo ? files : files.items - if (!base.accept && info.suffixes) base.accept = info.suffixes.join(',') + if (!base.accept && info.suffixes) base.accept = info.suffixes } // Do not show steps diff --git a/src/renderer/src/stories/FileSystemSelector.js b/src/renderer/src/stories/FileSystemSelector.js index 6ce05f340..eb906ad9e 100644 --- a/src/renderer/src/stories/FileSystemSelector.js +++ b/src/renderer/src/stories/FileSystemSelector.js @@ -129,7 +129,7 @@ export class FilesystemSelector extends LitElement { const options = { ...this.dialogOptions }; if (!options.filters && this.accept) { - options.filters = [{ name: "Selected Files", extensions: this.accept.split(",") }]; + options.filters = [{ name: "Selected Files", extensions: this.accept }]; } options.properties = [ @@ -149,18 +149,29 @@ export class FilesystemSelector extends LitElement { return result; }; - #checkType = (value) => { + #check = (value) => { + + // Check type const isLikelyFile = fs ? fs.statSync(value).isFile() : value.split(".").length; if ((this.type === "directory" && isLikelyFile) || (this.type === "file" && !isLikelyFile)) this.#onThrow("Incorrect filesystem object", `Please provide a ${this.type} instead.`); + + if (this.accept) { + const ext = value.split(".").pop(); + if (!this.accept.includes(ext)) + this.#onThrow( + "Incorrect file extension", + `Please provide a file with the extension ${this.accept.join(', ')} instead.` + ); + } }; #handleFiles = async (pathOrPaths, type) => { const resolvedType = type ?? this.type; if (pathOrPaths) { - if (Array.isArray(pathOrPaths)) pathOrPaths.forEach(this.#checkType); - else if (!type) this.#checkType(pathOrPaths); + if (Array.isArray(pathOrPaths)) pathOrPaths.forEach(this.#check); + else if (!type) this.#check(pathOrPaths); } let resolvedValue = pathOrPaths; From 5915f23b246bdc955150d8ae47c49078b690c6f3 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 15 May 2024 16:43:40 +0000 Subject: [PATCH 07/25] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/renderer/src/stories/FileSystemSelector.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/renderer/src/stories/FileSystemSelector.js b/src/renderer/src/stories/FileSystemSelector.js index eb906ad9e..2013d8aac 100644 --- a/src/renderer/src/stories/FileSystemSelector.js +++ b/src/renderer/src/stories/FileSystemSelector.js @@ -150,7 +150,6 @@ export class FilesystemSelector extends LitElement { }; #check = (value) => { - // Check type const isLikelyFile = fs ? fs.statSync(value).isFile() : value.split(".").length; if ((this.type === "directory" && isLikelyFile) || (this.type === "file" && !isLikelyFile)) @@ -161,7 +160,7 @@ export class FilesystemSelector extends LitElement { if (!this.accept.includes(ext)) this.#onThrow( "Incorrect file extension", - `Please provide a file with the extension ${this.accept.join(', ')} instead.` + `Please provide a file with the extension ${this.accept.join(", ")} instead.` ); } }; From 5c5e0f469f14241ee59cfb7b1d5990c4e291d75c Mon Sep 17 00:00:00 2001 From: Garrett Michael Flynn Date: Wed, 15 May 2024 09:59:00 -0700 Subject: [PATCH 08/25] Small fixes --- pyflask/manageNeuroconv/manage_neuroconv.py | 3 +-- .../src/stories/pages/guided-mode/data/GuidedMetadata.js | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index 983d66785..5b12014da 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -443,7 +443,6 @@ def on_sorting_interface(name, sorting_interface): return sorting_interface def on_recording_interface(name, recording_interface): - global aggregate_electrode_columns electrode_columns = get_electrode_columns_json(recording_interface) @@ -777,7 +776,7 @@ def update_conversion_progress(**kwargs): {"name": entry["name"], "description": entry["description"]} for entry in shared_electrode_columns ] - del ecephys_metadata["ElectrodeColumns"] + del ecephys_metadata["ElectrodeColumns"] # Actually run the conversion converter.run_conversion( diff --git a/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js b/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js index 2e0e77d75..06739e4d9 100644 --- a/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js +++ b/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js @@ -116,6 +116,7 @@ const propsToIgnore = { ElectricalSeries: true, ElectricalSeriesLF: true, ElectricalSeriesAP: true, + ElectricalSeriesNIDQ: true, Units: { "*": { UnitColumns: { From d5ec9a853f73405cb344dd3efc9f97772e48933d Mon Sep 17 00:00:00 2001 From: Garrett Michael Flynn Date: Wed, 15 May 2024 10:25:37 -0700 Subject: [PATCH 09/25] Update manage_neuroconv.py --- pyflask/manageNeuroconv/manage_neuroconv.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index 5b12014da..c4c2f8272 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -772,9 +772,9 @@ def update_conversion_progress(**kwargs): electrode_column_info=shared_electrode_columns, ) - ecephys_metadata["Electrodes"] = [ - {"name": entry["name"], "description": entry["description"]} for entry in shared_electrode_columns - ] + ecephys_metadata["Electrodes"] = [ + {"name": entry["name"], "description": entry["description"]} for entry in shared_electrode_columns + ] del ecephys_metadata["ElectrodeColumns"] From 48dd6a2765da264ef3c3705c048af9f9d5d2ff41 Mon Sep 17 00:00:00 2001 From: Cody Baker Date: Wed, 15 May 2024 13:30:39 -0400 Subject: [PATCH 10/25] fix subconverter nesting during update --- pyflask/manageNeuroconv/manage_neuroconv.py | 51 ++++++++++++++----- .../pages/guided-mode/data/GuidedMetadata.js | 18 ++++--- 2 files changed, 47 insertions(+), 22 deletions(-) diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index 983d66785..8c1bf49ac 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -8,7 +8,7 @@ import hashlib from pathlib import Path from datetime import datetime -from typing import Dict, Optional +from typing import Dict, Optional, Union from shutil import rmtree, copytree from pathlib import Path from typing import Any, Dict, List, Optional @@ -360,7 +360,7 @@ def get_source_schema(interface_class_dict: dict) -> dict: return CustomNWBConverter.get_source_schema() -def map_interfaces(callback, converter, to_match=None, parent_name=None): +def map_interfaces(callback, converter, to_match: Union["BaseDataInterface", None] = None, parent_name=None): from neuroconv import NWBConverter output = [] @@ -369,9 +369,10 @@ def map_interfaces(callback, converter, to_match=None, parent_name=None): associated_name = f"{parent_name} — {name}" if parent_name else name if isinstance(interface, NWBConverter): - result = map_interfaces(callback, interface, to_match, associated_name) + result = map_interfaces(callback=callback, converter=interface, to_match=to_match, + parent_name=associated_name) output.extend(result) - elif to_match == None or isinstance(interface, to_match): + elif to_match is None or isinstance(interface, to_match): result = callback(associated_name, interface) output.append(result) @@ -443,8 +444,6 @@ def on_sorting_interface(name, sorting_interface): return sorting_interface def on_recording_interface(name, recording_interface): - global aggregate_electrode_columns - electrode_columns = get_electrode_columns_json(recording_interface) # Aggregate electrode column information across recording interfaces @@ -482,10 +481,10 @@ def on_recording_interface(name, recording_interface): from neuroconv.datainterfaces.ecephys.basesortingextractorinterface import BaseSortingExtractorInterface # Map recording interfaces to metadata - map_interfaces(on_recording_interface, converter, BaseRecordingExtractorInterface) + map_interfaces(on_recording_interface, converter=converter, to_match=BaseRecordingExtractorInterface) # Map sorting interfaces to metadata - map_interfaces(on_sorting_interface, converter, BaseSortingExtractorInterface) + map_interfaces(on_sorting_interface, converter=converter, to_match=BaseSortingExtractorInterface) if has_ecephys: @@ -577,7 +576,14 @@ def on_recording_interface(name, recording_interface): "additionalProperties": True, # Allow for new columns } - return json.loads(json.dumps(replace_nan_with_none(dict(results=metadata, schema=schema)), cls=NWBMetaDataEncoder)) + # TODO: generalize as log + with open(file="C:/Users/theac/Downloads/file_metadata_page_schema.json", mode="w") as fp: + json.dump(obj=dict(schema=schema), fp=fp, cls=NWBMetaDataEncoder, indent=2) + with open(file="C:/Users/theac/Downloads/file_metadata_page_results.json", mode="w") as fp: + json.dump(obj=dict(results=metadata), fp=fp, cls=NWBMetaDataEncoder, indent=2) + + return json.loads(json.dumps(obj=replace_nan_with_none(dict(results=metadata, schema=schema)), + cls=NWBMetaDataEncoder)) def get_check_function(check_function_name: str) -> callable: @@ -681,6 +687,8 @@ def get_interface_alignment(info: dict) -> dict: def convert_to_nwb(info: dict) -> str: """Function used to convert the source data to NWB format using the specified metadata.""" + from neuroconv import NWBConverter + nwbfile_path = Path(info["nwbfile_path"]) custom_output_directory = info.get("output_folder") project_name = info.get("project_name") @@ -762,13 +770,28 @@ def update_conversion_progress(**kwargs): shared_electrode_columns = ecephys_metadata["ElectrodeColumns"] for interface_name, interface_electrode_results in ecephys_metadata["Electrodes"].items(): + name_split = interface_name.split(" — ") + + if len(name_split) == 1: + sub_interface = name_split[0] + elif len(name_split) == 2: + sub_interface, sub_sub_interface = name_split + + interface_or_subconverter = converter.data_interface_objects[sub_interface] - interface = converter - for sub_interface in interface_name.split(" — "): - interface = interface.data_interface_objects[sub_interface] + if isinstance(interface_or_subconverter, NWBConverter): + subconverter = interface_or_subconverter + + update_recording_properties_from_table_as_json( + recording_interface=subconverter.data_interface_objects[sub_sub_interface], + electrode_table_json=interface_electrode_results, + electrode_column_info=shared_electrode_columns, + ) + else: + interface = interface_or_subconverter update_recording_properties_from_table_as_json( - interface, + recording_interface=interface, electrode_table_json=interface_electrode_results, electrode_column_info=shared_electrode_columns, ) @@ -777,7 +800,7 @@ def update_conversion_progress(**kwargs): {"name": entry["name"], "description": entry["description"]} for entry in shared_electrode_columns ] - del ecephys_metadata["ElectrodeColumns"] + del ecephys_metadata["ElectrodeColumns"] # Actually run the conversion converter.run_conversion( diff --git a/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js b/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js index 2e0e77d75..3a29b406b 100644 --- a/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js +++ b/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js @@ -85,6 +85,15 @@ const tableRenderConfig = { const imagingPlaneKey = "imaging_plane"; const propsToIgnore = { + NWBFile: { + session_id: true, + source_script: true, + source_script_file_name: true, + identifier: true, + }, + Subject: { + subject_id: true, + }, Ophys: { "*": { starting_time: true, @@ -116,6 +125,7 @@ const propsToIgnore = { ElectricalSeries: true, ElectricalSeriesLF: true, ElectricalSeriesAP: true, + ElectricalSeriesNIDQ: true, Units: { "*": { UnitColumns: { @@ -129,14 +139,6 @@ const propsToIgnore = { Icephys: true, // Always ignore icephys metadata (for now) Behavior: true, // Always ignore behavior metadata (for now) "ndx-dandi-icephys": true, - Subject: { - subject_id: true, - }, - NWBFile: { - session_id: true, - source_script: true, - source_script_file_name: true, - }, }; import { preprocessMetadataSchema } from "../../../../../../../schemas/base-metadata.schema"; From f7bb15181471dd50e0a8cb39fcfc04ad176fdc7c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 15 May 2024 17:32:50 +0000 Subject: [PATCH 11/25] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pyflask/manageNeuroconv/manage_neuroconv.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index 981fedfe9..7f33f86d2 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -369,8 +369,9 @@ def map_interfaces(callback, converter, to_match: Union["BaseDataInterface", Non associated_name = f"{parent_name} — {name}" if parent_name else name if isinstance(interface, NWBConverter): - result = map_interfaces(callback=callback, converter=interface, to_match=to_match, - parent_name=associated_name) + result = map_interfaces( + callback=callback, converter=interface, to_match=to_match, parent_name=associated_name + ) output.extend(result) elif to_match is None or isinstance(interface, to_match): result = callback(associated_name, interface) @@ -582,8 +583,9 @@ def on_recording_interface(name, recording_interface): with open(file="C:/Users/theac/Downloads/file_metadata_page_results.json", mode="w") as fp: json.dump(obj=dict(results=metadata), fp=fp, cls=NWBMetaDataEncoder, indent=2) - return json.loads(json.dumps(obj=replace_nan_with_none(dict(results=metadata, schema=schema)), - cls=NWBMetaDataEncoder)) + return json.loads( + json.dumps(obj=replace_nan_with_none(dict(results=metadata, schema=schema)), cls=NWBMetaDataEncoder) + ) def get_check_function(check_function_name: str) -> callable: From 9f2f8b1eb6884489c5c3efe816bbc6e2a61cf15e Mon Sep 17 00:00:00 2001 From: Garrett Michael Flynn Date: Wed, 15 May 2024 10:43:23 -0700 Subject: [PATCH 12/25] Remove capsules shown in header --- src/renderer/src/stories/Main.js | 29 ++-------- .../pages/guided-mode/GuidedCapsules.js | 54 ------------------- 2 files changed, 3 insertions(+), 80 deletions(-) delete mode 100644 src/renderer/src/stories/pages/guided-mode/GuidedCapsules.js diff --git a/src/renderer/src/stories/Main.js b/src/renderer/src/stories/Main.js index 172c036b8..bbf19d025 100644 --- a/src/renderer/src/stories/Main.js +++ b/src/renderer/src/stories/Main.js @@ -1,7 +1,6 @@ import { LitElement, html } from "lit"; import useGlobalStyles from "./utils/useGlobalStyles.js"; import { GuidedFooter } from "./pages/guided-mode/GuidedFooter"; -import { GuidedCapsules } from "./pages/guided-mode/GuidedCapsules.js"; import { GuidedHeader } from "./pages/guided-mode/GuidedHeader.js"; import { unsafeHTML } from "lit/directives/unsafe-html.js"; @@ -131,7 +130,6 @@ export class Main extends LitElement { let footer = page?.footer; // Page-specific footer let header = page?.header; // Page-specific header - let capsules = page?.capsules; // Page-specific capsules if (page) { this.to = page.to; @@ -160,23 +158,9 @@ export class Main extends LitElement { if (footer === true) footer = {}; if (footer && "onNext" in footer && !("next" in footer)) footer.next = "Next"; - // Default Capsules Behavior + // Define header states const section = sections[info.section]; if (section) { - if (capsules === true || !("capsules" in page)) { - let pages = Object.values(section.pages); - const pageIds = Object.keys(section.pages); - if (pages.length > 1) { - const capsulesProps = { - n: pages.length, - skipped: pages.map((page) => page.skipped), - selected: pages.map((page) => page.pageLabel).indexOf(page.info.label), - }; - - capsules = new GuidedCapsules(capsulesProps); - capsules.onClick = (i) => this.toRender.page.to(pageIds[i]); - } - } if (header === true || !("header" in page) || !("sections" in page.header)) { const sectionNames = Object.entries(sections) @@ -207,17 +191,10 @@ export class Main extends LitElement { return html` ${headerEl} - ${ - capsules - ? html`
${capsules}
` - : html`` - } ${ title ? html`
@@ -232,7 +209,7 @@ export class Main extends LitElement { }
${page}
diff --git a/src/renderer/src/stories/pages/guided-mode/GuidedCapsules.js b/src/renderer/src/stories/pages/guided-mode/GuidedCapsules.js deleted file mode 100644 index 95db22a8d..000000000 --- a/src/renderer/src/stories/pages/guided-mode/GuidedCapsules.js +++ /dev/null @@ -1,54 +0,0 @@ -import { LitElement, html } from "lit"; - -export class GuidedCapsules extends LitElement { - constructor({ n = 0, selected = 0, skipped = [] } = {}) { - super(); - this.n = n; - this.selected = selected; - this.skipped = skipped; - this.style.width = "100%"; - } - - static get properties() { - return { - n: { type: Number, reflect: true }, - selected: { type: Number, reflect: true }, - skipped: { type: Array }, - }; - } - - attributeChangedCallback(...args) { - const attrs = ["n", "selected"]; - super.attributeChangedCallback(...args); - if (attrs.includes(args[0])) this.requestUpdate(); - } - - createRenderRoot() { - return this; - } - - onClick = () => {}; - - render() { - if (!this.n) return html``; - - return html` -
-
- ${Array.from( - { length: this.n }, - (_, i) => - html`
this.onClick(i)} - class="guided--capsule ${i === this.selected ? `active` : ""} ${this.skipped[i] - ? `skipped` - : ""}" - >
` - )} -
-
- `; - } -} - -customElements.get("nwb-guided-capsules") || customElements.define("nwb-guided-capsules", GuidedCapsules); From 2c7bb8bb9a304b105e434d58f3a7375aaca4b1c4 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 15 May 2024 17:44:27 +0000 Subject: [PATCH 13/25] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/renderer/src/stories/Main.js | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/renderer/src/stories/Main.js b/src/renderer/src/stories/Main.js index bbf19d025..9a68987cd 100644 --- a/src/renderer/src/stories/Main.js +++ b/src/renderer/src/stories/Main.js @@ -161,7 +161,6 @@ export class Main extends LitElement { // Define header states const section = sections[info.section]; if (section) { - if (header === true || !("header" in page) || !("sections" in page.header)) { const sectionNames = Object.entries(sections) .filter(([name, info]) => !Object.values(info.pages).every((state) => state.skipped)) @@ -208,9 +207,7 @@ export class Main extends LitElement { : "" } -
+
${page}
${footerEl} From 41aa7658c0562bf7e5673c4061161c8fe5fe5cc9 Mon Sep 17 00:00:00 2001 From: Cody Baker Date: Wed, 15 May 2024 14:48:39 -0400 Subject: [PATCH 14/25] fix temporary logging --- pyflask/manageNeuroconv/manage_neuroconv.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index 981fedfe9..243e54b7a 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -576,10 +576,11 @@ def on_recording_interface(name, recording_interface): "additionalProperties": True, # Allow for new columns } - # TODO: generalize as log - with open(file="C:/Users/theac/Downloads/file_metadata_page_schema.json", mode="w") as fp: + # TODO: generalize logging stuff + log_base = GUIDE_ROOT_FOLDER / "logs" + with open(file=log_base / "file_metadata_page_schema.json", mode="w") as fp: json.dump(obj=dict(schema=schema), fp=fp, cls=NWBMetaDataEncoder, indent=2) - with open(file="C:/Users/theac/Downloads/file_metadata_page_results.json", mode="w") as fp: + with open(file=log_base / "file_metadata_page_results.json", mode="w") as fp: json.dump(obj=dict(results=metadata), fp=fp, cls=NWBMetaDataEncoder, indent=2) return json.loads(json.dumps(obj=replace_nan_with_none(dict(results=metadata, schema=schema)), From d6d275f8658a2ddcf253a72e497e36f26c89effb Mon Sep 17 00:00:00 2001 From: Cody Baker Date: Wed, 15 May 2024 14:55:59 -0400 Subject: [PATCH 15/25] ensure log folder exists --- pyflask/manageNeuroconv/manage_neuroconv.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index 45757a75a..b399b681d 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -579,6 +579,7 @@ def on_recording_interface(name, recording_interface): # TODO: generalize logging stuff log_base = GUIDE_ROOT_FOLDER / "logs" + log_base.mkdir(exist_ok=True) with open(file=log_base / "file_metadata_page_schema.json", mode="w") as fp: json.dump(obj=dict(schema=schema), fp=fp, cls=NWBMetaDataEncoder, indent=2) with open(file=log_base / "file_metadata_page_results.json", mode="w") as fp: From 2e3160132ad3f9a21260e2178374406a6842d20b Mon Sep 17 00:00:00 2001 From: Garrett Michael Flynn Date: Thu, 16 May 2024 07:55:40 -0700 Subject: [PATCH 16/25] Allow all. Ensure proper extension format --- src/renderer/src/stories/FileSystemSelector.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/renderer/src/stories/FileSystemSelector.js b/src/renderer/src/stories/FileSystemSelector.js index 2013d8aac..daf38b142 100644 --- a/src/renderer/src/stories/FileSystemSelector.js +++ b/src/renderer/src/stories/FileSystemSelector.js @@ -129,7 +129,10 @@ export class FilesystemSelector extends LitElement { const options = { ...this.dialogOptions }; if (!options.filters && this.accept) { - options.filters = [{ name: "Selected Files", extensions: this.accept }]; + options.filters = [ + { name: "Suggested Files", extensions: this.accept.map(ext => ext[0] === '.' ? ext.slice(1) : ext) }, + { name: "All Files", extensions: [ "*" ]} + ]; } options.properties = [ From 366066917f0c6e7079209a8560649219e22d62cd Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 16 May 2024 14:55:58 +0000 Subject: [PATCH 17/25] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/renderer/src/stories/FileSystemSelector.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/renderer/src/stories/FileSystemSelector.js b/src/renderer/src/stories/FileSystemSelector.js index daf38b142..76b9d313a 100644 --- a/src/renderer/src/stories/FileSystemSelector.js +++ b/src/renderer/src/stories/FileSystemSelector.js @@ -130,8 +130,11 @@ export class FilesystemSelector extends LitElement { if (!options.filters && this.accept) { options.filters = [ - { name: "Suggested Files", extensions: this.accept.map(ext => ext[0] === '.' ? ext.slice(1) : ext) }, - { name: "All Files", extensions: [ "*" ]} + { + name: "Suggested Files", + extensions: this.accept.map((ext) => (ext[0] === "." ? ext.slice(1) : ext)), + }, + { name: "All Files", extensions: ["*"] }, ]; } From 45337358a4d41fb967b5550a4fa3987686c40ada Mon Sep 17 00:00:00 2001 From: Garrett Michael Flynn Date: Thu, 16 May 2024 08:10:42 -0700 Subject: [PATCH 18/25] Fix and comment check for dragged files --- src/renderer/src/stories/FileSystemSelector.js | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/renderer/src/stories/FileSystemSelector.js b/src/renderer/src/stories/FileSystemSelector.js index daf38b142..4908fb3fa 100644 --- a/src/renderer/src/stories/FileSystemSelector.js +++ b/src/renderer/src/stories/FileSystemSelector.js @@ -153,19 +153,21 @@ export class FilesystemSelector extends LitElement { }; #check = (value) => { + // Check type const isLikelyFile = fs ? fs.statSync(value).isFile() : value.split(".").length; if ((this.type === "directory" && isLikelyFile) || (this.type === "file" && !isLikelyFile)) this.#onThrow("Incorrect filesystem object", `Please provide a ${this.type} instead.`); - if (this.accept) { - const ext = value.split(".").pop(); - if (!this.accept.includes(ext)) - this.#onThrow( - "Incorrect file extension", - `Please provide a file with the extension ${this.accept.join(", ")} instead.` - ); - } + // if (this.accept) { + // const ext = value.split(".").pop(); + // const noDot = this.accept.map(ext => ext[0] === "." ? ext.slice(1) : ext) + // if (!noDot.includes(ext)) + // this.#onThrow( + // "Incorrect file extension", + // `Please provide a file with the extension ${this.accept.join(", ")} instead.` + // ); + // } }; #handleFiles = async (pathOrPaths, type) => { From de5d8a995a19b8da53ad133148fea1d71d57d6ef Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 16 May 2024 15:12:18 +0000 Subject: [PATCH 19/25] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/renderer/src/stories/FileSystemSelector.js | 1 - 1 file changed, 1 deletion(-) diff --git a/src/renderer/src/stories/FileSystemSelector.js b/src/renderer/src/stories/FileSystemSelector.js index 5ae2cfefe..3a9f686c4 100644 --- a/src/renderer/src/stories/FileSystemSelector.js +++ b/src/renderer/src/stories/FileSystemSelector.js @@ -156,7 +156,6 @@ export class FilesystemSelector extends LitElement { }; #check = (value) => { - // Check type const isLikelyFile = fs ? fs.statSync(value).isFile() : value.split(".").length; if ((this.type === "directory" && isLikelyFile) || (this.type === "file" && !isLikelyFile)) From b157877cc17599ef35a38bf21fdd95c1fdc3900a Mon Sep 17 00:00:00 2001 From: Garrett Michael Flynn Date: Fri, 17 May 2024 06:30:35 -0700 Subject: [PATCH 20/25] Update FileSystemSelector.js --- src/renderer/src/stories/FileSystemSelector.js | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/src/renderer/src/stories/FileSystemSelector.js b/src/renderer/src/stories/FileSystemSelector.js index 3a9f686c4..5e33c5be8 100644 --- a/src/renderer/src/stories/FileSystemSelector.js +++ b/src/renderer/src/stories/FileSystemSelector.js @@ -160,16 +160,6 @@ export class FilesystemSelector extends LitElement { const isLikelyFile = fs ? fs.statSync(value).isFile() : value.split(".").length; if ((this.type === "directory" && isLikelyFile) || (this.type === "file" && !isLikelyFile)) this.#onThrow("Incorrect filesystem object", `Please provide a ${this.type} instead.`); - - // if (this.accept) { - // const ext = value.split(".").pop(); - // const noDot = this.accept.map(ext => ext[0] === "." ? ext.slice(1) : ext) - // if (!noDot.includes(ext)) - // this.#onThrow( - // "Incorrect file extension", - // `Please provide a file with the extension ${this.accept.join(", ")} instead.` - // ); - // } }; #handleFiles = async (pathOrPaths, type) => { From d770c541605f3d565596eff99e84733856a5f9b7 Mon Sep 17 00:00:00 2001 From: Garrett Michael Flynn Date: Fri, 17 May 2024 07:18:17 -0700 Subject: [PATCH 21/25] Style version dropdown --- docs/_static/css/custom.css | 35 +++++++++++++++++++++++++++++++++++ docs/conf.py | 10 +++++++--- 2 files changed, 42 insertions(+), 3 deletions(-) create mode 100644 docs/_static/css/custom.css diff --git a/docs/_static/css/custom.css b/docs/_static/css/custom.css new file mode 100644 index 000000000..da99f3305 --- /dev/null +++ b/docs/_static/css/custom.css @@ -0,0 +1,35 @@ +/* Improve spacing */ +.version-switcher__container.dropdown { + margin-left: 10px; +} + + +button.btn.version-switcher__button { + margin-bottom: 0px; +} + +/* Show on hover */ +.version-switcher__container.dropdown:hover .dropdown-menu { + display: block; + left: 0; + margin-top: var(--bs-dropdown-spacer); + top: 100%; +} + +.dropdown-menu.show { + display: none; +} + +/* Remove underline and borders */ +button.btn.version-switcher__button:hover { + text-decoration: none; +} + +.version-switcher__menu a.list-group-item { + border: none !important; + +} + +.version-switcher__menu a.list-group-item:hover { + text-decoration: none !important; +} \ No newline at end of file diff --git a/docs/conf.py b/docs/conf.py index 52a59ff6f..29b44b12a 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -45,9 +45,9 @@ ] # These paths are either relative to html_static_path or fully qualified paths (eg. https://...) -# html_css_files = [ -# "css/custom.css", -# ] +html_css_files = [ + "css/custom.css", +] linkcheck_anchors = False @@ -85,6 +85,7 @@ version_match = os.environ.get("READTHEDOCS_VERSION") with open("../package.json") as f: release = json.load(f)["version"] + # If READTHEDOCS_VERSION doesn't exist, we're not on RTD # If it is an integer, we're in a PR build and the version isn't correct. # If it's "latest" → change to "dev" (that's what we want the switcher to call it) @@ -144,3 +145,6 @@ def _correct_signatures(app, what, name, obj, options, signature, return_annotat def setup(app): # This makes the data-interfaces signatures display on the docs/api, they don't otherwise app.connect("autodoc-process-signature", _correct_signatures) + + # Add custom CSS + app.add_css_file("css/custom.css") From 2b2468d75b17af3362c2626a89b0594ce5727d21 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 17 May 2024 14:19:21 +0000 Subject: [PATCH 22/25] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- docs/_static/css/custom.css | 4 +--- docs/conf.py | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/docs/_static/css/custom.css b/docs/_static/css/custom.css index da99f3305..4e72cecdb 100644 --- a/docs/_static/css/custom.css +++ b/docs/_static/css/custom.css @@ -3,7 +3,6 @@ margin-left: 10px; } - button.btn.version-switcher__button { margin-bottom: 0px; } @@ -27,9 +26,8 @@ button.btn.version-switcher__button:hover { .version-switcher__menu a.list-group-item { border: none !important; - } .version-switcher__menu a.list-group-item:hover { text-decoration: none !important; -} \ No newline at end of file +} diff --git a/docs/conf.py b/docs/conf.py index 29b44b12a..5067e13b6 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -85,7 +85,7 @@ version_match = os.environ.get("READTHEDOCS_VERSION") with open("../package.json") as f: release = json.load(f)["version"] - + # If READTHEDOCS_VERSION doesn't exist, we're not on RTD # If it is an integer, we're in a PR build and the version isn't correct. # If it's "latest" → change to "dev" (that's what we want the switcher to call it) From e5dfcb31dea76dfb3577107606bed51628e54a7b Mon Sep 17 00:00:00 2001 From: Ben Dichter Date: Sat, 18 May 2024 00:39:35 -0500 Subject: [PATCH 23/25] cleanup: typehinting and import optimization --- pyflask/manageNeuroconv/manage_neuroconv.py | 44 ++++++++++----------- 1 file changed, 21 insertions(+), 23 deletions(-) diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index b399b681d..860776276 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -1,17 +1,16 @@ """Collection of utility functions used by the NeuroConv Flask API.""" -import os +import copy +import hashlib import json import math -import copy +import os import re -import hashlib -from pathlib import Path from datetime import datetime -from typing import Dict, Optional, Union -from shutil import rmtree, copytree from pathlib import Path +from shutil import rmtree, copytree from typing import Any, Dict, List, Optional +from typing import Union from .info import GUIDE_ROOT_FOLDER, STUB_SAVE_FOLDER_PATH, CONVERSION_SAVE_FOLDER_PATH, announcer @@ -112,7 +111,7 @@ def replace_nan_with_none(data): return data -def resolve_references(schema, root_schema=None): +def resolve_references(schema: dict, root_schema: Optional[dict] = None) -> dict: """ Recursively resolve references in a JSON schema based on the root schema. @@ -142,7 +141,7 @@ def resolve_references(schema, root_schema=None): return schema -def replace_none_with_nan(json_object, json_schema): +def replace_none_with_nan(json_object: dict, json_schema: dict) -> dict: """ Recursively search a JSON object and replace None values with NaN where appropriate. @@ -254,7 +253,7 @@ def locate_data(info: dict) -> dict: return json.loads(json.dumps(obj=organized_output, cls=NWBMetaDataEncoder)) -def module_to_dict(my_module): +def module_to_dict(my_module) -> dict: # Create an empty dictionary module_dict = {} @@ -278,7 +277,7 @@ def get_class_ref_in_docstring(input_string): return match.group(1) -def derive_interface_info(interface): +def derive_interface_info(interface) -> dict: info = {"keywords": getattr(interface, "keywords", []), "description": ""} @@ -349,7 +348,7 @@ class CustomNWBConverter(NWBConverter): return CustomNWBConverter -def instantiate_custom_converter(source_data, interface_class_dict): # -> NWBConverter: +def instantiate_custom_converter(source_data: dict, interface_class_dict: dict): # -> NWBConverter: CustomNWBConverter = get_custom_converter(interface_class_dict) return CustomNWBConverter(source_data) @@ -360,7 +359,7 @@ def get_source_schema(interface_class_dict: dict) -> dict: return CustomNWBConverter.get_source_schema() -def map_interfaces(callback, converter, to_match: Union["BaseDataInterface", None] = None, parent_name=None): +def map_interfaces(callback, converter, to_match: Union["BaseDataInterface", None] = None, parent_name=None) -> list: from neuroconv import NWBConverter output = [] @@ -836,7 +835,7 @@ def update_conversion_progress(**kwargs): return dict(file=str(resolved_output_path)) -def upload_multiple_filesystem_objects_to_dandi(**kwargs): +def upload_multiple_filesystem_objects_to_dandi(**kwargs) -> list[Path]: tmp_folder_path = _aggregate_symlinks_in_new_directory(kwargs["filesystem_paths"], "upload") innerKwargs = {**kwargs} del innerKwargs["filesystem_paths"] @@ -855,7 +854,7 @@ def upload_folder_to_dandi( number_of_jobs: Optional[int] = None, number_of_threads: Optional[int] = None, ignore_cache: bool = False, -): +) -> list[Path]: from neuroconv.tools.data_transfers import automatic_dandi_upload os.environ["DANDI_API_KEY"] = api_key # Update API Key @@ -884,7 +883,7 @@ def upload_project_to_dandi( number_of_jobs: Optional[int] = None, number_of_threads: Optional[int] = None, ignore_cache: bool = False, -): +) -> list[Path]: from neuroconv.tools.data_transfers import automatic_dandi_upload # CONVERSION_SAVE_FOLDER_PATH.mkdir(exist_ok=True, parents=True) # Ensure base directory exists @@ -914,7 +913,7 @@ def listen_to_neuroconv_events(): yield msg -def generate_dataset(input_path: str, output_path: str): +def generate_dataset(input_path: str, output_path: str) -> dict: base_path = Path(input_path) output_path = Path(output_path) @@ -956,7 +955,7 @@ def generate_dataset(input_path: str, output_path: str): return {"output_path": str(output_path)} -def inspect_nwb_file(payload): +def inspect_nwb_file(payload) -> dict: from nwbinspector import inspect_nwbfile, load_config from nwbinspector.inspector_tools import format_messages, get_report_header from nwbinspector.nwbinspector import InspectorOutputJSONEncoder @@ -987,7 +986,7 @@ def _inspect_file_per_job( url, ignore: Optional[List[str]] = None, request_id: Optional[str] = None, -): +) -> list: from nwbinspector import nwbinspector from pynwb import NWBHDF5IO @@ -1085,7 +1084,7 @@ def on_progress_update(message): return messages -def inspect_nwb_folder(url, payload): +def inspect_nwb_folder(url, payload) -> dict: from nwbinspector import load_config from nwbinspector.inspector_tools import format_messages, get_report_header from nwbinspector.nwbinspector import InspectorOutputJSONEncoder @@ -1118,7 +1117,7 @@ def inspect_nwb_folder(url, payload): return json.loads(json.dumps(obj=json_report, cls=InspectorOutputJSONEncoder)) -def _aggregate_symlinks_in_new_directory(paths, reason="", folder_path=None): +def _aggregate_symlinks_in_new_directory(paths, reason="", folder_path=None) -> Path: if folder_path is None: folder_path = GUIDE_ROOT_FOLDER / ".temp" / reason / f"temp_{datetime.now().strftime('%Y%m%d-%H%M%S')}" @@ -1137,7 +1136,7 @@ def _aggregate_symlinks_in_new_directory(paths, reason="", folder_path=None): return folder_path -def inspect_multiple_filesystem_objects(url, paths, **kwargs): +def inspect_multiple_filesystem_objects(url, paths, **kwargs) -> dict: tmp_folder_path = _aggregate_symlinks_in_new_directory(paths, "inspect") result = inspect_nwb_folder(url, {"path": tmp_folder_path, **kwargs}) rmtree(tmp_folder_path) @@ -1211,7 +1210,6 @@ def generate_test_data(output_path: str): Consists of a single-probe single-segment SpikeGLX recording (both AP and LF bands) as well as Phy spiking data. """ import spikeinterface - from spikeinterface.extractors import NumpyRecording from spikeinterface.exporters import export_to_phy from spikeinterface.preprocessing import scale, bandpass_filter, resample @@ -1283,7 +1281,7 @@ def map_dtype(dtype: str) -> str: return dtype -def get_property_dtype(extractor, property_name: str, ids: list, extra_props: dict): +def get_property_dtype(extractor, property_name: str, ids: list, extra_props: dict) -> str: if property_name in extra_props: dtype = extra_props[property_name]["data_type"] else: From ad0bce55541f24412adf7bb906f016e21bbe9b90 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Sat, 18 May 2024 12:33:39 -0400 Subject: [PATCH 24/25] enable isort --- .pre-commit-config.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9273adef7..248023898 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -10,6 +10,10 @@ repos: hooks: - id: black exclude: ^docs/ +- repo: https://github.com/PyCQA/isort + rev: 5.13.2 + hooks: + - id: isort - repo: https://github.com/pre-commit/mirrors-prettier rev: "v4.0.0-alpha.8" hooks: From f234c90b28f7bae13203aa72e596003cc08c07ec Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 18 May 2024 16:34:23 +0000 Subject: [PATCH 25/25] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- demos/sse/test_sse_display_of_tqdm.py | 12 +++---- docs/conf.py | 5 ++- generateInterfaceSchema.py | 5 +-- pyflask/apis/__init__.py | 4 +-- pyflask/apis/data.py | 5 ++- pyflask/apis/neuroconv.py | 30 ++++++++--------- pyflask/apis/startup.py | 3 +- pyflask/app.py | 23 ++++++------- pyflask/manageNeuroconv/__init__.py | 30 ++++++++--------- pyflask/manageNeuroconv/info/__init__.py | 7 ++-- pyflask/manageNeuroconv/info/sse.py | 2 +- pyflask/manageNeuroconv/info/urls.py | 2 +- pyflask/manageNeuroconv/manage_neuroconv.py | 34 +++++++++++++------- pyflask/tests/conftest.py | 2 +- pyflask/tests/test_generate_tutorial_data.py | 3 +- pyflask/tests/test_neuroconv.py | 2 +- pyflask/tests/test_startup.py | 2 +- 17 files changed, 87 insertions(+), 84 deletions(-) diff --git a/demos/sse/test_sse_display_of_tqdm.py b/demos/sse/test_sse_display_of_tqdm.py index 0b705a971..b32452bd6 100644 --- a/demos/sse/test_sse_display_of_tqdm.py +++ b/demos/sse/test_sse_display_of_tqdm.py @@ -1,12 +1,12 @@ -from flask import Flask, render_template, Response -from typing import List -import random import asyncio +import os +import random +import sys import time -from tqdm import tqdm as base_tqdm +from typing import List -import sys -import os +from flask import Flask, Response, render_template +from tqdm import tqdm as base_tqdm SCRIPT_DIR = os.path.dirname(os.path.abspath(os.path.join(__file__, "..", "..", "pyflask"))) sys.path.append(os.path.dirname(SCRIPT_DIR)) diff --git a/docs/conf.py b/docs/conf.py index 5067e13b6..0c466f25e 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,15 +1,14 @@ -import sys import inspect -from pathlib import Path import json import os +import sys +from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parents[0])) sys.path.insert(0, str(Path(__file__).resolve().parents[1])) from conf_extlinks import extlinks, intersphinx_mapping # noqa: E402, F401 - project = "NWB GUIDE" copyright = "2022, CatalystNeuro" # TODO: how to include NWB? author = "Garrett Flynn, Cody Baker, Ryan Ly, Oliver Ruebel, and Ben Dichter" diff --git a/generateInterfaceSchema.py b/generateInterfaceSchema.py index 9a04a8b85..f63eb41f4 100644 --- a/generateInterfaceSchema.py +++ b/generateInterfaceSchema.py @@ -1,6 +1,7 @@ -from pathlib import Path import json -from neuroconv import converters, datainterfaces, NWBConverter +from pathlib import Path + +from neuroconv import NWBConverter, converters, datainterfaces filepath = Path("guideGlobalMetadata.json") generatedJSONSchemaPath = Path("schemas", "json", "generated") diff --git a/pyflask/apis/__init__.py b/pyflask/apis/__init__.py index 9368a0aa8..f492dc7df 100644 --- a/pyflask/apis/__init__.py +++ b/pyflask/apis/__init__.py @@ -1,3 +1,3 @@ -from .startup import startup_api -from .neuroconv import neuroconv_api from .data import data_api +from .neuroconv import neuroconv_api +from .startup import startup_api diff --git a/pyflask/apis/data.py b/pyflask/apis/data.py index 122559150..de6dfdc6d 100644 --- a/pyflask/apis/data.py +++ b/pyflask/apis/data.py @@ -2,10 +2,9 @@ import traceback -from flask_restx import Namespace, Resource, reqparse - -from manageNeuroconv import generate_test_data, generate_dataset from errorHandlers import notBadRequestException +from flask_restx import Namespace, Resource, reqparse +from manageNeuroconv import generate_dataset, generate_test_data data_api = Namespace("data", description="API route for dataset generation in the NWB GUIDE.") diff --git a/pyflask/apis/neuroconv.py b/pyflask/apis/neuroconv.py index 1877e763e..015cb61e8 100644 --- a/pyflask/apis/neuroconv.py +++ b/pyflask/apis/neuroconv.py @@ -2,32 +2,28 @@ import traceback -from flask_restx import Namespace, Resource, reqparse +from errorHandlers import notBadRequestException from flask import Response, request - -from manageNeuroconv.info import announcer - +from flask_restx import Namespace, Resource, reqparse from manageNeuroconv import ( - get_all_interface_info, - get_all_converter_info, - locate_data, autocomplete_format_string, - get_source_schema, - get_metadata_schema, convert_to_nwb, - validate_metadata, - listen_to_neuroconv_events, + get_all_converter_info, + get_all_interface_info, + get_interface_alignment, + get_metadata_schema, + get_source_schema, + inspect_multiple_filesystem_objects, inspect_nwb_file, inspect_nwb_folder, - inspect_multiple_filesystem_objects, - upload_project_to_dandi, + listen_to_neuroconv_events, + locate_data, upload_folder_to_dandi, upload_multiple_filesystem_objects_to_dandi, - get_interface_alignment, + upload_project_to_dandi, + validate_metadata, ) - -from errorHandlers import notBadRequestException - +from manageNeuroconv.info import announcer neuroconv_api = Namespace("neuroconv", description="Neuroconv neuroconv_api for the NWB GUIDE.") diff --git a/pyflask/apis/startup.py b/pyflask/apis/startup.py index feb59d871..63be4fd0a 100644 --- a/pyflask/apis/startup.py +++ b/pyflask/apis/startup.py @@ -1,8 +1,7 @@ """API endpoint definitions for startup operations.""" -from flask_restx import Namespace, Resource - from errorHandlers import notBadRequestException +from flask_restx import Namespace, Resource startup_api = Namespace("startup", description="API for startup commands related to the NWB GUIDE.") diff --git a/pyflask/app.py b/pyflask/app.py index 924e9454a..d84b27bae 100644 --- a/pyflask/app.py +++ b/pyflask/app.py @@ -1,28 +1,29 @@ """The primary Flask server for the Python backend.""" -import sys import json import multiprocessing -from os import kill, getpid -from os.path import isabs - -from signal import SIGINT -from logging import Formatter, DEBUG +import sys +from logging import DEBUG, Formatter from logging.handlers import RotatingFileHandler +from os import getpid, kill +from os.path import isabs from pathlib import Path +from signal import SIGINT from urllib.parse import unquote - # https://stackoverflow.com/questions/32672596/pyinstaller-loads-script-multiple-times#comment103216434_32677108 multiprocessing.freeze_support() -from flask import Flask, request, send_from_directory, send_file +from apis import data_api, neuroconv_api, startup_api +from flask import Flask, request, send_file, send_from_directory from flask_cors import CORS from flask_restx import Api, Resource - -from apis import startup_api, neuroconv_api, data_api -from manageNeuroconv.info import resource_path, STUB_SAVE_FOLDER_PATH, CONVERSION_SAVE_FOLDER_PATH +from manageNeuroconv.info import ( + CONVERSION_SAVE_FOLDER_PATH, + STUB_SAVE_FOLDER_PATH, + resource_path, +) app = Flask(__name__) diff --git a/pyflask/manageNeuroconv/__init__.py b/pyflask/manageNeuroconv/__init__.py index ddcc2b8b6..f9f55e082 100644 --- a/pyflask/manageNeuroconv/__init__.py +++ b/pyflask/manageNeuroconv/__init__.py @@ -1,23 +1,21 @@ +from .info import CONVERSION_SAVE_FOLDER_PATH, STUB_SAVE_FOLDER_PATH from .manage_neuroconv import ( - get_all_interface_info, - get_all_converter_info, - locate_data, autocomplete_format_string, - get_source_schema, - get_metadata_schema, convert_to_nwb, - validate_metadata, - upload_project_to_dandi, - upload_folder_to_dandi, - upload_multiple_filesystem_objects_to_dandi, - listen_to_neuroconv_events, generate_dataset, + generate_test_data, + get_all_converter_info, + get_all_interface_info, + get_interface_alignment, + get_metadata_schema, + get_source_schema, + inspect_multiple_filesystem_objects, inspect_nwb_file, inspect_nwb_folder, - inspect_multiple_filesystem_objects, - get_interface_alignment, - generate_test_data, + listen_to_neuroconv_events, + locate_data, + upload_folder_to_dandi, + upload_multiple_filesystem_objects_to_dandi, + upload_project_to_dandi, + validate_metadata, ) - - -from .info import STUB_SAVE_FOLDER_PATH, CONVERSION_SAVE_FOLDER_PATH diff --git a/pyflask/manageNeuroconv/info/__init__.py b/pyflask/manageNeuroconv/info/__init__.py index 915d74aee..edde04113 100644 --- a/pyflask/manageNeuroconv/info/__init__.py +++ b/pyflask/manageNeuroconv/info/__init__.py @@ -1,8 +1,7 @@ +from .sse import announcer, format_sse from .urls import ( - resource_path, + CONVERSION_SAVE_FOLDER_PATH, GUIDE_ROOT_FOLDER, STUB_SAVE_FOLDER_PATH, - CONVERSION_SAVE_FOLDER_PATH, + resource_path, ) - -from .sse import announcer, format_sse diff --git a/pyflask/manageNeuroconv/info/sse.py b/pyflask/manageNeuroconv/info/sse.py index bb90f34f7..b9593cba5 100644 --- a/pyflask/manageNeuroconv/info/sse.py +++ b/pyflask/manageNeuroconv/info/sse.py @@ -1,5 +1,5 @@ -import queue import json +import queue def format_sse(data: str, event=None) -> str: diff --git a/pyflask/manageNeuroconv/info/urls.py b/pyflask/manageNeuroconv/info/urls.py index 261f1188c..bf8a65116 100644 --- a/pyflask/manageNeuroconv/info/urls.py +++ b/pyflask/manageNeuroconv/info/urls.py @@ -1,7 +1,7 @@ -from pathlib import Path import json import os import sys +from pathlib import Path def resource_path(relative_path): diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index 860776276..dedabd683 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -8,11 +8,15 @@ import re from datetime import datetime from pathlib import Path -from shutil import rmtree, copytree -from typing import Any, Dict, List, Optional -from typing import Union +from shutil import copytree, rmtree +from typing import Any, Dict, List, Optional, Union -from .info import GUIDE_ROOT_FOLDER, STUB_SAVE_FOLDER_PATH, CONVERSION_SAVE_FOLDER_PATH, announcer +from .info import ( + CONVERSION_SAVE_FOLDER_PATH, + GUIDE_ROOT_FOLDER, + STUB_SAVE_FOLDER_PATH, + announcer, +) EXCLUDED_RECORDING_INTERFACE_PROPERTIES = ["contact_vector", "contact_shapes", "group", "location"] @@ -337,7 +341,7 @@ def get_all_interface_info() -> dict: # Combine Multiple Interfaces def get_custom_converter(interface_class_dict: dict): # -> NWBConverter: - from neuroconv import converters, datainterfaces, NWBConverter + from neuroconv import NWBConverter, converters, datainterfaces class CustomNWBConverter(NWBConverter): data_interface_classes = { @@ -477,8 +481,12 @@ def on_recording_interface(name, recording_interface): return recording_interface - from neuroconv.datainterfaces.ecephys.baserecordingextractorinterface import BaseRecordingExtractorInterface - from neuroconv.datainterfaces.ecephys.basesortingextractorinterface import BaseSortingExtractorInterface + from neuroconv.datainterfaces.ecephys.baserecordingextractorinterface import ( + BaseRecordingExtractorInterface, + ) + from neuroconv.datainterfaces.ecephys.basesortingextractorinterface import ( + BaseSortingExtractorInterface, + ) # Map recording interfaces to metadata map_interfaces(on_recording_interface, converter=converter, to_match=BaseRecordingExtractorInterface) @@ -605,7 +613,7 @@ def get_check_function(check_function_name: str) -> callable: def run_check_function(check_function: callable, arg: dict) -> dict: """.Function used to run an arbitrary NWB Inspector function.""" - from nwbinspector.register_checks import InspectorMessage, Importance + from nwbinspector.register_checks import Importance, InspectorMessage output = check_function(arg) if isinstance(output, InspectorMessage): @@ -648,8 +656,8 @@ def validate_nwbfile_metadata( def validate_metadata(metadata: dict, check_function_name: str) -> dict: """Function used to validate data using an arbitrary NWB Inspector function.""" - from pynwb.file import NWBFile, Subject from nwbinspector.nwbinspector import InspectorOutputJSONEncoder + from pynwb.file import NWBFile, Subject check_function = get_check_function(check_function_name) @@ -988,10 +996,10 @@ def _inspect_file_per_job( request_id: Optional[str] = None, ) -> list: + import requests from nwbinspector import nwbinspector from pynwb import NWBHDF5IO from tqdm_publisher import TQDMProgressSubscriber - import requests checks = nwbinspector.configure_checks( checks=nwbinspector.available_checks, @@ -1024,6 +1032,7 @@ def _inspect_file_per_job( def inspect_all(url, config): from concurrent.futures import ProcessPoolExecutor, as_completed + from nwbinspector.utils import calculate_number_of_cpu from tqdm_publisher import TQDMProgressSubscriber @@ -1085,10 +1094,11 @@ def on_progress_update(message): def inspect_nwb_folder(url, payload) -> dict: + from pickle import PicklingError + from nwbinspector import load_config from nwbinspector.inspector_tools import format_messages, get_report_header from nwbinspector.nwbinspector import InspectorOutputJSONEncoder - from pickle import PicklingError kwargs = dict( ignore=[ @@ -1211,7 +1221,7 @@ def generate_test_data(output_path: str): """ import spikeinterface from spikeinterface.exporters import export_to_phy - from spikeinterface.preprocessing import scale, bandpass_filter, resample + from spikeinterface.preprocessing import bandpass_filter, resample, scale base_path = Path(output_path) spikeglx_output_folder = base_path / "spikeglx" diff --git a/pyflask/tests/conftest.py b/pyflask/tests/conftest.py index 4ecf00e8b..ec72c792b 100644 --- a/pyflask/tests/conftest.py +++ b/pyflask/tests/conftest.py @@ -1,5 +1,5 @@ -import pytest import app as flask +import pytest def pytest_addoption(parser): diff --git a/pyflask/tests/test_generate_tutorial_data.py b/pyflask/tests/test_generate_tutorial_data.py index 80a5f8c53..d87e20281 100644 --- a/pyflask/tests/test_generate_tutorial_data.py +++ b/pyflask/tests/test_generate_tutorial_data.py @@ -1,6 +1,7 @@ -from utils import post from pathlib import Path +from utils import post + def test_generate_test_data(client, tmp_path: Path): # assert client is None diff --git a/pyflask/tests/test_neuroconv.py b/pyflask/tests/test_neuroconv.py index 7e6bd33e8..07563763e 100644 --- a/pyflask/tests/test_neuroconv.py +++ b/pyflask/tests/test_neuroconv.py @@ -1,5 +1,5 @@ from jsonschema import validate -from utils import get, post, get_converter_output_schema +from utils import get, get_converter_output_schema, post def test_get_all_interfaces(client): diff --git a/pyflask/tests/test_startup.py b/pyflask/tests/test_startup.py index 42cead8c7..8fa114535 100644 --- a/pyflask/tests/test_startup.py +++ b/pyflask/tests/test_startup.py @@ -1,4 +1,4 @@ -from utils import get, post, get_converter_output_schema +from utils import get, get_converter_output_schema, post def test_preload_imports(client):