From 222c744aecc9e689e9425173ef98058cea1e7ec9 Mon Sep 17 00:00:00 2001 From: Garrett Date: Wed, 18 Oct 2023 11:15:35 -0700 Subject: [PATCH 1/6] Expose converters and organize --- pyflask/apis/neuroconv.py | 9 +- pyflask/manageNeuroconv/__init__.py | 1 + pyflask/manageNeuroconv/manage_neuroconv.py | 63 ++++++---- src/renderer/src/stories/Search.js | 112 +++++++++++++++--- .../pages/guided-mode/data/GuidedStructure.js | 14 ++- 5 files changed, 157 insertions(+), 42 deletions(-) diff --git a/pyflask/apis/neuroconv.py b/pyflask/apis/neuroconv.py index 5ae604b3a..f1eb69ca9 100644 --- a/pyflask/apis/neuroconv.py +++ b/pyflask/apis/neuroconv.py @@ -6,6 +6,7 @@ from manageNeuroconv import ( get_all_interface_info, + get_all_converter_info, locate_data, get_source_schema, get_metadata_schema, @@ -40,7 +41,13 @@ class AllInterfaces(Resource): @neuroconv_api.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"}) def get(self): try: - return get_all_interface_info() + # return get_all_interface_info() + # return get_all_converter_info() + + return { + **get_all_interface_info(), + **get_all_converter_info(), + } except Exception as e: if notBadRequestException(e): neuroconv_api.abort(500, str(e)) diff --git a/pyflask/manageNeuroconv/__init__.py b/pyflask/manageNeuroconv/__init__.py index 38fd2077b..95cc9bc03 100644 --- a/pyflask/manageNeuroconv/__init__.py +++ b/pyflask/manageNeuroconv/__init__.py @@ -1,5 +1,6 @@ from .manage_neuroconv import ( get_all_interface_info, + get_all_converter_info, locate_data, get_source_schema, get_metadata_schema, diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index 8b0715c3c..34a7f6a7e 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -128,41 +128,60 @@ def locate_data(info: dict) -> dict: return organized_output + +def module_to_dict(my_module): + # Create an empty dictionary + module_dict = {} + + # Iterate through the module's attributes + for attr_name in dir(my_module): + if not attr_name.startswith("__"): # Exclude special attributes + attr_value = getattr(my_module, attr_name) + module_dict[attr_name] = attr_value + + return module_dict + +def get_all_converter_info() -> dict: + + from neuroconv import converters + + return { + name: { + "keywords": [], + "description": f"{converter.__doc__.split('.')[0]}." if converter.__doc__ else '' + } + for name, converter in module_to_dict(converters).items() + } + def get_all_interface_info() -> dict: """Format an information structure to be used for selecting interfaces based on modality and technique.""" from neuroconv.datainterfaces import interface_list exclude_interfaces_from_selection = [ # Deprecated - "SpikeGLXLFP", + "SpikeGLXLFPInterface", # Aliased - "CEDRecording", - "OpenEphysBinaryRecording", - "OpenEphysLegacyRecording", + "CEDRecordingInterface", + "OpenEphysBinaryRecordingInterface", + "OpenEphysLegacyRecordingInterface", # Ignored - "AxonaPositionData", - "AxonaUnitRecording", - "CsvTimeIntervals", - "ExcelTimeIntervals", - "Hdf5Imaging", - "MaxOneRecording", - "OpenEphysSorting", - "SimaSegmentation", - ] # Should have 'interface' stripped from name - - interfaces_to_load = {interface.__name__.replace("Interface", ""): interface for interface in interface_list} - for excluded_interface in exclude_interfaces_from_selection: - interfaces_to_load.pop(excluded_interface) + "AxonaPositionDataInterface", + "AxonaUnitRecordingInterface", + "CsvTimeIntervalsInterface", + "ExcelTimeIntervalsInterface", + "Hdf5ImagingInterface", + "MaxOneRecordingInterface", + "OpenEphysSortingInterface", + "SimaSegmentationInterface", + ] + return { interface.__name__: { "keywords": interface.keywords, - # Once we use the raw neuroconv list, we will want to ensure that the interfaces themselves - # have a label property - "label": format_name - # Can also add a description here if we want to provide more information about the interface + "description": f"{interface.__doc__.split('.')[0]}." if interface.__doc__ else '' } - for format_name, interface in interfaces_to_load.items() + for interface in interface_list if not interface.__name__ in exclude_interfaces_from_selection } diff --git a/src/renderer/src/stories/Search.js b/src/renderer/src/stories/Search.js index 90a8d7b54..c218d34d3 100644 --- a/src/renderer/src/stories/Search.js +++ b/src/renderer/src/stories/Search.js @@ -1,5 +1,6 @@ import { LitElement, html, css } from "lit"; -import { styleMap } from "lit/directives/style-map.js"; + +import tippy from "tippy.js"; export class Search extends LitElement { constructor({ options, showAllWhenEmpty, disabledLabel } = {}) { @@ -17,20 +18,18 @@ export class Search extends LitElement { :host { position: relative; - display: block; + display: flex; + flex-direction: column; background: white; border-radius: 5px; width: 100%; height: 100%; - overflow: auto; + overflow: hidden; } .header { padding: 25px; background: white; - position: sticky; - top: 0; - z-index: 1; } input { @@ -45,9 +44,8 @@ export class Search extends LitElement { list-style: none; padding: 0; margin: 0; - position: absolute; - left: 0; - right: 0; + position: relative; + overflow: auto; background: white; } @@ -56,6 +54,15 @@ export class Search extends LitElement { border-top: 1px solid #f2f2f2; } + .category{ + padding: 10px 25px; + background: #f2f2f2; + font-weight: bold; + position: sticky; + top: 0; + z-index: 1; + } + .option:hover { background: #f2f2f2; cursor: pointer; @@ -65,7 +72,12 @@ export class Search extends LitElement { margin: 0; } - [disabled] { + .label { + display: flex; + gap: 10px; + } + + [disabled]:not([hidden]) { display: flex; justify-content: space-between; align-items: center; @@ -118,7 +130,13 @@ export class Search extends LitElement { list = document.createElement("ul"); + categories = {} + render() { + + this.categories = {} + + // Update list this.list.remove(); this.list = document.createElement("ul"); @@ -130,7 +148,13 @@ export class Search extends LitElement { this.list.appendChild(slot); if (this.options) { - const unsupported = this.options + + const options = this.options.map(o => { return { + label: o.key, + ...o, + }}) + + const itemEls = options .sort((a, b) => { if (a.label < b.label) return -1; if (a.label > b.label) return 1; @@ -141,7 +165,7 @@ export class Search extends LitElement { else if (a.disabled) return 1; else if (b.disabled) return -1; }) // Sort with the disabled options at the bottom - .filter((option) => { + .map((option) => { const li = document.createElement("li"); li.classList.add("option"); li.setAttribute("hidden", ""); @@ -155,6 +179,23 @@ export class Search extends LitElement { const label = document.createElement("h4"); label.classList.add("label"); label.innerText = option.label; + + const info = document.createElement("span"); + + if (option.description) { + + info.innerText = 'ℹ️'; + label.append(info) + + tippy(info, { + content: `

${option.description}

`, + allowHTML: true, + placement: "right", + }) + } + + + container.appendChild(label); const keywords = document.createElement("small"); @@ -163,16 +204,43 @@ export class Search extends LitElement { container.appendChild(keywords); li.append(container); - this.list.appendChild(li); - return option.disabled; - }) - .map((o) => o.value); + if (option.category) { + let category = this.categories[option.category] + if (!category) { + category = document.createElement("div") + category.innerText = option.category + category.classList.add("category") + this.categories[option.category] = { + entries: [], + element: category, + } + } + + this.categories[option.category].entries.push(li) + return + } - console.warn(`Enabled: ${this.options.length - unsupported.length}/${this.options.length}`); - console.warn("Disabled Options:", unsupported); + + return el + + }).filter(el => el) + + this.list.append(...itemEls) } + + // Categories sorted alphabetically + const categories = Object.values(this.categories).sort((a, b) => { + if (a.element.innerText < b.element.innerText) return -1; + if (a.element.innerText > b.element.innerText) return 1; + return 0; + }) + + categories.forEach(({ entries, element }) => { + this.list.append(element, ...entries) + }) + return html`
{ @@ -204,6 +272,14 @@ export class Search extends LitElement { option.setAttribute("hidden", ""); } }); + + categories.forEach(({ entries, element }) => { + if (entries.reduce((acc, el) => acc + el.hasAttribute("hidden"), 0) === entries.length) element.setAttribute("hidden", "") + else element.removeAttribute("hidden") + }) + + + }}>
${this.list} diff --git a/src/renderer/src/stories/pages/guided-mode/data/GuidedStructure.js b/src/renderer/src/stories/pages/guided-mode/data/GuidedStructure.js index 648f91402..22f864c8a 100644 --- a/src/renderer/src/stories/pages/guided-mode/data/GuidedStructure.js +++ b/src/renderer/src/stories/pages/guided-mode/data/GuidedStructure.js @@ -10,6 +10,14 @@ import { List } from "../../../List"; const defaultEmptyMessage = "No interfaces selected"; +const categories = [{ + test: /.*Interface.*/, + value: 'Single-Stream Interfaces' +}, { + test: /.*Converter.*/, + value: 'Multi-Stream Converters' +}] + export class GuidedStructurePage extends Page { constructor(...args) { super(...args); @@ -91,10 +99,14 @@ export class GuidedStructurePage extends Page { .then((res) => res.json()) .then((json) => Object.entries(json).map(([key, value]) => { + + const category = categories.find(({ test }) => test.test(key))?.value + return { ...value, - key: key.replace("Interface", ""), + key, value: key, + category, disabled: !supportedInterfaces.includes(key), }; // Has label and keywords property already }) From 3efe84bff10d44f18805bafdf2025948ea55ed47 Mon Sep 17 00:00:00 2001 From: Garrett Date: Wed, 18 Oct 2023 11:46:07 -0700 Subject: [PATCH 2/6] Update backend and tests --- pyflask/manageNeuroconv/manage_neuroconv.py | 4 ++-- pyflask/tests/test_neuroconv.py | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index 34a7f6a7e..ea5df305d 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -187,11 +187,11 @@ def get_all_interface_info() -> dict: # Combine Multiple Interfaces def get_custom_converter(interface_class_dict: dict): # -> NWBConverter: - from neuroconv import datainterfaces, NWBConverter + from neuroconv import converters, datainterfaces, NWBConverter class CustomNWBConverter(NWBConverter): data_interface_classes = { - custom_name: getattr(datainterfaces, interface_name) + custom_name: getattr(datainterfaces, interface_name, getattr(converters, interface_name)) for custom_name, interface_name in interface_class_dict.items() } diff --git a/pyflask/tests/test_neuroconv.py b/pyflask/tests/test_neuroconv.py index 4f0e793f5..a2c8228dd 100644 --- a/pyflask/tests/test_neuroconv.py +++ b/pyflask/tests/test_neuroconv.py @@ -13,9 +13,11 @@ def test_get_all_interfaces(client): "type": "object", "properties": { "label": {"type": "string"}, + "description": {"type": "string"}, "keywords": {"type": "array", "items": {"type": "string"}}, }, - "required": ["label", "keywords"], + "additionalProperties": False, + "required": ["keywords"], } }, }, From 80cd2aebc57dc6d440bed8181153c998f4693c5d Mon Sep 17 00:00:00 2001 From: Garrett Date: Wed, 18 Oct 2023 11:46:20 -0700 Subject: [PATCH 3/6] Enable SpikeGLXConverterPipe as a test --- guideGlobalMetadata.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/guideGlobalMetadata.json b/guideGlobalMetadata.json index c0e1073c1..59571a3d6 100644 --- a/guideGlobalMetadata.json +++ b/guideGlobalMetadata.json @@ -36,6 +36,7 @@ "FicTracDataInterface", "AudioInterface", "MiniscopeBehaviorInterface", - "EDFRecordingInterface" + "EDFRecordingInterface", + "SpikeGLXConverterPipe" ] } From 892735d83a6849bd355ab899a25f73ca44582d68 Mon Sep 17 00:00:00 2001 From: Garrett Date: Wed, 18 Oct 2023 11:51:07 -0700 Subject: [PATCH 4/6] Update manage_neuroconv.py --- pyflask/manageNeuroconv/manage_neuroconv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index ea5df305d..ae9d7457d 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -191,7 +191,7 @@ def get_custom_converter(interface_class_dict: dict): # -> NWBConverter: class CustomNWBConverter(NWBConverter): data_interface_classes = { - custom_name: getattr(datainterfaces, interface_name, getattr(converters, interface_name)) + custom_name: getattr(datainterfaces, interface_name, getattr(converters, interface_name, None)) for custom_name, interface_name in interface_class_dict.items() } From faa65ba43f624db0c45cfe5a2c9b749df5346e5a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 18 Oct 2023 18:53:36 +0000 Subject: [PATCH 5/6] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pyflask/manageNeuroconv/manage_neuroconv.py | 15 ++-- src/renderer/src/stories/Search.js | 68 ++++++++----------- .../pages/guided-mode/data/GuidedStructure.js | 20 +++--- 3 files changed, 47 insertions(+), 56 deletions(-) diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index ae9d7457d..81d506c9a 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -128,7 +128,6 @@ def locate_data(info: dict) -> dict: return organized_output - def module_to_dict(my_module): # Create an empty dictionary module_dict = {} @@ -141,18 +140,16 @@ def module_to_dict(my_module): return module_dict -def get_all_converter_info() -> dict: +def get_all_converter_info() -> dict: from neuroconv import converters return { - name: { - "keywords": [], - "description": f"{converter.__doc__.split('.')[0]}." if converter.__doc__ else '' - } + name: {"keywords": [], "description": f"{converter.__doc__.split('.')[0]}." if converter.__doc__ else ""} for name, converter in module_to_dict(converters).items() } + def get_all_interface_info() -> dict: """Format an information structure to be used for selecting interfaces based on modality and technique.""" from neuroconv.datainterfaces import interface_list @@ -175,13 +172,13 @@ def get_all_interface_info() -> dict: "SimaSegmentationInterface", ] - return { interface.__name__: { "keywords": interface.keywords, - "description": f"{interface.__doc__.split('.')[0]}." if interface.__doc__ else '' + "description": f"{interface.__doc__.split('.')[0]}." if interface.__doc__ else "", } - for interface in interface_list if not interface.__name__ in exclude_interfaces_from_selection + for interface in interface_list + if not interface.__name__ in exclude_interfaces_from_selection } diff --git a/src/renderer/src/stories/Search.js b/src/renderer/src/stories/Search.js index c218d34d3..765d3b122 100644 --- a/src/renderer/src/stories/Search.js +++ b/src/renderer/src/stories/Search.js @@ -54,7 +54,7 @@ export class Search extends LitElement { border-top: 1px solid #f2f2f2; } - .category{ + .category { padding: 10px 25px; background: #f2f2f2; font-weight: bold; @@ -130,12 +130,10 @@ export class Search extends LitElement { list = document.createElement("ul"); - categories = {} + categories = {}; render() { - - this.categories = {} - + this.categories = {}; // Update list this.list.remove(); @@ -148,11 +146,12 @@ export class Search extends LitElement { this.list.appendChild(slot); if (this.options) { - - const options = this.options.map(o => { return { - label: o.key, - ...o, - }}) + const options = this.options.map((o) => { + return { + label: o.key, + ...o, + }; + }); const itemEls = options .sort((a, b) => { @@ -183,19 +182,16 @@ export class Search extends LitElement { const info = document.createElement("span"); if (option.description) { - - info.innerText = 'ℹ️'; - label.append(info) + info.innerText = "ℹ️"; + label.append(info); tippy(info, { content: `

${option.description}

`, allowHTML: true, placement: "right", - }) + }); } - - container.appendChild(label); const keywords = document.createElement("small"); @@ -206,40 +202,38 @@ export class Search extends LitElement { li.append(container); if (option.category) { - let category = this.categories[option.category] + let category = this.categories[option.category]; if (!category) { - category = document.createElement("div") - category.innerText = option.category - category.classList.add("category") + category = document.createElement("div"); + category.innerText = option.category; + category.classList.add("category"); this.categories[option.category] = { entries: [], element: category, - } + }; } - this.categories[option.category].entries.push(li) - return + this.categories[option.category].entries.push(li); + return; } + return el; + }) + .filter((el) => el); - return el - - }).filter(el => el) - - this.list.append(...itemEls) + this.list.append(...itemEls); } - // Categories sorted alphabetically const categories = Object.values(this.categories).sort((a, b) => { if (a.element.innerText < b.element.innerText) return -1; if (a.element.innerText > b.element.innerText) return 1; return 0; - }) + }); categories.forEach(({ entries, element }) => { - this.list.append(element, ...entries) - }) + this.list.append(element, ...entries); + }); return html`
@@ -274,12 +268,10 @@ export class Search extends LitElement { }); categories.forEach(({ entries, element }) => { - if (entries.reduce((acc, el) => acc + el.hasAttribute("hidden"), 0) === entries.length) element.setAttribute("hidden", "") - else element.removeAttribute("hidden") - }) - - - + if (entries.reduce((acc, el) => acc + el.hasAttribute("hidden"), 0) === entries.length) + element.setAttribute("hidden", ""); + else element.removeAttribute("hidden"); + }); }}>
${this.list} diff --git a/src/renderer/src/stories/pages/guided-mode/data/GuidedStructure.js b/src/renderer/src/stories/pages/guided-mode/data/GuidedStructure.js index 22f864c8a..9e7a6cd4c 100644 --- a/src/renderer/src/stories/pages/guided-mode/data/GuidedStructure.js +++ b/src/renderer/src/stories/pages/guided-mode/data/GuidedStructure.js @@ -10,13 +10,16 @@ import { List } from "../../../List"; const defaultEmptyMessage = "No interfaces selected"; -const categories = [{ - test: /.*Interface.*/, - value: 'Single-Stream Interfaces' -}, { - test: /.*Converter.*/, - value: 'Multi-Stream Converters' -}] +const categories = [ + { + test: /.*Interface.*/, + value: "Single-Stream Interfaces", + }, + { + test: /.*Converter.*/, + value: "Multi-Stream Converters", + }, +]; export class GuidedStructurePage extends Page { constructor(...args) { @@ -99,8 +102,7 @@ export class GuidedStructurePage extends Page { .then((res) => res.json()) .then((json) => Object.entries(json).map(([key, value]) => { - - const category = categories.find(({ test }) => test.test(key))?.value + const category = categories.find(({ test }) => test.test(key))?.value; return { ...value, From 623279fd93ad281d5699355425b7ab9a0ff71594 Mon Sep 17 00:00:00 2001 From: Garrett Date: Wed, 18 Oct 2023 12:46:18 -0700 Subject: [PATCH 6/6] Update manage_neuroconv.py --- pyflask/manageNeuroconv/manage_neuroconv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index ae9d7457d..5c380d7c3 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -391,7 +391,7 @@ def update_conversion_progress(**kwargs): options = ( { interface: {"stub_test": info["stub_test"]} # , "iter_opts": {"report_hook": update_conversion_progress}} - if available_options.get("properties").get(interface).get("properties").get("stub_test") + if available_options.get("properties").get(interface).get("properties", {}).get("stub_test") else {} for interface in info["source_data"] }