From 515c5897e3d65998e87dc8a091bdd01b5d8e9c4d Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Wed, 7 Feb 2024 13:47:23 -0500 Subject: [PATCH 01/24] add some draft electrode helpers --- pyflask/manageNeuroconv/manage_neuroconv.py | 56 +++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index 326577d57..19fd7ec8f 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -852,3 +852,59 @@ def generate_test_data(output_path: str): export_to_phy( waveform_extractor=waveform_extractor, output_folder=phy_output_folder, remove_if_exists=True, copy_binary=False ) + + + def get_electrode_columns_json(interface) -> List[Dict[str, Any]]: + """A convenience function for collecting and organizing the property values of the underlying recording extractor.""" + recording = interface.recording_extractor + + property_names = recording.get_property_keys() + + default_column_metadata = interface.get_metadata()["Ecephys"]["ElectrodeColumns"]["properties"] + property_descriptions = {column_name: column_fields ["description"] for column_name, column_fields in default_column_metadata} + + channel_ids = recording.get_channel_ids() + property_dtypes= {property_name: str(recording.get_property(key=property_name, ids=[channel_ids[0]]).dtype) for property_name in property_names} + + table = list() + for property_name in property_names: + table_row = dict(name=property_name, description=property_descriptions.get(property_name, ""), dtype=property_dtypes.get(property_name, "")) + table.append(electrode_column) + table_as_json = json.loads(json.dumps(obj=table)) + + return table_as_json + + def get_electrode_table_json(interface) -> List[Dict[str, Any]]: + """A convenience function for collecting and organizing the property values of the underlying recording extractor.""" + from neuroconv.utils.json_schema import NWBMetaDataEncoder + + recording = interface.recording_extractor + + property_names = set(recording.get_property_keys()) + electrode_ids = recording.get_channel_ids() + + table = list() + for electrode_id in electrode_ids: + electrode_column = dict() + for property_name in property_names: + recording_property_value = recording.get_property(key=property_name, ids=[electrode_id])[ + 0 # First axis is always electrodes in SI + ] # Since only fetching one electrode at a time, use trivial zero-index + electrode_column.update({property_name: recording_property_value}) + table.append(electrode_column) + table_as_json = json.loads(json.dumps(obj=table, cls=NWBMetaDataEncoder)) + + return table_as_json + + + def update_recording_properties_from_table_as_json(interface, electrode_table_as_json: List[Dict[str, Any]], column_table_as_json: List[Dict[str, Any]]) ->None: + """A convenience function for setting the property values of the underlying recording extractor.""" + recording = interface.recording_extractor + property_names = list(table_as_json[0].keys()) # Assumes no dict in the list will have missing or inconsitent keys + + for property_name in property_names: + dtype = column_table_as_json[property_name in row for row in column_table_as_json].index(True)]["data_type"] + property_values = np.array([row[propery_name] for row in table_as_json], dtype=dtype) + recording.set_property(key=property_name, values=property_values) + + return table_as_json From 4c83d506d540d1944fae659090ff62ad3f1266b5 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 7 Feb 2024 18:48:41 +0000 Subject: [PATCH 02/24] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pyflask/manageNeuroconv/manage_neuroconv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index 19fd7ec8f..f22e4c29a 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -896,7 +896,7 @@ def get_electrode_table_json(interface) -> List[Dict[str, Any]]: return table_as_json - + def update_recording_properties_from_table_as_json(interface, electrode_table_as_json: List[Dict[str, Any]], column_table_as_json: List[Dict[str, Any]]) ->None: """A convenience function for setting the property values of the underlying recording extractor.""" recording = interface.recording_extractor From cb1340b4d1c7780635165d6adfb7659712b09d18 Mon Sep 17 00:00:00 2001 From: Garrett Michael Flynn Date: Fri, 9 Feb 2024 10:25:39 -0800 Subject: [PATCH 03/24] Updated to allow visualization on the frontend. Has commented out code that doesn't work as intended --- pyflask/manageNeuroconv/manage_neuroconv.py | 202 ++++++++++++------ schemas/base-metadata.schema.ts | 5 +- .../pages/guided-mode/data/GuidedMetadata.js | 5 +- 3 files changed, 141 insertions(+), 71 deletions(-) diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index f22e4c29a..d5ea28e57 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -11,6 +11,7 @@ from typing import Dict, Optional from shutil import rmtree, copytree from pathlib import Path +from typing import Any, Dict, List, Optional from sse import MessageAnnouncer from .info import GUIDE_ROOT_FOLDER, STUB_SAVE_FOLDER_PATH, CONVERSION_SAVE_FOLDER_PATH @@ -244,22 +245,6 @@ def map_recording_interfaces(callback, converter): return output -def is_supported_recording_interface(recording_interface, metadata): - """ - Temporary conditioned access to functionality still in development on NeuroConv. - - Used to determine display of ecephys metadata depending on the environment. - - Alpha build release should therefore always return False for this. - """ - return ( - recording_interface - and recording_interface.get_electrode_table_json - and metadata["Ecephys"].get("Electrodes") - # and all(row.get("data_type") for row in metadata["Ecephys"]["Electrodes"]) - ) - - def get_metadata_schema(source_data: Dict[str, dict], interfaces: dict) -> Dict[str, dict]: """Function used to fetch the metadata schema from a CustomNWBConverter instantiated from the source_data.""" from neuroconv.utils import NWBMetaDataEncoder @@ -274,25 +259,43 @@ def get_metadata_schema(source_data: Dict[str, dict], interfaces: dict) -> Dict[ # Clear the Electrodes information for being set as a collection of Interfaces has_ecephys = "Ecephys" in metadata + if has_ecephys: metadata["Ecephys"]["Electrodes"] = {} + schema["properties"]["Ecephys"]["required"].append('Electrodes') ecephys_properties = schema["properties"]["Ecephys"]["properties"] original_electrodes_schema = ecephys_properties["Electrodes"] - ecephys_properties["Electrodes"] = {"type": "object", "properties": {}} + ecephys_properties["Electrodes"] = {"type": "object", "properties": {}, "required": []} defs = ecephys_properties["definitions"] def on_recording_interface(name, recording_interface): - metadata["Ecephys"]["Electrodes"][name] = recording_interface.get_electrode_table_json() + metadata["Ecephys"]["Electrodes"][name] = dict( + Electrodes = get_electrode_table_json(recording_interface), + ElectrodeColumns = get_electrode_columns_json(recording_interface) + ) + + ecephys_properties["Electrodes"]["properties"][name] = dict( + type = 'object', + properties = dict( + Electrodes = { + "type": "array", + "minItems": 0, + "items": {"$ref": "#/properties/Ecephys/properties/definitions/Electrode"}, + }, + ElectrodeColumns = { + "type": "array", + "minItems": 0, + "items": {"$ref": "#/properties/Ecephys/properties/definitions/ElectrodeColumn"}, + } + ), + required = ["Electrodes", "ElectrodeColumns"] + ) - ecephys_properties["Electrodes"]["properties"][name] = { - "type": "array", - "minItems": 0, - "items": {"$ref": "#/properties/Ecephys/properties/definitions/Electrode"}, - } + ecephys_properties["Electrodes"]["required"].append(name) return recording_interface @@ -308,14 +311,39 @@ def on_recording_interface(name, recording_interface): defs = ecephys_properties["definitions"] electrode_def = defs["Electrodes"] + numbers = ['int', 'float'] + n_bits = [ + "8", + "16" + "32", + "64" + ] + # NOTE: Update to output from NeuroConv - electrode_def["properties"]["dtype"] = {"type": "string", "enum": ["array", "int", "float", "bool", "str"]} + electrode_def["properties"]["data_type"] = { + "type": "string", + "enum": [ + "bool", + "str", + *[item for row in list(map(lambda bits: map(lambda type: f"{type}{bits}", numbers), n_bits)) for item in row] + ], + "enumLabels": { + "bool": "logical", + "str": "string", + "float8": "8-bit number", + "float16": "16-bit number", + "float32": "32-bit number", + "float64": "64-bit number", + "int8": "8-bit integer", + "int16": "16-bit integer", + "int32": "32-bit integer", + "int64": "64-bit integer" + } + } # Configure electrode columns - # NOTE: Update to output ALL columns and associated dtypes... - metadata["Ecephys"]["ElectrodeColumns"] = original_electrodes_schema["default"] - ecephys_properties["ElectrodeColumns"] = {"type": "array", "items": electrode_def} - ecephys_properties["ElectrodeColumns"]["items"]["required"] = list(electrode_def["properties"].keys()) + defs["ElectrodeColumn"] = electrode_def + defs["ElectrodeColumn"]["required"] = list(electrode_def["properties"].keys()) new_electrodes_properties = { properties["name"]: {key: value for key, value in properties.items() if key != "name"} @@ -492,7 +520,7 @@ def update_conversion_progress(**kwargs): interface = converter.data_interface_objects[interface_name] # NOTE: Must have a method to update the electrode table - # interface.update_electrode_table(electrode_table_json=electrode_results, electrode_column_info=electrode_column_results) + update_recording_properties_from_table_as_json(interface, electrode_table_json=electrode_results, electrode_column_info=electrode_column_results) # Update with the latest metadata for the electrodes ecephys_metadata["Electrodes"] = electrode_column_results @@ -854,57 +882,93 @@ def generate_test_data(output_path: str): ) - def get_electrode_columns_json(interface) -> List[Dict[str, Any]]: - """A convenience function for collecting and organizing the property values of the underlying recording extractor.""" - recording = interface.recording_extractor +## Ecephys Helper Functions +def get_electrode_properties(recording_interface): + """A convenience function for uniformly excluding certain properties of the provided recording extractor.""" - property_names = recording.get_property_keys() + return set(recording_interface.recording_extractor.get_property_keys()) - { + "location", # this is mapped to (rel_x,rel_y,(rel_z)))]), + "group", # this is auto-assigned as a link using the group_name + "contact_vector", # contains various probeinterface related info but not yet unpacked or fully used + } - default_column_metadata = interface.get_metadata()["Ecephys"]["ElectrodeColumns"]["properties"] - property_descriptions = {column_name: column_fields ["description"] for column_name, column_fields in default_column_metadata} - channel_ids = recording.get_channel_ids() - property_dtypes= {property_name: str(recording.get_property(key=property_name, ids=[channel_ids[0]]).dtype) for property_name in property_names} +def get_electrode_columns_json(interface) -> List[Dict[str, Any]]: + """A convenience function for collecting and organizing the property values of the underlying recording extractor.""" + recording = interface.recording_extractor + property_names = get_electrode_properties(interface) + + # Hardcuded for SpikeGLX + property_descriptions = dict( + channel_name="The name of this channel.", + group_name="The name of the ElectrodeGroup this channel's electrode is a part of.", + shank_electrode_number="0-based index of the electrode on the shank.", + contact_shapes="The shape of the electrode.", + inter_sample_shift="Time-delay of each channel sampling in proportion to the per-frame sampling period.", + gain_to_uV="The scaling factor from the data type to microVolts, applied before the offset.", + offset_to_uV="The offset from the data type to microVolts, applied after the gain.", + ) - table = list() - for property_name in property_names: - table_row = dict(name=property_name, description=property_descriptions.get(property_name, ""), dtype=property_dtypes.get(property_name, "")) - table.append(electrode_column) - table_as_json = json.loads(json.dumps(obj=table)) + # default_column_metadata = interface.get_metadata()["Ecephys"]["ElectrodeColumns"]["properties"] # NOTE: This doesn't exist... + # property_descriptions = {column_name: column_fields["description"] for column_name, column_fields in default_column_metadata} - return table_as_json + property_dtypes= {property_name: type(recording.get_property(key=property_name)[0]).__name__.replace("_", "") for property_name in property_names} + + # channel_ids = recording.get_channel_ids() + # property_dtypes= {property_name: str(recording.get_property(key=property_name, ids=[channel_ids[0]]).dtype) for property_name in property_names} - def get_electrode_table_json(interface) -> List[Dict[str, Any]]: - """A convenience function for collecting and organizing the property values of the underlying recording extractor.""" - from neuroconv.utils.json_schema import NWBMetaDataEncoder + # Return table as JSON + return json.loads(json.dumps(obj=[ + dict( + name=property_name, + description=property_descriptions.get(property_name, "No description."), + data_type=property_dtypes.get(property_name, ""), + ) + for property_name in property_names + ])) - recording = interface.recording_extractor - property_names = set(recording.get_property_keys()) - electrode_ids = recording.get_channel_ids() +def get_electrode_table_json(interface) -> List[Dict[str, Any]]: + """ + A convenience function for collecting and organizing the property values of the underlying recording extractor. + """ - table = list() - for electrode_id in electrode_ids: - electrode_column = dict() - for property_name in property_names: - recording_property_value = recording.get_property(key=property_name, ids=[electrode_id])[ - 0 # First axis is always electrodes in SI - ] # Since only fetching one electrode at a time, use trivial zero-index - electrode_column.update({property_name: recording_property_value}) - table.append(electrode_column) - table_as_json = json.loads(json.dumps(obj=table, cls=NWBMetaDataEncoder)) + from neuroconv.utils import NWBMetaDataEncoder - return table_as_json + recording = interface.recording_extractor + property_names = get_electrode_properties(interface) - def update_recording_properties_from_table_as_json(interface, electrode_table_as_json: List[Dict[str, Any]], column_table_as_json: List[Dict[str, Any]]) ->None: - """A convenience function for setting the property values of the underlying recording extractor.""" - recording = interface.recording_extractor - property_names = list(table_as_json[0].keys()) # Assumes no dict in the list will have missing or inconsitent keys + electrode_ids = recording.get_channel_ids() + table = list() + for electrode_id in electrode_ids: + electrode_column = dict() for property_name in property_names: - dtype = column_table_as_json[property_name in row for row in column_table_as_json].index(True)]["data_type"] - property_values = np.array([row[propery_name] for row in table_as_json], dtype=dtype) - recording.set_property(key=property_name, values=property_values) - - return table_as_json + recording_property_value = recording.get_property(key=property_name, ids=[electrode_id])[ + 0 # First axis is always electodes in SI + ] # Since only fetching one electrode at a time, use trivial zero-index + electrode_column.update({property_name: recording_property_value}) + table.append(electrode_column) + table_as_json = json.loads(json.dumps(table, cls=NWBMetaDataEncoder)) + + return table_as_json + +def update_recording_properties_from_table_as_json(recording_interface, electrode_column_info: dict, electrode_table_json: List[Dict[str, Any]]): + import numpy as np + + electrode_column_data_types = {column["name"]: column["data_type"] for column in electrode_column_info} + + recording = recording_interface.recording_extractor + channel_ids = recording.get_channel_ids() + stream_prefix = channel_ids[0].split("#")[0] # TODO: see if this generalized across formats + + for entry in electrode_table_json: + electrode_properties = dict(entry) # copy + channel_name = electrode_properties.pop("channel_name") + for property_name, property_value in electrode_properties.items(): + recording.set_property( + key=property_name, + values=np.array([property_value], dtype=electrode_column_data_types[property_name]), + ids=[stream_prefix + "#" + channel_name], + ) diff --git a/schemas/base-metadata.schema.ts b/schemas/base-metadata.schema.ts index 7bf1b712e..8e4552e06 100644 --- a/schemas/base-metadata.schema.ts +++ b/schemas/base-metadata.schema.ts @@ -101,7 +101,10 @@ export const preprocessMetadataSchema = (schema: any = baseMetadataSchema, globa // Change rendering order for electrode table columns const electrodesProp = ecephys.properties["Electrodes"] for (let name in electrodesProp.properties) { - electrodesProp.properties[name].items.order = ["channel_name", "group_name", "shank_electrode_number"]; + const interfaceProps = electrodesProp.properties[name].properties + interfaceProps["Electrodes"].items.order = ["channel_name", "group_name", "shank_electrode_number"]; + interfaceProps["ElectrodeColumns"].items.order = ["name", "description", "data_type"]; + } } diff --git a/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js b/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js index 2773d3f7e..65d3d61b6 100644 --- a/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js +++ b/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js @@ -204,6 +204,9 @@ export class GuidedMetadataPage extends ManagedPage { ); } + + console.log(schema) + // Create the form const form = new JSONSchemaForm({ identifier: instanceId, @@ -394,7 +397,7 @@ export class GuidedMetadataPage extends ManagedPage { metadata.schema = updatedSchema; // NOTE: Handsontable will occasionally have a context menu that doesn't actually trigger any behaviors - if (fullPath.slice(-1)[0] !== "Electrodes") return new SimpleTable(metadata); + if (name !== "Electrodes") return new SimpleTable(metadata); else return true; // All other tables are handled by the default behavior // if (name !== "ElectrodeColumns" && name !== "Electrodes") return new Table(metadata); }, From f5186fc21b9a44b12ea58c1483766e1ffa71a64f Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 9 Feb 2024 18:25:55 +0000 Subject: [PATCH 04/24] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pyflask/manageNeuroconv/manage_neuroconv.py | 73 +++++++++++-------- .../pages/guided-mode/data/GuidedMetadata.js | 3 +- 2 files changed, 43 insertions(+), 33 deletions(-) diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index d5ea28e57..da8388a40 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -263,7 +263,7 @@ def get_metadata_schema(source_data: Dict[str, dict], interfaces: dict) -> Dict[ if has_ecephys: metadata["Ecephys"]["Electrodes"] = {} - schema["properties"]["Ecephys"]["required"].append('Electrodes') + schema["properties"]["Ecephys"]["required"].append("Electrodes") ecephys_properties = schema["properties"]["Ecephys"]["properties"] original_electrodes_schema = ecephys_properties["Electrodes"] @@ -274,25 +274,25 @@ def get_metadata_schema(source_data: Dict[str, dict], interfaces: dict) -> Dict[ def on_recording_interface(name, recording_interface): metadata["Ecephys"]["Electrodes"][name] = dict( - Electrodes = get_electrode_table_json(recording_interface), - ElectrodeColumns = get_electrode_columns_json(recording_interface) + Electrodes=get_electrode_table_json(recording_interface), + ElectrodeColumns=get_electrode_columns_json(recording_interface), ) ecephys_properties["Electrodes"]["properties"][name] = dict( - type = 'object', - properties = dict( - Electrodes = { + type="object", + properties=dict( + Electrodes={ "type": "array", "minItems": 0, "items": {"$ref": "#/properties/Ecephys/properties/definitions/Electrode"}, }, - ElectrodeColumns = { + ElectrodeColumns={ "type": "array", "minItems": 0, "items": {"$ref": "#/properties/Ecephys/properties/definitions/ElectrodeColumn"}, - } + }, ), - required = ["Electrodes", "ElectrodeColumns"] + required=["Electrodes", "ElectrodeColumns"], ) ecephys_properties["Electrodes"]["required"].append(name) @@ -311,13 +311,8 @@ def on_recording_interface(name, recording_interface): defs = ecephys_properties["definitions"] electrode_def = defs["Electrodes"] - numbers = ['int', 'float'] - n_bits = [ - "8", - "16" - "32", - "64" - ] + numbers = ["int", "float"] + n_bits = ["8", "16" "32", "64"] # NOTE: Update to output from NeuroConv electrode_def["properties"]["data_type"] = { @@ -325,7 +320,11 @@ def on_recording_interface(name, recording_interface): "enum": [ "bool", "str", - *[item for row in list(map(lambda bits: map(lambda type: f"{type}{bits}", numbers), n_bits)) for item in row] + *[ + item + for row in list(map(lambda bits: map(lambda type: f"{type}{bits}", numbers), n_bits)) + for item in row + ], ], "enumLabels": { "bool": "logical", @@ -337,8 +336,8 @@ def on_recording_interface(name, recording_interface): "int8": "8-bit integer", "int16": "16-bit integer", "int32": "32-bit integer", - "int64": "64-bit integer" - } + "int64": "64-bit integer", + }, } # Configure electrode columns @@ -520,7 +519,9 @@ def update_conversion_progress(**kwargs): interface = converter.data_interface_objects[interface_name] # NOTE: Must have a method to update the electrode table - update_recording_properties_from_table_as_json(interface, electrode_table_json=electrode_results, electrode_column_info=electrode_column_results) + update_recording_properties_from_table_as_json( + interface, electrode_table_json=electrode_results, electrode_column_info=electrode_column_results + ) # Update with the latest metadata for the electrodes ecephys_metadata["Electrodes"] = electrode_column_results @@ -912,20 +913,27 @@ def get_electrode_columns_json(interface) -> List[Dict[str, Any]]: # default_column_metadata = interface.get_metadata()["Ecephys"]["ElectrodeColumns"]["properties"] # NOTE: This doesn't exist... # property_descriptions = {column_name: column_fields["description"] for column_name, column_fields in default_column_metadata} - property_dtypes= {property_name: type(recording.get_property(key=property_name)[0]).__name__.replace("_", "") for property_name in property_names} - + property_dtypes = { + property_name: type(recording.get_property(key=property_name)[0]).__name__.replace("_", "") + for property_name in property_names + } + # channel_ids = recording.get_channel_ids() # property_dtypes= {property_name: str(recording.get_property(key=property_name, ids=[channel_ids[0]]).dtype) for property_name in property_names} # Return table as JSON - return json.loads(json.dumps(obj=[ - dict( - name=property_name, - description=property_descriptions.get(property_name, "No description."), - data_type=property_dtypes.get(property_name, ""), + return json.loads( + json.dumps( + obj=[ + dict( + name=property_name, + description=property_descriptions.get(property_name, "No description."), + data_type=property_dtypes.get(property_name, ""), + ) + for property_name in property_names + ] ) - for property_name in property_names - ])) + ) def get_electrode_table_json(interface) -> List[Dict[str, Any]]: @@ -954,7 +962,10 @@ def get_electrode_table_json(interface) -> List[Dict[str, Any]]: return table_as_json -def update_recording_properties_from_table_as_json(recording_interface, electrode_column_info: dict, electrode_table_json: List[Dict[str, Any]]): + +def update_recording_properties_from_table_as_json( + recording_interface, electrode_column_info: dict, electrode_table_json: List[Dict[str, Any]] +): import numpy as np electrode_column_data_types = {column["name"]: column["data_type"] for column in electrode_column_info} @@ -962,7 +973,7 @@ def update_recording_properties_from_table_as_json(recording_interface, electrod recording = recording_interface.recording_extractor channel_ids = recording.get_channel_ids() stream_prefix = channel_ids[0].split("#")[0] # TODO: see if this generalized across formats - + for entry in electrode_table_json: electrode_properties = dict(entry) # copy channel_name = electrode_properties.pop("channel_name") diff --git a/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js b/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js index 65d3d61b6..fdeec53c1 100644 --- a/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js +++ b/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js @@ -204,9 +204,8 @@ export class GuidedMetadataPage extends ManagedPage { ); } + console.log(schema); - console.log(schema) - // Create the form const form = new JSONSchemaForm({ identifier: instanceId, From e39ccd714b4ea63e043fd9aa73d927b185848f7a Mon Sep 17 00:00:00 2001 From: Garrett Michael Flynn Date: Mon, 12 Feb 2024 11:39:22 -0800 Subject: [PATCH 05/24] Generalized handler substantially --- pyflask/manageNeuroconv/manage_neuroconv.py | 99 +++++++++---------- src/renderer/src/stories/BasicTable.js | 9 +- src/renderer/src/stories/JSONSchemaForm.js | 5 +- .../pages/guided-mode/data/GuidedMetadata.js | 8 +- 4 files changed, 61 insertions(+), 60 deletions(-) diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index bee606ee3..62fac02d7 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -309,33 +309,26 @@ def on_recording_interface(name, recording_interface): defs = ecephys_properties["definitions"] electrode_def = defs["Electrodes"] - numbers = ["int", "float"] - n_bits = ["8", "16" "32", "64"] + dtype_descriptions = { + "bool": "logical", + "str": "string", + "ndarray": "n-dimensional array", + "float8": "8-bit number", + "float16": "16-bit number", + "float32": "32-bit number", + "float64": "64-bit number", + "int8": "8-bit integer", + "int16": "16-bit integer", + "int32": "32-bit integer", + "int64": "64-bit integer", + } # NOTE: Update to output from NeuroConv electrode_def["properties"]["data_type"] = { "type": "string", - "enum": [ - "bool", - "str", - *[ - item - for row in list(map(lambda bits: map(lambda type: f"{type}{bits}", numbers), n_bits)) - for item in row - ], - ], - "enumLabels": { - "bool": "logical", - "str": "string", - "float8": "8-bit number", - "float16": "16-bit number", - "float32": "32-bit number", - "float64": "64-bit number", - "int8": "8-bit integer", - "int16": "16-bit integer", - "int32": "32-bit integer", - "int64": "64-bit integer", - }, + "strict": False, + "enum": list(dtype_descriptions.keys()), + "enumLabels": dtype_descriptions, } # Configure electrode columns @@ -502,29 +495,21 @@ def update_conversion_progress(**kwargs): else None ) - if "Ecephys" not in info["metadata"]: - info["metadata"].update(Ecephys=dict()) - # Ensure Ophys NaN values are resolved resolved_metadata = replace_none_with_nan(info["metadata"], resolve_references(converter.get_metadata_schema())) - ecephys_metadata = resolved_metadata["Ecephys"] - electrode_column_results = ecephys_metadata[ - "ElectrodeColumns" - ] # NOTE: Need more specificity from the ElectrodeColumns (e.g. dtype, not always provided...) + ecephys_metadata = resolved_metadata.get('Ecephys') - for interface_name, electrode_results in ecephys_metadata["Electrodes"].items(): - interface = converter.data_interface_objects[interface_name] + if ecephys_metadata: - # NOTE: Must have a method to update the electrode table - update_recording_properties_from_table_as_json( - interface, electrode_table_json=electrode_results, electrode_column_info=electrode_column_results - ) + for interface_name, interface_electrode_results in ecephys_metadata["Electrodes"].items(): + interface = converter.data_interface_objects[interface_name] - # Update with the latest metadata for the electrodes - ecephys_metadata["Electrodes"] = electrode_column_results + update_recording_properties_from_table_as_json( + interface, electrode_table_json=interface_electrode_results["Electrodes"], electrode_column_info=interface_electrode_results["ElectrodeColumns"] + ) - ecephys_metadata.pop("ElectrodeColumns", dict()) + del ecephys_metadata["Electrodes"] # NOTE: Not sure what this should be now... # Actually run the conversion converter.run_conversion( @@ -881,23 +866,34 @@ def generate_test_data(output_path: str): ) +dtype_map = { + " List[Dict[str, Any]]: """A convenience function for collecting and organizing the property values of the underlying recording extractor.""" - recording = interface.recording_extractor + property_names = get_electrode_properties(interface) - # Hardcuded for SpikeGLX + # Hardcuded for SpikeGLX (NOTE: Update for more interfaces) property_descriptions = dict( channel_name="The name of this channel.", group_name="The name of the ElectrodeGroup this channel's electrode is a part of.", @@ -911,13 +907,8 @@ def get_electrode_columns_json(interface) -> List[Dict[str, Any]]: # default_column_metadata = interface.get_metadata()["Ecephys"]["ElectrodeColumns"]["properties"] # NOTE: This doesn't exist... # property_descriptions = {column_name: column_fields["description"] for column_name, column_fields in default_column_metadata} - property_dtypes = { - property_name: type(recording.get_property(key=property_name)[0]).__name__.replace("_", "") - for property_name in property_names - } - - # channel_ids = recording.get_channel_ids() - # property_dtypes= {property_name: str(recording.get_property(key=property_name, ids=[channel_ids[0]]).dtype) for property_name in property_names} + recording = interface.recording_extractor + channel_ids = recording.get_channel_ids() # Return table as JSON return json.loads( @@ -926,7 +917,7 @@ def get_electrode_columns_json(interface) -> List[Dict[str, Any]]: dict( name=property_name, description=property_descriptions.get(property_name, "No description."), - data_type=property_dtypes.get(property_name, ""), + data_type=get_property_dtype(interface, property_name, [ channel_ids[0] ]), ) for property_name in property_names ] diff --git a/src/renderer/src/stories/BasicTable.js b/src/renderer/src/stories/BasicTable.js index 4a400334c..fb9ff6453 100644 --- a/src/renderer/src/stories/BasicTable.js +++ b/src/renderer/src/stories/BasicTable.js @@ -9,6 +9,7 @@ import * as promises from "../promises"; import "./Button"; import { sortTable } from "./Table"; import tippy from "tippy.js"; +import { getIgnore } from "./JSONSchemaForm"; export class BasicTable extends LitElement { static get styles() { @@ -368,8 +369,6 @@ export class BasicTable extends LitElement { this.#updateRendered(); const entries = this.#itemProps; - for (let key in this.ignore) delete entries[key]; - for (let key in this.ignore["*"] ?? {}) delete entries[key]; // Add existing additional properties to the entries variable if necessary if (this.#itemSchema.additionalProperties) { @@ -385,6 +384,10 @@ export class BasicTable extends LitElement { }, entries); } + // Ignore any additions in the ignore configuration + for (let key in this.ignore) delete entries[key]; + for (let key in this.ignore["*"] ?? {}) delete entries[key]; + // Sort Columns by Key Column and Requirement const keys = (this.#keys = @@ -419,7 +422,7 @@ export class BasicTable extends LitElement { ${data.map( (row, i) => html` - ${row.map((col, j) => html`
${col}
`)} + ${row.map((col, j) => html`
${JSON.stringify(col)}
`)} ` )} diff --git a/src/renderer/src/stories/JSONSchemaForm.js b/src/renderer/src/stories/JSONSchemaForm.js index 7bcca50af..cb9aca4d6 100644 --- a/src/renderer/src/stories/JSONSchemaForm.js +++ b/src/renderer/src/stories/JSONSchemaForm.js @@ -388,18 +388,19 @@ export class JSONSchemaForm extends LitElement { const isRow = typeof rowName === "number"; const resolvedValue = e.path.reduce((acc, token) => acc[token], resolved); + const resolvedSchema = this.getSchema(e.path, schema) // ------------ Exclude Certain Errors ------------ // Allow for constructing types from object types - if (e.message.includes("is not of a type(s)") && "properties" in schema && schema.type === "string") + if (e.message.includes("is not of a type(s)") && "properties" in resolvedSchema && resolvedSchema.type === "string") return; // Ignore required errors if value is empty if (e.name === "required" && !this.validateEmptyValues && !(e.property in e.instance)) return; // Non-Strict Rule - if (schema.strict === false && e.message.includes("is not one of enum values")) return; + if (resolvedSchema.strict === false && e.message.includes("is not one of enum values")) return; // Allow referring to floats as null (i.e. JSON NaN representation) if (e.message === "is not of a type(s) number") { diff --git a/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js b/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js index 25906d809..1cb868351 100644 --- a/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js +++ b/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js @@ -28,7 +28,6 @@ import globalIcon from "../../../assets/global.svg?raw"; const imagingPlaneKey = "imaging_plane"; const propsToIgnore = { Ophys: { - // NOTE: Get this to work "*": { starting_time: true, rate: true, @@ -59,6 +58,13 @@ const propsToIgnore = { UnitProperties: true, ElectricalSeriesLF: true, ElectricalSeriesAP: true, + Electrodes: { + '*': { + location: true, + group: true, + contact_vector: true + } + } }, Icephys: true, // Always ignore icephys metadata (for now) Behavior: true, // Always ignore behavior metadata (for now) From 1060fd9a854afb46653ab3f5a004e2aa2bb5dfa6 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 12 Feb 2024 19:39:37 +0000 Subject: [PATCH 06/24] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pyflask/manageNeuroconv/manage_neuroconv.py | 19 ++++++++++--------- src/renderer/src/stories/BasicTable.js | 12 +++++++----- src/renderer/src/stories/JSONSchemaForm.js | 8 ++++++-- .../pages/guided-mode/data/GuidedMetadata.js | 8 ++++---- 4 files changed, 27 insertions(+), 20 deletions(-) diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index 62fac02d7..8f43e968b 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -498,7 +498,7 @@ def update_conversion_progress(**kwargs): # Ensure Ophys NaN values are resolved resolved_metadata = replace_none_with_nan(info["metadata"], resolve_references(converter.get_metadata_schema())) - ecephys_metadata = resolved_metadata.get('Ecephys') + ecephys_metadata = resolved_metadata.get("Ecephys") if ecephys_metadata: @@ -506,10 +506,12 @@ def update_conversion_progress(**kwargs): interface = converter.data_interface_objects[interface_name] update_recording_properties_from_table_as_json( - interface, electrode_table_json=interface_electrode_results["Electrodes"], electrode_column_info=interface_electrode_results["ElectrodeColumns"] + interface, + electrode_table_json=interface_electrode_results["Electrodes"], + electrode_column_info=interface_electrode_results["ElectrodeColumns"], ) - del ecephys_metadata["Electrodes"] # NOTE: Not sure what this should be now... + del ecephys_metadata["Electrodes"] # NOTE: Not sure what this should be now... # Actually run the conversion converter.run_conversion( @@ -866,12 +868,10 @@ def generate_test_data(output_path: str): ) -dtype_map = { - " List[Dict[str, Any]]: dict( name=property_name, description=property_descriptions.get(property_name, "No description."), - data_type=get_property_dtype(interface, property_name, [ channel_ids[0] ]), + data_type=get_property_dtype(interface, property_name, [channel_ids[0]]), ) for property_name in property_names ] diff --git a/src/renderer/src/stories/BasicTable.js b/src/renderer/src/stories/BasicTable.js index fb9ff6453..957b83d66 100644 --- a/src/renderer/src/stories/BasicTable.js +++ b/src/renderer/src/stories/BasicTable.js @@ -384,10 +384,10 @@ export class BasicTable extends LitElement { }, entries); } - // Ignore any additions in the ignore configuration - for (let key in this.ignore) delete entries[key]; - for (let key in this.ignore["*"] ?? {}) delete entries[key]; - + // Ignore any additions in the ignore configuration + for (let key in this.ignore) delete entries[key]; + for (let key in this.ignore["*"] ?? {}) delete entries[key]; + // Sort Columns by Key Column and Requirement const keys = (this.#keys = @@ -422,7 +422,9 @@ export class BasicTable extends LitElement { ${data.map( (row, i) => html` - ${row.map((col, j) => html`
${JSON.stringify(col)}
`)} + ${row.map( + (col, j) => html`
${JSON.stringify(col)}
` + )} ` )} diff --git a/src/renderer/src/stories/JSONSchemaForm.js b/src/renderer/src/stories/JSONSchemaForm.js index cb9aca4d6..2d4978ad4 100644 --- a/src/renderer/src/stories/JSONSchemaForm.js +++ b/src/renderer/src/stories/JSONSchemaForm.js @@ -388,12 +388,16 @@ export class JSONSchemaForm extends LitElement { const isRow = typeof rowName === "number"; const resolvedValue = e.path.reduce((acc, token) => acc[token], resolved); - const resolvedSchema = this.getSchema(e.path, schema) + const resolvedSchema = this.getSchema(e.path, schema); // ------------ Exclude Certain Errors ------------ // Allow for constructing types from object types - if (e.message.includes("is not of a type(s)") && "properties" in resolvedSchema && resolvedSchema.type === "string") + if ( + e.message.includes("is not of a type(s)") && + "properties" in resolvedSchema && + resolvedSchema.type === "string" + ) return; // Ignore required errors if value is empty diff --git a/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js b/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js index 1cb868351..b007a24b6 100644 --- a/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js +++ b/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js @@ -59,12 +59,12 @@ const propsToIgnore = { ElectricalSeriesLF: true, ElectricalSeriesAP: true, Electrodes: { - '*': { + "*": { location: true, group: true, - contact_vector: true - } - } + contact_vector: true, + }, + }, }, Icephys: true, // Always ignore icephys metadata (for now) Behavior: true, // Always ignore behavior metadata (for now) From eae9aa4b3cd234d819222b3b9ad505910505e7ef Mon Sep 17 00:00:00 2001 From: CodyCBakerPhD Date: Wed, 28 Feb 2024 00:18:37 -0500 Subject: [PATCH 07/24] unpack contact vector --- .gitignore | 3 + pyflask/manageNeuroconv/manage_neuroconv.py | 65 +++++++++++++-------- 2 files changed, 44 insertions(+), 24 deletions(-) diff --git a/.gitignore b/.gitignore index beacf96bb..613226a0c 100644 --- a/.gitignore +++ b/.gitignore @@ -36,3 +36,6 @@ src/build .env .env.local .env.production + +# Spyder +.spyproject/ \ No newline at end of file diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index b80d473e8..2ad350cc4 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -100,7 +100,9 @@ def coerce_schema_compliance_recursive(obj, schema): elif key in schema.get("properties", {}): prop_schema = schema["properties"][key] if prop_schema.get("type") == "number" and (value is None or value == "NaN"): - obj[key] = ( + obj[ + key + ] = ( math.nan ) # Turn None into NaN if a number is expected (JavaScript JSON.stringify turns NaN into None) elif prop_schema.get("type") == "number" and isinstance(value, int): @@ -929,28 +931,29 @@ def generate_test_data(output_path: str): dtype_map = {" Dict[str, Any]: """A convenience function for uniformly excluding certain properties of the provided recording extractor.""" - - properties = set(recording_interface.recording_extractor.get_property_keys()) + property_names = list(recording_interface.recording_extractor.get_property_keys()) + properties = { + property_name: recording_interface.recording_extractor.get_property(key=property_name) + for property_name in property_names + } return properties def get_electrode_columns_json(interface) -> List[Dict[str, Any]]: - """A convenience function for collecting and organizing the property values of the underlying recording extractor.""" - - property_names = get_electrode_properties(interface) + """A convenience function for collecting and organizing the properties of the underlying recording extractor.""" + properties = get_recording_interface_properties(interface) # Hardcuded for SpikeGLX (NOTE: Update for more interfaces) property_descriptions = dict( @@ -969,19 +972,33 @@ def get_electrode_columns_json(interface) -> List[Dict[str, Any]]: recording = interface.recording_extractor channel_ids = recording.get_channel_ids() - # Return table as JSON - return json.loads( - json.dumps( - obj=[ - dict( - name=property_name, - description=property_descriptions.get(property_name, "No description."), - data_type=get_property_dtype(interface, property_name, [channel_ids[0]]), - ) - for property_name in property_names - ] + contact_vector = properties.pop("contact_vector", None) + + electrode_columns = [ + dict( + name=property_name, + description=property_descriptions.get(property_name, "No description."), + data_type=get_property_dtype( + recording_extractor=recording, property_name=property_name, channel_ids=[channel_ids[0]] + ), ) - ) + for property_name in properties.keys() + ] + + if contact_vector is None: + return json.loads(json.dumps(obj=electrode_columns)) + + # Unpack contact vector + for property_name in contact_vector.dtype.names: + electrode_columns.append( + dict( + name=property_name, + description=property_descriptions.get(property_name, ""), + dtype=str(contact_vector.dtype.fields[property_name][0]), + ) + ) + + return json.loads(json.dumps(obj=electrode_columns)) def get_electrode_table_json(interface) -> List[Dict[str, Any]]: @@ -993,7 +1010,7 @@ def get_electrode_table_json(interface) -> List[Dict[str, Any]]: recording = interface.recording_extractor - property_names = get_electrode_properties(interface) + property_names = get_recording_interface_properties(interface) electrode_ids = recording.get_channel_ids() From d421849b98aac1f622ba6dcd1e1b531a0693f854 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 28 Feb 2024 05:18:53 +0000 Subject: [PATCH 08/24] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .gitignore | 2 +- pyflask/manageNeuroconv/manage_neuroconv.py | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 613226a0c..827e5b48e 100644 --- a/.gitignore +++ b/.gitignore @@ -38,4 +38,4 @@ src/build .env.production # Spyder -.spyproject/ \ No newline at end of file +.spyproject/ diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index 2ad350cc4..7995f931e 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -100,9 +100,7 @@ def coerce_schema_compliance_recursive(obj, schema): elif key in schema.get("properties", {}): prop_schema = schema["properties"][key] if prop_schema.get("type") == "number" and (value is None or value == "NaN"): - obj[ - key - ] = ( + obj[key] = ( math.nan ) # Turn None into NaN if a number is expected (JavaScript JSON.stringify turns NaN into None) elif prop_schema.get("type") == "number" and isinstance(value, int): From 42453278ddb2fd6a2fcfd334097240421365acbe Mon Sep 17 00:00:00 2001 From: Garrett Michael Flynn Date: Wed, 28 Feb 2024 16:25:37 -0800 Subject: [PATCH 09/24] Add option to set DANDI_CACHE environment variable to "ignore" (#623) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> --- pyflask/manageNeuroconv/manage_neuroconv.py | 12 ++++++++++++ schemas/json/dandi/upload.json | 8 +++++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index a47a94db9..26432a513 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -582,11 +582,17 @@ def upload_folder_to_dandi( cleanup: Optional[bool] = None, number_of_jobs: Optional[int] = None, number_of_threads: Optional[int] = None, + ignore_cache: bool = False, ): from neuroconv.tools.data_transfers import automatic_dandi_upload os.environ["DANDI_API_KEY"] = api_key # Update API Key + if ignore_cache: + os.environ["DANDI_CACHE"] = "ignore" + else: + os.environ["DANDI_CACHE"] = "" + return automatic_dandi_upload( dandiset_id=dandiset_id, nwb_folder_path=Path(nwb_folder_path), @@ -605,6 +611,7 @@ def upload_project_to_dandi( cleanup: Optional[bool] = None, number_of_jobs: Optional[int] = None, number_of_threads: Optional[int] = None, + ignore_cache: bool = False, ): from neuroconv.tools.data_transfers import automatic_dandi_upload @@ -612,6 +619,11 @@ def upload_project_to_dandi( os.environ["DANDI_API_KEY"] = api_key # Update API Key + if ignore_cache: + os.environ["DANDI_CACHE"] = "ignore" + else: + os.environ["DANDI_CACHE"] = "" + return automatic_dandi_upload( dandiset_id=dandiset_id, nwb_folder_path=CONVERSION_SAVE_FOLDER_PATH / project, # Scope valid DANDI upload paths to GUIDE projects diff --git a/schemas/json/dandi/upload.json b/schemas/json/dandi/upload.json index ced194f5e..6c1e60811 100644 --- a/schemas/json/dandi/upload.json +++ b/schemas/json/dandi/upload.json @@ -20,9 +20,15 @@ "min": 1, "default": 1 }, + "ignore_cache": { + "type": "boolean", + "description": "Ignore the cache used by DANDI to speed up repeated operations.", + "default": false + }, "cleanup": { "type": "boolean", - "title": "Delete Local Files After Upload", + "title": "Cleanup Local Filesystem", + "description": "Delete local files after upload", "default": false } } From 8e0589bfd6b952827a42d5c0f593b461ee04b299 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Thu, 29 Feb 2024 07:36:52 -0500 Subject: [PATCH 10/24] Version bump (#631) --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index ceea93333..dc69a9e93 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "nwb-guide", "productName": "NWB GUIDE", - "version": "0.0.14", + "version": "0.0.15", "description": "NWB GUIDE is a desktop app that provides a no-code user interface for converting neurophysiology data to NWB.", "main": "./build/main/main.js", "engine": { From 58d069962cce20665207ef3f154b6c4dfa753ee8 Mon Sep 17 00:00:00 2001 From: Garrett Michael Flynn Date: Thu, 29 Feb 2024 11:08:16 -0800 Subject: [PATCH 11/24] Propertly unfold dtypes in contact_vector, but provide original info when submitted --- pyflask/manageNeuroconv/manage_neuroconv.py | 39 +++++++++++++++---- .../pages/guided-mode/data/GuidedMetadata.js | 2 - 2 files changed, 31 insertions(+), 10 deletions(-) diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index 7995f931e..e51d12351 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -991,8 +991,8 @@ def get_electrode_columns_json(interface) -> List[Dict[str, Any]]: electrode_columns.append( dict( name=property_name, - description=property_descriptions.get(property_name, ""), - dtype=str(contact_vector.dtype.fields[property_name][0]), + description=property_descriptions.get(property_name, "No description."), + data_type=str(contact_vector.dtype.fields[property_name][0]), ) ) @@ -1031,7 +1031,27 @@ def update_recording_properties_from_table_as_json( ): import numpy as np - electrode_column_data_types = {column["name"]: column["data_type"] for column in electrode_column_info} + # # Extract contact vector properties + properties = get_recording_interface_properties(recording_interface) + contact_vector = properties.pop("contact_vector", None) + + contact_vector_dtypes = {} + + if contact_vector is not None: + + # Remove names from contact vector from the electrode_column_info and add to reconstructed_contact_vector_info + contact_vector_dtypes = contact_vector.dtype + # contact_vector_dtypes = { property_name: next((item for item in electrode_column_info if item['name'] == property_name), None)["data_type"] for property_name in contact_vector.dtype.names} + + # Remove contact vector properties from electrode_column_info + for property_name in contact_vector.dtype.names: + found = next((item for item in electrode_column_info if item['name'] == property_name), None) + if (found): + electrode_column_info.remove(found) + + # Organize dtypes + electrode_column_data_types = { column["name"]: column["data_type"] for column in electrode_column_info } + electrode_column_data_types["contact_vector"] = contact_vector_dtypes # Provide contact vector information recording = recording_interface.recording_extractor channel_ids = recording.get_channel_ids() @@ -1041,8 +1061,11 @@ def update_recording_properties_from_table_as_json( electrode_properties = dict(entry) # copy channel_name = electrode_properties.pop("channel_name") for property_name, property_value in electrode_properties.items(): - recording.set_property( - key=property_name, - values=np.array([property_value], dtype=electrode_column_data_types[property_name]), - ids=[stream_prefix + "#" + channel_name], - ) + + # Skip data with missing column information + if (property_name in electrode_column_data_types): + recording.set_property( + key=property_name, + values=np.array([property_value], dtype=electrode_column_data_types[property_name]), + ids=[stream_prefix + "#" + channel_name], + ) diff --git a/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js b/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js index a2aa7bbfc..65e8cbd07 100644 --- a/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js +++ b/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js @@ -208,8 +208,6 @@ export class GuidedMetadataPage extends ManagedPage { ); } - console.log(schema); - // Create the form const form = new JSONSchemaForm({ identifier: instanceId, From 66b026248bb008180d0cce99d8e12707b29946d5 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 29 Feb 2024 19:09:11 +0000 Subject: [PATCH 12/24] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pyflask/manageNeuroconv/manage_neuroconv.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index e51d12351..aad1a6c49 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -1045,13 +1045,13 @@ def update_recording_properties_from_table_as_json( # Remove contact vector properties from electrode_column_info for property_name in contact_vector.dtype.names: - found = next((item for item in electrode_column_info if item['name'] == property_name), None) - if (found): + found = next((item for item in electrode_column_info if item["name"] == property_name), None) + if found: electrode_column_info.remove(found) # Organize dtypes - electrode_column_data_types = { column["name"]: column["data_type"] for column in electrode_column_info } - electrode_column_data_types["contact_vector"] = contact_vector_dtypes # Provide contact vector information + electrode_column_data_types = {column["name"]: column["data_type"] for column in electrode_column_info} + electrode_column_data_types["contact_vector"] = contact_vector_dtypes # Provide contact vector information recording = recording_interface.recording_extractor channel_ids = recording.get_channel_ids() @@ -1063,7 +1063,7 @@ def update_recording_properties_from_table_as_json( for property_name, property_value in electrode_properties.items(): # Skip data with missing column information - if (property_name in electrode_column_data_types): + if property_name in electrode_column_data_types: recording.set_property( key=property_name, values=np.array([property_value], dtype=electrode_column_data_types[property_name]), From 6476d0d003de66ea069003bfc75c0734f567bab3 Mon Sep 17 00:00:00 2001 From: CodyCBakerPhD Date: Sat, 2 Mar 2024 20:23:19 -0500 Subject: [PATCH 13/24] ignore contact_vector --- pyflask/manageNeuroconv/manage_neuroconv.py | 67 +++++++++++++-------- 1 file changed, 42 insertions(+), 25 deletions(-) diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index aad1a6c49..910c54171 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -100,7 +100,9 @@ def coerce_schema_compliance_recursive(obj, schema): elif key in schema.get("properties", {}): prop_schema = schema["properties"][key] if prop_schema.get("type") == "number" and (value is None or value == "NaN"): - obj[key] = ( + obj[ + key + ] = ( math.nan ) # Turn None into NaN if a number is expected (JavaScript JSON.stringify turns NaN into None) elif prop_schema.get("type") == "number" and isinstance(value, int): @@ -967,34 +969,34 @@ def get_electrode_columns_json(interface) -> List[Dict[str, Any]]: # default_column_metadata = interface.get_metadata()["Ecephys"]["ElectrodeColumns"]["properties"] # NOTE: This doesn't exist... # property_descriptions = {column_name: column_fields["description"] for column_name, column_fields in default_column_metadata} - recording = interface.recording_extractor - channel_ids = recording.get_channel_ids() - - contact_vector = properties.pop("contact_vector", None) + recording_extractor = interface.recording_extractor + channel_ids = recording_extractor.get_channel_ids() electrode_columns = [ dict( name=property_name, description=property_descriptions.get(property_name, "No description."), data_type=get_property_dtype( - recording_extractor=recording, property_name=property_name, channel_ids=[channel_ids[0]] + recording_extractor=recording_extractor, property_name=property_name, channel_ids=[channel_ids[0]] ), ) for property_name in properties.keys() + if property_name != "contact_vector" ] - if contact_vector is None: - return json.loads(json.dumps(obj=electrode_columns)) - - # Unpack contact vector - for property_name in contact_vector.dtype.names: - electrode_columns.append( - dict( - name=property_name, - description=property_descriptions.get(property_name, "No description."), - data_type=str(contact_vector.dtype.fields[property_name][0]), - ) - ) + # TODO: uncomment when neuroconv supports contact vectors (probe interface) + # contact_vector = properties.pop("contact_vector", None) + # if contact_vector is None: + # return json.loads(json.dumps(obj=electrode_columns)) + # # Unpack contact vector + # for property_name in contact_vector.dtype.names: + # electrode_columns.append( + # dict( + # name=property_name, + # description=property_descriptions.get(property_name, "No description."), + # data_type=str(contact_vector.dtype.fields[property_name][0]), + # ) + # ) return json.loads(json.dumps(obj=electrode_columns)) @@ -1053,19 +1055,34 @@ def update_recording_properties_from_table_as_json( electrode_column_data_types = {column["name"]: column["data_type"] for column in electrode_column_info} electrode_column_data_types["contact_vector"] = contact_vector_dtypes # Provide contact vector information - recording = recording_interface.recording_extractor - channel_ids = recording.get_channel_ids() + recording_extractor = recording_interface.recording_extractor + channel_ids = recording_extractor.get_channel_ids() stream_prefix = channel_ids[0].split("#")[0] # TODO: see if this generalized across formats - for entry in electrode_table_json: + property_names = recording_extractor.get_property_keys() + + # TODO: uncomment when neuroconv supports contact vectors (probe interface) + # if "contact_vector" in property_names: + # modified_contact_vector = np.array(recording_extractor.get_property(key="contact_vector")) # copy + # contact_vector_property_names = list(modified_contact_vector.dtype.names) + + for entry_index, entry in enumerate(electrode_table_json): electrode_properties = dict(entry) # copy channel_name = electrode_properties.pop("channel_name") for property_name, property_value in electrode_properties.items(): - - # Skip data with missing column information - if property_name in electrode_column_data_types: - recording.set_property( + if property_name not in electrode_column_data_types: # Skip data with missing column information + continue + # TODO: uncomment when neuroconv supports contact vectors (probe interface) + # elif property_name in contact_vector_property_names: + # property_index = contact_vector_property_names.index(property_name) + # modified_contact_vector[entry_index][property_index] = property_value + else: + recording_extractor.set_property( key=property_name, values=np.array([property_value], dtype=electrode_column_data_types[property_name]), ids=[stream_prefix + "#" + channel_name], ) + + # TODO: uncomment when neuroconv supports contact vectors (probe interface) + # if "contact_vector" in property_names: + # recording_extractor.set_property(key="contact_vector", values=modified_contact_vector) From c23d6c931b3729a8f5f6e3d46006e07a4c1931c2 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 3 Mar 2024 01:23:37 +0000 Subject: [PATCH 14/24] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pyflask/manageNeuroconv/manage_neuroconv.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index 910c54171..0ad98796e 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -100,9 +100,7 @@ def coerce_schema_compliance_recursive(obj, schema): elif key in schema.get("properties", {}): prop_schema = schema["properties"][key] if prop_schema.get("type") == "number" and (value is None or value == "NaN"): - obj[ - key - ] = ( + obj[key] = ( math.nan ) # Turn None into NaN if a number is expected (JavaScript JSON.stringify turns NaN into None) elif prop_schema.get("type") == "number" and isinstance(value, int): From 7e1fdf85972b05183d91ad3271ecb070f7662d7c Mon Sep 17 00:00:00 2001 From: Garrett Michael Flynn Date: Mon, 4 Mar 2024 09:34:09 -0800 Subject: [PATCH 15/24] Clean up property exclusion and commented future code --- pyflask/manageNeuroconv/manage_neuroconv.py | 36 +++++++++++---------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index 0ad98796e..9cfef29e2 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -941,9 +941,14 @@ def get_property_dtype(recording_extractor, property_name: str, channel_ids: lis def get_recording_interface_properties(recording_interface) -> Dict[str, Any]: """A convenience function for uniformly excluding certain properties of the provided recording extractor.""" property_names = list(recording_interface.recording_extractor.get_property_keys()) + + excluded_properties = ["contact_vector"] + + properties = { property_name: recording_interface.recording_extractor.get_property(key=property_name) for property_name in property_names + if property_name not in excluded_properties } return properties @@ -979,7 +984,6 @@ def get_electrode_columns_json(interface) -> List[Dict[str, Any]]: ), ) for property_name in properties.keys() - if property_name != "contact_vector" ] # TODO: uncomment when neuroconv supports contact vectors (probe interface) @@ -1033,33 +1037,31 @@ def update_recording_properties_from_table_as_json( # # Extract contact vector properties properties = get_recording_interface_properties(recording_interface) - contact_vector = properties.pop("contact_vector", None) - - contact_vector_dtypes = {} - if contact_vector is not None: - # Remove names from contact vector from the electrode_column_info and add to reconstructed_contact_vector_info - contact_vector_dtypes = contact_vector.dtype - # contact_vector_dtypes = { property_name: next((item for item in electrode_column_info if item['name'] == property_name), None)["data_type"] for property_name in contact_vector.dtype.names} - - # Remove contact vector properties from electrode_column_info - for property_name in contact_vector.dtype.names: - found = next((item for item in electrode_column_info if item["name"] == property_name), None) - if found: - electrode_column_info.remove(found) + # TODO: uncomment and adapt when neuroconv supports contact vectors (probe interface) + # contact_vector = properties.pop("contact_vector", None) + # contact_vector_dtypes = {} + # if contact_vector is not None: + # # Remove names from contact vector from the electrode_column_info and add to reconstructed_contact_vector_info + # contact_vector_dtypes = contact_vector.dtype + # # contact_vector_dtypes = { property_name: next((item for item in electrode_column_info if item['name'] == property_name), None)["data_type"] for property_name in contact_vector.dtype.names} + # # Remove contact vector properties from electrode_column_info + # for property_name in contact_vector.dtype.names: + # found = next((item for item in electrode_column_info if item["name"] == property_name), None) + # if found: + # electrode_column_info.remove(found) # Organize dtypes electrode_column_data_types = {column["name"]: column["data_type"] for column in electrode_column_info} - electrode_column_data_types["contact_vector"] = contact_vector_dtypes # Provide contact vector information + # electrode_column_data_types["contact_vector"] = contact_vector_dtypes # Provide contact vector information recording_extractor = recording_interface.recording_extractor channel_ids = recording_extractor.get_channel_ids() stream_prefix = channel_ids[0].split("#")[0] # TODO: see if this generalized across formats - property_names = recording_extractor.get_property_keys() - # TODO: uncomment when neuroconv supports contact vectors (probe interface) + # property_names = recording_extractor.get_property_keys() # if "contact_vector" in property_names: # modified_contact_vector = np.array(recording_extractor.get_property(key="contact_vector")) # copy # contact_vector_property_names = list(modified_contact_vector.dtype.names) From 7b6ad130cda0a38be8f30ed878fb2d0e6b946d80 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 4 Mar 2024 17:34:24 +0000 Subject: [PATCH 16/24] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pyflask/manageNeuroconv/manage_neuroconv.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index 9cfef29e2..699cfe505 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -944,7 +944,6 @@ def get_recording_interface_properties(recording_interface) -> Dict[str, Any]: excluded_properties = ["contact_vector"] - properties = { property_name: recording_interface.recording_extractor.get_property(key=property_name) for property_name in property_names @@ -1038,7 +1037,6 @@ def update_recording_properties_from_table_as_json( # # Extract contact vector properties properties = get_recording_interface_properties(recording_interface) - # TODO: uncomment and adapt when neuroconv supports contact vectors (probe interface) # contact_vector = properties.pop("contact_vector", None) # contact_vector_dtypes = {} From f671e9d5f562ca83b2b824adfcf2a5debe54f532 Mon Sep 17 00:00:00 2001 From: Garrett Michael Flynn Date: Mon, 4 Mar 2024 10:05:48 -0800 Subject: [PATCH 17/24] Ignore contact shapes --- pyflask/manageNeuroconv/manage_neuroconv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index 699cfe505..337381d7d 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -942,7 +942,7 @@ def get_recording_interface_properties(recording_interface) -> Dict[str, Any]: """A convenience function for uniformly excluding certain properties of the provided recording extractor.""" property_names = list(recording_interface.recording_extractor.get_property_keys()) - excluded_properties = ["contact_vector"] + excluded_properties = [ "contact_vector", "contact_shapes" ] properties = { property_name: recording_interface.recording_extractor.get_property(key=property_name) From 9dbac92ddc80e9c796569b7f40b68a0b85e56553 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 4 Mar 2024 18:06:54 +0000 Subject: [PATCH 18/24] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pyflask/manageNeuroconv/manage_neuroconv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index 337381d7d..9d126f29d 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -942,7 +942,7 @@ def get_recording_interface_properties(recording_interface) -> Dict[str, Any]: """A convenience function for uniformly excluding certain properties of the provided recording extractor.""" property_names = list(recording_interface.recording_extractor.get_property_keys()) - excluded_properties = [ "contact_vector", "contact_shapes" ] + excluded_properties = ["contact_vector", "contact_shapes"] properties = { property_name: recording_interface.recording_extractor.get_property(key=property_name) From 86a5e9da83c31e80c12e495dc1b78a64fc20aa98 Mon Sep 17 00:00:00 2001 From: Garrett Michael Flynn Date: Tue, 5 Mar 2024 15:10:48 -0800 Subject: [PATCH 19/24] Fix validation and auto-update group name changes --- src/renderer/src/stories/BasicTable.js | 2 +- src/renderer/src/stories/JSONSchemaForm.js | 88 +++++- src/renderer/src/stories/JSONSchemaInput.js | 6 +- src/renderer/src/stories/SimpleTable.js | 2 +- src/renderer/src/stories/Table.js | 4 +- src/renderer/src/validation/index.js | 58 ++-- src/renderer/src/validation/validation.json | 6 +- src/renderer/src/validation/validation.ts | 326 ++++++++++++-------- 8 files changed, 311 insertions(+), 181 deletions(-) diff --git a/src/renderer/src/stories/BasicTable.js b/src/renderer/src/stories/BasicTable.js index 957b83d66..7ff894fd6 100644 --- a/src/renderer/src/stories/BasicTable.js +++ b/src/renderer/src/stories/BasicTable.js @@ -245,7 +245,7 @@ export class BasicTable extends LitElement { result = [{ message: `${col} is expected to be of type ${ogType}, not ${inferredType}`, type: "error" }]; } // Otherwise validate using the specified onChange function - else result = this.validateOnChange(col, parent, value, this.#itemProps[col]); + else result = this.validateOnChange([ col ], parent, value, this.#itemProps[col]); // Will run synchronously if not a promise result return promises.resolve(result, () => { diff --git a/src/renderer/src/stories/JSONSchemaForm.js b/src/renderer/src/stories/JSONSchemaForm.js index a599969a5..15f98cb0a 100644 --- a/src/renderer/src/stories/JSONSchemaForm.js +++ b/src/renderer/src/stories/JSONSchemaForm.js @@ -253,6 +253,70 @@ export class JSONSchemaForm extends LitElement { if (props.base) this.base = props.base; } + + // Handle wildcards to grab multiple form elements + getAllFormElements = (path, config = { forms: true, tables: true, inputs: true }) => { + + + const name = path[0]; + const upcomingPath = path.slice(1); + + + const isWildcard = name === "*" + const last = !upcomingPath.length; + + if (isWildcard) { + + if (last) { + const allElements = [ ]; + if (config.forms) allElements.push(...this.forms.values()); + if (config.tables) allElements.push(...this.tables.values()); + if (config.inputs) allElements.push(...this.inputs.values()); + return allElements + } + + else return Object.values(this.forms).map((form) => form.getAllFormElements(upcomingPath, config)).flat() + } + + + // Get Single element + else { + const result = this.#getElementOnForm(path) + if (!result) return [] + + if (last) { + if (result instanceof JSONSchemaForm && config.forms) return [ result ] + else if (result instanceof JSONSchemaInput && config.inputs) return [ result ] + else if (config.tables) return [ result ] + + return [ ] + } + else { + if (result instanceof JSONSchemaForm) return result.getAllFormElements(upcomingPath, config) + else return [ result ] + } + } + + } + + // Single later only + #getElementOnForm = (path, { forms = true, tables = true, inputs = true } = {}) => { + if (typeof path === "string") path = path.split("."); + if (!path.length) return this; + + const name = path[0]; + + const form = this.forms[name] + if (form && forms) return form + + const table = this.tables[name] + if (table && tables) return table + + const foundInput = this.inputs[path.join(".")]; // Check Inputs + if (foundInput && inputs) return foundInput; + + } + // Get the form element defined by the path (stops before table cells) getFormElement = ( path, @@ -260,29 +324,21 @@ export class JSONSchemaForm extends LitElement { forms: true, tables: true, inputs: true, - } + }, ) => { + if (typeof path === "string") path = path.split("."); if (!path.length) return this; - const name = path[0]; const updatedPath = path.slice(1); - const form = this.forms[name]; // Check forms - if (!form) { - const table = this.tables[name]; // Check tables - if (table && tables) return table; // Skip table cells - } else if (!updatedPath.length && forms) return form; - - // Check Inputs - // const inputContainer = this.shadowRoot.querySelector(`#${encode(path.join("-"))}`); - // if (inputContainer && inputs) return inputContainer.querySelector("jsonschema-input");; - - const foundInput = this.inputs[path.join(".")]; // Check Inputs - if (foundInput && inputs) return foundInput; + const result = this.#getElementOnForm(path, { forms, tables, inputs }); + if (result instanceof JSONSchemaForm) { + if (!updatedPath.length) return result; + else return result.getElementOnForm(updatedPath, { forms, tables, inputs }); + } - // Check Nested Form Inputs - return form?.getFormElement(updatedPath, { forms, tables, inputs }); + return result; }; #requirements = {}; diff --git a/src/renderer/src/stories/JSONSchemaInput.js b/src/renderer/src/stories/JSONSchemaInput.js index 6a55bb2de..c49798447 100644 --- a/src/renderer/src/stories/JSONSchemaInput.js +++ b/src/renderer/src/stories/JSONSchemaInput.js @@ -30,6 +30,7 @@ function resolveDateTime(value) { export function createTable(fullPath, { onUpdate, onThrow, overrides = {} }) { const name = fullPath.slice(-1)[0]; const path = fullPath.slice(0, -1); + const relativePath = this.form?.base ? fullPath.slice(this.form.base.length) : fullPath; const schema = this.schema; const validateOnChange = this.validateOnChange; @@ -256,6 +257,7 @@ export function createTable(fullPath, { onUpdate, onThrow, overrides = {} }) { merge(overrides.schema, schemaCopy, { arrays: true }); + // Normal table parsing const tableMetadata = { schema: schemaCopy, @@ -264,10 +266,10 @@ export function createTable(fullPath, { onUpdate, onThrow, overrides = {} }) { ignore: nestedIgnore, // According to schema onUpdate: function () { - return onUpdate.call(this, fullPath, this.data); // Update all table data + return onUpdate.call(this, relativePath, this.data); // Update all table data }, - validateOnChange: (...args) => commonValidationFunction(fullPath, ...args), + validateOnChange: (...args) => commonValidationFunction(relativePath, ...args), ...commonTableMetadata, }; diff --git a/src/renderer/src/stories/SimpleTable.js b/src/renderer/src/stories/SimpleTable.js index 412483722..8d301041c 100644 --- a/src/renderer/src/stories/SimpleTable.js +++ b/src/renderer/src/stories/SimpleTable.js @@ -503,7 +503,7 @@ export class SimpleTable extends LitElement { Object.keys(cols).map((k) => (cols[k] = "")); if (this.validateOnChange) Object.keys(cols).map((k) => { - const res = this.validateOnChange(k, { ...cols }, cols[k]); + const res = this.validateOnChange([ k ], { ...cols }, cols[k]); if (typeof res === "function") res(); }); diff --git a/src/renderer/src/stories/Table.js b/src/renderer/src/stories/Table.js index ec956b466..d554db844 100644 --- a/src/renderer/src/stories/Table.js +++ b/src/renderer/src/stories/Table.js @@ -295,7 +295,7 @@ export class Table extends LitElement { try { const valid = this.validateOnChange ? await this.validateOnChange( - k, + [ k ], { ...this.data[rowHeaders[row]] }, // Validate on a copy of the parent value, info @@ -551,7 +551,7 @@ export class Table extends LitElement { const rowName = rowHeaders[row]; // const cols = this.data[rowHeaders[row]] // Object.keys(cols).map(k => cols[k] = '') - // if (this.validateOnChange) Object.keys(cols).map(k => this.validateOnChange(k, { ...cols }, cols[k])) // Validate with empty values before removing + // if (this.validateOnChange) Object.keys(cols).map(k => this.validateOnChange([ k ], { ...cols }, cols[k])) // Validate with empty values before removing delete this.data[rowHeaders[row]]; delete unresolved[row]; this.onUpdate(rowName, null, undefined); // NOTE: Global metadata PR might simply set all data values to undefined diff --git a/src/renderer/src/validation/index.js b/src/renderer/src/validation/index.js index ae9206415..79d51187f 100644 --- a/src/renderer/src/validation/index.js +++ b/src/renderer/src/validation/index.js @@ -14,7 +14,7 @@ export async function validateOnChange(name, parent, path, value) { const fullPath = [...path, name]; const toIterate = fullPath; //fullPathNoRows // fullPath - + const copy = { ...parent }; // Validate on a copy of the parent if (arguments.length > 3) copy[name] = value; // Update value on copy @@ -24,35 +24,33 @@ export async function validateOnChange(name, parent, path, value) { else return; }, validationSchema); // Pass the top level until it runs out - let overridden = false; - // Skip wildcard check for categories marked with false if (lastResolved !== false && (functions === undefined || functions === true)) { - // let overridden = false; - let lastWildcard; - toIterate.reduce((acc, key) => { - // Disable the value is a hardcoded list of functions + a wildcard has already been specified - if (acc && lastWildcard && Array.isArray(acc[key] ?? {})) overridden = true; - else if (acc && "*" in acc) { - if (acc["*"] === false && lastWildcard) - overridden = true; // Disable if false and a wildcard has already been specified - // Otherwise set the last wildcard - else { - lastWildcard = typeof acc["*"] === "string" ? acc["*"].replace(`{*}`, `${name}`) : acc["*"]; - overridden = false; // Re-enable if a new one is specified below - } - } else if (lastWildcard && typeof lastWildcard === "object") { - const newWildcard = lastWildcard[key] ?? lastWildcard["*"] ?? lastWildcard["**"] ?? (acc && acc["**"]); // Drill wildcard objects once resolved - // Prioritize continuation of last wildcard - if (newWildcard) lastWildcard = newWildcard; - } - - return acc?.[key]; - }, validationSchema); - - if (overridden && functions !== true) lastWildcard = false; // Disable if not promised to exist - - if (typeof lastWildcard === "function" || typeof lastWildcard === "string") functions = [lastWildcard]; + + const getNestedMatches = (result, searchPath, toAlwaysCheck = []) => { + const matches = [] + const isUndefined = result === undefined; + if (Array.isArray(result)) matches.push(...result) + else if (result && typeof result === "object") matches.push(...getMatches(result, searchPath, toAlwaysCheck)); + else if (!isUndefined) matches.push(result); + if (searchPath.length) toAlwaysCheck.forEach((obj) => matches.push(...getMatches(obj, searchPath, toAlwaysCheck))) + return matches; + } + + const getMatches = (obj = {}, searchPath, toAlwaysCheck = []) => { + const updatedAlwaysCheck = [...toAlwaysCheck] + const updateSearchPath = [...searchPath] + const nextToken = updateSearchPath.shift(); + const matches = []; + if (obj['*']) matches.push(...getNestedMatches(obj['*'], updateSearchPath, updatedAlwaysCheck)) + if (obj['**']) updatedAlwaysCheck.push(obj['**']) + matches.push(...getNestedMatches(obj[nextToken], updateSearchPath, updatedAlwaysCheck)); // Always search to the end of the search path + return matches; + } + + const matches = getMatches(validationSchema, toIterate); + const overridden = matches.some((match) => match === false); + functions = (overridden && functions !== true) ? false : matches; // Disable if not promised to exist—or use matches } if (!functions || (Array.isArray(functions) && functions.length === 0)) return; // No validation for this field @@ -63,12 +61,14 @@ export async function validateOnChange(name, parent, path, value) { if (typeof func === "function") { return func.call(this, name, copy, path, value); // Can specify alternative client-side validation } else { + + const resolvedFunctionName = func.replace(`{*}`, `${name}`) return fetch(`${baseUrl}/neuroconv/validate`, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ parent: copy, - function_name: func, + function_name: resolvedFunctionName, }), }) .then((res) => res.json()) diff --git a/src/renderer/src/validation/validation.json b/src/renderer/src/validation/validation.json index 96569524f..f2af26e36 100644 --- a/src/renderer/src/validation/validation.json +++ b/src/renderer/src/validation/validation.json @@ -1,10 +1,10 @@ { - "*": "check_{*}", "name": false, "conversion_output_folder": false, "NWBFile": { + "*": "check_{*}", "identifier": false, "session_description": false, "lab": false, @@ -42,6 +42,7 @@ "Behavior": false, "Subject": { + "*": "check_subject_{*}", "sessions": false, "description": false, "genotype": false, @@ -51,7 +52,6 @@ "subject_id": "check_subject_id_exists", "species": ["check_subject_species_form", "check_subject_species_exists"], "date_of_birth": false, - "age": ["check_subject_age", "check_subject_proper_age_range"], - "*": "check_subject_{*}" + "age": ["check_subject_age", "check_subject_proper_age_range"] } } diff --git a/src/renderer/src/validation/validation.ts b/src/renderer/src/validation/validation.ts index 5237b11d3..0e166881f 100644 --- a/src/renderer/src/validation/validation.ts +++ b/src/renderer/src/validation/validation.ts @@ -2,14 +2,7 @@ import schema from './validation.json' import { JSONSchemaForm } from '../stories/JSONSchemaForm.js' import Swal from 'sweetalert2' -function rerenderTable (this: JSONSchemaForm, linkedPath: string[]) { - const element = this.getFormElement(linkedPath) - if (element) element.requestUpdate() // Re-render table to show updates - // if (element) setTimeout(() => { - // element.requestUpdate() - // }, 100); // Re-render table to show new column - return element -} +// ----------------- Validation Utility Functions ----------------- // const isNotUnique = (key, currentValue, rows, idx) => { @@ -24,7 +17,6 @@ const isNotUnique = (key, currentValue, rows, idx) => { type: 'error' } ] - } const get = (object: any, path: string[]) => { @@ -44,91 +36,95 @@ const get = (object: any, path: string[]) => { } } -// NOTE: Does this maintain separation between multiple sessions? -schema.Ecephys.ElectrodeGroup = { - ["*"]: { - name: function (this: JSONSchemaForm, _, __, ___, value) { - const groups = this.results.Ecephys.ElectrodeGroup.map(({ name }) => name) - // Check if the latest value will be new. Run function after validation - if (!value || !groups.includes(value)) { - return () => { - setTimeout(() => rerenderTable.call(this, ['Ecephys', 'Electrodes'])) // Allow for the updates to occur - } - } - }, - device: function (this: JSONSchemaForm, name, parent, path) { - const devices = this.results.Ecephys.Device.map(({ name }) => name) - if (devices.includes(parent[name])) return true - else { - return [ - { - message: 'Not a valid device', - type: 'error' - } - ] - } - } - } +function ensureUnique(this: JSONSchemaForm, name, parent, path, value) { + const { + values, + value: row + } = get(this.results, path) // NOTE: this.results is out of sync with the actual row contents at the moment of validation + + + if (!row) return true // Allow blank rows + + const rows = values.slice(-1)[0] + const idx = path.slice(-1)[0] + const isUniqueError = isNotUnique(name, value, rows, idx) + if (isUniqueError) return isUniqueError + + return true } -schema.Ecephys.Electrodes = { - ["*"]:{ - - // Label columns as invalid if not registered on the ElectrodeColumns table - // NOTE: If not present in the schema, these are not being rendered... - ['*']: function (this: JSONSchemaForm, name, parent, path) { - const electrodeColumns = this.results.ElectrodeColumns - if (electrodeColumns && !electrodeColumns.find((row: any) => row.name === name)) return [ - { - message: 'Not a valid column', - type: 'error' - } - ] - }, - group_name: function (this: JSONSchemaForm, _, __, ___, value) { +const getTablePathInfo = (path: string[]) => { + const modality = path[0] as Modality + const slice = path.slice(-2) + const table = slice[1] + const row = slice[2] - const groups = this.results.Ecephys.ElectrodeGroup.map(({ name }) => name) - if (groups.includes(value)) return true - else { - return [ - { - message: 'Not a valid group name', - type: 'error' - } - ] + return { modality, table, row } +} + + +// ----------------- Joint Ophys and Ecephys Validation ----------------- // + +const dependencies = { + Ophys: { + devices: [ + { + path: [ 'ImagingPlane' ], + key: 'device' + }, + { + path: [ 'TwoPhotonSeries' ], + key: 'imaging_plane' + }, + { + path: [ 'OnePhotonSeries' ], + key: 'imaging_plane' } - } + ] + }, + Ecephys: { + devices: [ + { + path: [ 'ElectrodeGroup' ], + key: 'device' + } + ], + groups: [ + { + path: [ 'Electrodes', '*', 'Electrodes' ], + key: 'group_name' + } + ] } } +type Modality = keyof typeof dependencies -// Update the columns available on the Electrodes table when there is a new name in the ElectrodeColumns table -schema.Ecephys.ElectrodeColumns = { +schema.Ophys = schema.Ecephys = { ['*']: { - ['*']: function (this: JSONSchemaForm, prop, parent, path) { - - const name = parent['name'] - if (!name) return true // Allow blank rows - - // NOTE: Reimplement across all separate tables... - // if (prop === 'name' && !(name in this.schema.properties.Ecephys.properties.Electrodes.items.properties)) { - // const element = rerender.call(this, ['Ecephys', 'Electrodes']) - // element.schema.properties[name] = {} // Ensure property is present in the schema now - // element.data.forEach(row => name in row ? undefined : row[name] = '') // Set column value as blank if not existent on row - // } - } + '**': { + ['name']: ensureUnique, + } } } -function ensureUnique(this: JSONSchemaForm, name, parent, path, value) { +async function safeRename (this: JSONSchemaForm, name, parent, path, value, options = {}) { + + const { + dependencies = {}, + swalOptions = {} + } = options + const { values, value: row } = get(this.results, path) + const info = getTablePathInfo(path) + if (!row) return true // Allow blank rows const rows = values.slice(-1)[0] @@ -136,82 +132,158 @@ function ensureUnique(this: JSONSchemaForm, name, parent, path, value) { const isUniqueError = isNotUnique(name, value, rows, idx) if (isUniqueError) return isUniqueError + const prevValue = row[name] + + if (prevValue === value || prevValue === undefined) return true // No change + + const prevUniqueError = isNotUnique(name, prevValue, rows, idx) + if (prevUniqueError) return true // Register as valid + + const resolvedSwalOptions = {} + for (const key in swalOptions) resolvedSwalOptions[key] = typeof swalOptions[key] === 'function' ? swalOptions[key](value, prevValue) : swalOptions[key] + + const result = await Swal.fire({ + ...resolvedSwalOptions, + icon: "warning", + heightAuto: false, + backdrop: "rgba(0,0,0, 0.4)", + confirmButtonText: "I understand", + showConfirmButton: true, + showCancelButton: true, + cancelButtonText: "Cancel" + }) + + if (!result.isConfirmed) return null + + // Update Dependent Tables + const modalityDependencies = dependencies[info.modality] ?? [] + + modalityDependencies.forEach(({ key, path }) => { + const fullPath = [info.modality, ...path] + const tables = this.getAllFormElements(fullPath, { tables: true }) + console.log('Got all tables', tables, fullPath) + tables.forEach(table => { + const data = table.data + data.forEach(row => { + if (row[key] === prevValue) row[key] = value + }) + table.data = data + table.requestUpdate() + }) + }) + return true } -schema.Ophys = { - ['*']: { - '**': { - ['name']: ensureUnique, - } +// Ophys +schema.Ophys.Device = schema.Ecephys.Device = { + ["*"]: { + + ['name']: function(...args) { + return safeRename.call(this, ...args, { + dependencies: { Ophys: dependencies.Ophys.devices, Ecephys: dependencies.Ecephys.devices }, + swalOptions: { + title: (current, prev) => `Are you sure you want to rename the ${prev} device?`, + text: () => `We will attempt to auto-update your Ophys devices to reflect this.`, + } + }) + }, + } } -// Ophys -schema.Ophys.Device = { +// ----------------- Ecephys Validation ----------------- // + +// NOTE: Does this maintain separation between multiple sessions? +schema.Ecephys.ElectrodeGroup = { ["*"]: { - ['name']: async function (this: JSONSchemaForm, name, parent, path, value) { + name: function(...args) { + return safeRename.call(this, ...args, { + dependencies: { Ecephys: dependencies.Ecephys.groups }, + swalOptions: { + title: (current, prev) => `Are you sure you want to rename the ${prev} group?`, + text: () => `We will attempt to auto-update your electrode groups to reflect this.`, + } + }) + }, - const { - values, - value: row - } = get(this.results, path) + device: function (this: JSONSchemaForm, name, parent, path, value) { + const devices = this.results.Ecephys.Device.map(({ name }) => name) - if (!row) return true // Allow blank rows + if (devices.includes(value)) return true + else { + return [ + { + message: 'Not a valid device', + type: 'error' + } + ] + } + } + } +} - const rows = values.slice(-1)[0] - const idx = path.slice(-1)[0] - const isUniqueError = isNotUnique(name, value, rows, idx) - if (isUniqueError) return isUniqueError - const prevValue = row[name] +// Label columns as invalid if not registered on the ElectrodeColumns table +// NOTE: If not present in the schema, these are not being rendered... - if (prevValue === value || prevValue === undefined) return true // No change +schema.Ecephys.Electrodes = { - const prevUniqueError = isNotUnique(name, prevValue, rows, idx) - if (prevUniqueError) return true // Register as valid + // All interfaces + ["*"]: { - const result = await Swal.fire({ - title: `Are you sure you want to rename the ${prevValue} device?`, - icon: "warning", - text: `We will attempt to auto-update your Ophys devices to reflect this.`, - heightAuto: false, - backdrop: "rgba(0,0,0, 0.4)", - confirmButtonText: "I understand", - showConfirmButton: true, - showCancelButton: true, - cancelButtonText: "Cancel" - }) + Electrodes: { - if (!result.isConfirmed) return null - - // Update Dependent Tables - const dependencies = [ - ['Ophys', 'ImagingPlane'], - ['Ophys', 'OnePhotonSeries'], - ['Ophys', 'TwoPhotonSeries'] - ] - - dependencies.forEach(path => { - const table = this.getFormElement(path, { tables: true }) - if (table) { - const data = table.data - data.forEach(row => { - if (row.device === prevValue) row.device = value - }) - table.data = data + // All other column + ['*']: function (this: JSONSchemaForm, name, parent, path) { + const electrodeColumns = this.results.ElectrodeColumns + if (electrodeColumns && !electrodeColumns.find((row: any) => row.name === name)) return [ + { + message: 'Not a valid column', + type: 'error' + } + ] + }, + + // Group name column + group_name: function (this: JSONSchemaForm, _, __, ___, value) { + + const groups = this.results.Ecephys.ElectrodeGroup.map(({ name }) => name) + if (groups.includes(value)) return true + else { + return [ + { + message: 'Not a valid group name', + type: 'error' + } + ] } + } + }, - rerenderTable.call(this, path) - }) + // Update the columns available on the Electrodes table when there is a new name in the ElectrodeColumns table + ElectrodeColumns: { + ['*']: { + ['*']: function (this: JSONSchemaForm, prop, parent, path) { - return true - } + const name = parent['name'] + if (!name) return true // Allow blank rows + // NOTE: Reimplement across all separate tables... + // if (prop === 'name' && !(name in this.schema.properties.Ecephys.properties.Electrodes.items.properties)) { + // const element = rerender.call(this, ['Ecephys', 'Electrodes']) + // element.schema.properties[name] = {} // Ensure property is present in the schema now + // element.data.forEach(row => name in row ? undefined : row[name] = '') // Set column value as blank if not existent on row + // } + } + } + } } } +// ----------------- Ophys Validation ----------------- // + schema.Ophys.ImagingPlane = { ["*"]: { device: function (this: JSONSchemaForm, name, parent, path, value) { From 340bc04b024d184ffe5a5de4700ad76093ed4e24 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 5 Mar 2024 23:11:11 +0000 Subject: [PATCH 20/24] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/renderer/src/stories/BasicTable.js | 2 +- src/renderer/src/stories/JSONSchemaForm.js | 57 +++++++++------------ src/renderer/src/stories/JSONSchemaInput.js | 1 - src/renderer/src/stories/SimpleTable.js | 2 +- src/renderer/src/stories/Table.js | 2 +- src/renderer/src/validation/index.js | 30 +++++------ src/renderer/src/validation/validation.ts | 6 +-- 7 files changed, 45 insertions(+), 55 deletions(-) diff --git a/src/renderer/src/stories/BasicTable.js b/src/renderer/src/stories/BasicTable.js index 7ff894fd6..f93d7d59f 100644 --- a/src/renderer/src/stories/BasicTable.js +++ b/src/renderer/src/stories/BasicTable.js @@ -245,7 +245,7 @@ export class BasicTable extends LitElement { result = [{ message: `${col} is expected to be of type ${ogType}, not ${inferredType}`, type: "error" }]; } // Otherwise validate using the specified onChange function - else result = this.validateOnChange([ col ], parent, value, this.#itemProps[col]); + else result = this.validateOnChange([col], parent, value, this.#itemProps[col]); // Will run synchronously if not a promise result return promises.resolve(result, () => { diff --git a/src/renderer/src/stories/JSONSchemaForm.js b/src/renderer/src/stories/JSONSchemaForm.js index 15f98cb0a..ce77cf10d 100644 --- a/src/renderer/src/stories/JSONSchemaForm.js +++ b/src/renderer/src/stories/JSONSchemaForm.js @@ -253,51 +253,44 @@ export class JSONSchemaForm extends LitElement { if (props.base) this.base = props.base; } - // Handle wildcards to grab multiple form elements getAllFormElements = (path, config = { forms: true, tables: true, inputs: true }) => { - - const name = path[0]; const upcomingPath = path.slice(1); - - const isWildcard = name === "*" + const isWildcard = name === "*"; const last = !upcomingPath.length; if (isWildcard) { - if (last) { - const allElements = [ ]; + const allElements = []; if (config.forms) allElements.push(...this.forms.values()); if (config.tables) allElements.push(...this.tables.values()); if (config.inputs) allElements.push(...this.inputs.values()); - return allElements - } - - else return Object.values(this.forms).map((form) => form.getAllFormElements(upcomingPath, config)).flat() + return allElements; + } else + return Object.values(this.forms) + .map((form) => form.getAllFormElements(upcomingPath, config)) + .flat(); } - // Get Single element else { - const result = this.#getElementOnForm(path) - if (!result) return [] + const result = this.#getElementOnForm(path); + if (!result) return []; if (last) { - if (result instanceof JSONSchemaForm && config.forms) return [ result ] - else if (result instanceof JSONSchemaInput && config.inputs) return [ result ] - else if (config.tables) return [ result ] - - return [ ] - } - else { - if (result instanceof JSONSchemaForm) return result.getAllFormElements(upcomingPath, config) - else return [ result ] + if (result instanceof JSONSchemaForm && config.forms) return [result]; + else if (result instanceof JSONSchemaInput && config.inputs) return [result]; + else if (config.tables) return [result]; + + return []; + } else { + if (result instanceof JSONSchemaForm) return result.getAllFormElements(upcomingPath, config); + else return [result]; } } - - } + }; // Single later only #getElementOnForm = (path, { forms = true, tables = true, inputs = true } = {}) => { @@ -306,16 +299,15 @@ export class JSONSchemaForm extends LitElement { const name = path[0]; - const form = this.forms[name] - if (form && forms) return form + const form = this.forms[name]; + if (form && forms) return form; - const table = this.tables[name] - if (table && tables) return table + const table = this.tables[name]; + if (table && tables) return table; const foundInput = this.inputs[path.join(".")]; // Check Inputs if (foundInput && inputs) return foundInput; - - } + }; // Get the form element defined by the path (stops before table cells) getFormElement = ( @@ -324,9 +316,8 @@ export class JSONSchemaForm extends LitElement { forms: true, tables: true, inputs: true, - }, + } ) => { - if (typeof path === "string") path = path.split("."); if (!path.length) return this; diff --git a/src/renderer/src/stories/JSONSchemaInput.js b/src/renderer/src/stories/JSONSchemaInput.js index c49798447..fd09b82ba 100644 --- a/src/renderer/src/stories/JSONSchemaInput.js +++ b/src/renderer/src/stories/JSONSchemaInput.js @@ -257,7 +257,6 @@ export function createTable(fullPath, { onUpdate, onThrow, overrides = {} }) { merge(overrides.schema, schemaCopy, { arrays: true }); - // Normal table parsing const tableMetadata = { schema: schemaCopy, diff --git a/src/renderer/src/stories/SimpleTable.js b/src/renderer/src/stories/SimpleTable.js index 8d301041c..9d63d3e86 100644 --- a/src/renderer/src/stories/SimpleTable.js +++ b/src/renderer/src/stories/SimpleTable.js @@ -503,7 +503,7 @@ export class SimpleTable extends LitElement { Object.keys(cols).map((k) => (cols[k] = "")); if (this.validateOnChange) Object.keys(cols).map((k) => { - const res = this.validateOnChange([ k ], { ...cols }, cols[k]); + const res = this.validateOnChange([k], { ...cols }, cols[k]); if (typeof res === "function") res(); }); diff --git a/src/renderer/src/stories/Table.js b/src/renderer/src/stories/Table.js index d554db844..27745fb5b 100644 --- a/src/renderer/src/stories/Table.js +++ b/src/renderer/src/stories/Table.js @@ -295,7 +295,7 @@ export class Table extends LitElement { try { const valid = this.validateOnChange ? await this.validateOnChange( - [ k ], + [k], { ...this.data[rowHeaders[row]] }, // Validate on a copy of the parent value, info diff --git a/src/renderer/src/validation/index.js b/src/renderer/src/validation/index.js index 79d51187f..e0d819e18 100644 --- a/src/renderer/src/validation/index.js +++ b/src/renderer/src/validation/index.js @@ -14,7 +14,7 @@ export async function validateOnChange(name, parent, path, value) { const fullPath = [...path, name]; const toIterate = fullPath; //fullPathNoRows // fullPath - + const copy = { ...parent }; // Validate on a copy of the parent if (arguments.length > 3) copy[name] = value; // Update value on copy @@ -26,31 +26,32 @@ export async function validateOnChange(name, parent, path, value) { // Skip wildcard check for categories marked with false if (lastResolved !== false && (functions === undefined || functions === true)) { - const getNestedMatches = (result, searchPath, toAlwaysCheck = []) => { - const matches = [] + const matches = []; const isUndefined = result === undefined; - if (Array.isArray(result)) matches.push(...result) - else if (result && typeof result === "object") matches.push(...getMatches(result, searchPath, toAlwaysCheck)); + if (Array.isArray(result)) matches.push(...result); + else if (result && typeof result === "object") + matches.push(...getMatches(result, searchPath, toAlwaysCheck)); else if (!isUndefined) matches.push(result); - if (searchPath.length) toAlwaysCheck.forEach((obj) => matches.push(...getMatches(obj, searchPath, toAlwaysCheck))) + if (searchPath.length) + toAlwaysCheck.forEach((obj) => matches.push(...getMatches(obj, searchPath, toAlwaysCheck))); return matches; - } + }; const getMatches = (obj = {}, searchPath, toAlwaysCheck = []) => { - const updatedAlwaysCheck = [...toAlwaysCheck] - const updateSearchPath = [...searchPath] + const updatedAlwaysCheck = [...toAlwaysCheck]; + const updateSearchPath = [...searchPath]; const nextToken = updateSearchPath.shift(); const matches = []; - if (obj['*']) matches.push(...getNestedMatches(obj['*'], updateSearchPath, updatedAlwaysCheck)) - if (obj['**']) updatedAlwaysCheck.push(obj['**']) + if (obj["*"]) matches.push(...getNestedMatches(obj["*"], updateSearchPath, updatedAlwaysCheck)); + if (obj["**"]) updatedAlwaysCheck.push(obj["**"]); matches.push(...getNestedMatches(obj[nextToken], updateSearchPath, updatedAlwaysCheck)); // Always search to the end of the search path return matches; - } + }; const matches = getMatches(validationSchema, toIterate); const overridden = matches.some((match) => match === false); - functions = (overridden && functions !== true) ? false : matches; // Disable if not promised to exist—or use matches + functions = overridden && functions !== true ? false : matches; // Disable if not promised to exist—or use matches } if (!functions || (Array.isArray(functions) && functions.length === 0)) return; // No validation for this field @@ -61,8 +62,7 @@ export async function validateOnChange(name, parent, path, value) { if (typeof func === "function") { return func.call(this, name, copy, path, value); // Can specify alternative client-side validation } else { - - const resolvedFunctionName = func.replace(`{*}`, `${name}`) + const resolvedFunctionName = func.replace(`{*}`, `${name}`); return fetch(`${baseUrl}/neuroconv/validate`, { method: "POST", headers: { "Content-Type": "application/json" }, diff --git a/src/renderer/src/validation/validation.ts b/src/renderer/src/validation/validation.ts index 0e166881f..cff460cd7 100644 --- a/src/renderer/src/validation/validation.ts +++ b/src/renderer/src/validation/validation.ts @@ -74,7 +74,7 @@ const dependencies = { { path: [ 'ImagingPlane' ], key: 'device' - }, + }, { path: [ 'TwoPhotonSeries' ], key: 'imaging_plane' @@ -184,7 +184,7 @@ schema.Ophys.Device = schema.Ecephys.Device = { dependencies: { Ophys: dependencies.Ophys.devices, Ecephys: dependencies.Ecephys.devices }, swalOptions: { title: (current, prev) => `Are you sure you want to rename the ${prev} device?`, - text: () => `We will attempt to auto-update your Ophys devices to reflect this.`, + text: () => `We will attempt to auto-update your Ophys devices to reflect this.`, } }) }, @@ -203,7 +203,7 @@ schema.Ecephys.ElectrodeGroup = { dependencies: { Ecephys: dependencies.Ecephys.groups }, swalOptions: { title: (current, prev) => `Are you sure you want to rename the ${prev} group?`, - text: () => `We will attempt to auto-update your electrode groups to reflect this.`, + text: () => `We will attempt to auto-update your electrode groups to reflect this.`, } }) }, From 4dbb245de7a01c8e1d2fe9a40938353a3b4494ff Mon Sep 17 00:00:00 2001 From: Garrett Michael Flynn Date: Tue, 5 Mar 2024 15:39:59 -0800 Subject: [PATCH 21/24] Catch invalid columns in the electrode table --- src/renderer/src/stories/JSONSchemaForm.js | 85 +++++++++++----------- src/renderer/src/validation/validation.ts | 57 ++++++++++----- 2 files changed, 82 insertions(+), 60 deletions(-) diff --git a/src/renderer/src/stories/JSONSchemaForm.js b/src/renderer/src/stories/JSONSchemaForm.js index ce77cf10d..4d8efcc37 100644 --- a/src/renderer/src/stories/JSONSchemaForm.js +++ b/src/renderer/src/stories/JSONSchemaForm.js @@ -21,6 +21,47 @@ const encode = (str) => { } }; +export const get = (path, object, omitted = [], skipped = []) => { + // path = path.slice(this.base.length); // Correct for base path + if (!path) throw new Error("Path not specified"); + return path.reduce((acc, curr, i) => { + const tempAcc = acc?.[curr] ?? acc?.[omitted.find((str) => acc[str] && acc[str][curr])]?.[curr]; + if (tempAcc) return tempAcc; + else { + const level1 = acc?.[skipped.find((str) => acc[str])]; + if (level1) { + // Handle items-like objects + const result = get(path.slice(i), level1, omitted, skipped); + if (result) return result; + + // Handle pattern properties objects + const got = Object.keys(level1).find((key) => { + const result = get(path.slice(i + 1), level1[key], omitted, skipped); + if (result && typeof result === "object") return result; // Schema are objects... + }); + + if (got) return level1[got]; + } + } + }, object); +}; + +export const getSchema = (path, schema, base = []) => { + if (typeof path === "string") path = path.split("."); + + // NOTE: Still must correct for the base here + if (base.length) { + const indexOf = path.indexOf(base.slice(-1)[0]); + if (indexOf !== -1) path = path.slice(indexOf + 1); + } + + // NOTE: Refs are now pre-resolved + const resolved = get(path, schema, ["properties", "patternProperties"], ["patternProperties", "items"]); + // if (resolved?.["$ref"]) return this.getSchema(resolved["$ref"].split("/").slice(1)); // NOTE: This assumes reference to the root of the schema + + return resolved; +}; + const additionalPropPattern = "additional"; const templateNaNMessage = `
Type NaN to represent an unknown value.`; @@ -326,7 +367,7 @@ export class JSONSchemaForm extends LitElement { const result = this.#getElementOnForm(path, { forms, tables, inputs }); if (result instanceof JSONSchemaForm) { if (!updatedPath.length) return result; - else return result.getElementOnForm(updatedPath, { forms, tables, inputs }); + else return result.getFormElement(updatedPath, { forms, tables, inputs }); } return result; @@ -557,30 +598,7 @@ export class JSONSchemaForm extends LitElement { return true; }; - #get = (path, object = this.resolved, omitted = [], skipped = []) => { - // path = path.slice(this.base.length); // Correct for base path - if (!path) throw new Error("Path not specified"); - return path.reduce((acc, curr, i) => { - const tempAcc = acc?.[curr] ?? acc?.[omitted.find((str) => acc[str] && acc[str][curr])]?.[curr]; - if (tempAcc) return tempAcc; - else { - const level1 = acc?.[skipped.find((str) => acc[str])]; - if (level1) { - // Handle items-like objects - const result = this.#get(path.slice(i), level1, omitted, skipped); - if (result) return result; - - // Handle pattern properties objects - const got = Object.keys(level1).find((key) => { - const result = this.#get(path.slice(i + 1), level1[key], omitted, skipped); - if (result && typeof result === "object") return result; // Schema are objects... - }); - - if (got) return level1[got]; - } - } - }, object); - }; + #get = (path, object = this.resolved, omitted = [], skipped = []) => get(path, object, omitted, skipped); #checkRequiredAfterChange = async (localPath) => { const path = [...localPath]; @@ -601,22 +619,7 @@ export class JSONSchemaForm extends LitElement { return this.#schema; } - getSchema(path, schema = this.schema) { - if (typeof path === "string") path = path.split("."); - - // NOTE: Still must correct for the base here - if (this.base.length) { - const base = this.base.slice(-1)[0]; - const indexOf = path.indexOf(base); - if (indexOf !== -1) path = path.slice(indexOf + 1); - } - - // NOTE: Refs are now pre-resolved - const resolved = this.#get(path, schema, ["properties", "patternProperties"], ["patternProperties", "items"]); - // if (resolved?.["$ref"]) return this.getSchema(resolved["$ref"].split("/").slice(1)); // NOTE: This assumes reference to the root of the schema - - return resolved; - } + getSchema = (path, schema = this.schema) => getSchema(path, schema, this.base); #renderInteractiveElement = (name, info, required, path = [], value, propertyType) => { let isRequired = this.#isRequired([...path, name]); diff --git a/src/renderer/src/validation/validation.ts b/src/renderer/src/validation/validation.ts index cff460cd7..c83b5d961 100644 --- a/src/renderer/src/validation/validation.ts +++ b/src/renderer/src/validation/validation.ts @@ -1,7 +1,8 @@ import schema from './validation.json' -import { JSONSchemaForm } from '../stories/JSONSchemaForm.js' +import { JSONSchemaForm, getSchema } from '../stories/JSONSchemaForm' import Swal from 'sweetalert2' + // ----------------- Validation Utility Functions ----------------- // const isNotUnique = (key, currentValue, rows, idx) => { @@ -237,19 +238,30 @@ schema.Ecephys.Electrodes = { // All other column ['*']: function (this: JSONSchemaForm, name, parent, path) { - const electrodeColumns = this.results.ElectrodeColumns - if (electrodeColumns && !electrodeColumns.find((row: any) => row.name === name)) return [ - { - message: 'Not a valid column', - type: 'error' - } - ] + + const commonPath = path.slice(0, -1) // NOTE: Path does not account for the row index + const colPath = [...commonPath, 'ElectrodeColumns'] + + const { value: electrodeColumns } = get(this.results, colPath) // NOTE: this.results is out of sync with the actual row contents at the moment of validation + + // console.log('Electrode Columns', electrodeColumns, colPath, name, path, this.results.Ecephys.ElectrodeColumns) + + if (electrodeColumns && !electrodeColumns.find((row: any) => row.name === name)) { + // console.error('Not a valid column', name, electrodeColumns, path, this.results.Ecephys.ElectrodeColumns) + return [ + { + message: 'Not a valid column', + type: 'error' + } + ] + } }, // Group name column - group_name: function (this: JSONSchemaForm, _, __, ___, value) { + group_name: function (this: JSONSchemaForm, _, __, path, value) { + + const groups = this.results.Ecephys.ElectrodeGroup.map(({ name }) => name) // Groups are validated across all interfaces - const groups = this.results.Ecephys.ElectrodeGroup.map(({ name }) => name) if (groups.includes(value)) return true else { return [ @@ -265,17 +277,24 @@ schema.Ecephys.Electrodes = { // Update the columns available on the Electrodes table when there is a new name in the ElectrodeColumns table ElectrodeColumns: { ['*']: { - ['*']: function (this: JSONSchemaForm, prop, parent, path) { + name: function (this: JSONSchemaForm, _, __, path, value) { - const name = parent['name'] - if (!name) return true // Allow blank rows + const name = value - // NOTE: Reimplement across all separate tables... - // if (prop === 'name' && !(name in this.schema.properties.Ecephys.properties.Electrodes.items.properties)) { - // const element = rerender.call(this, ['Ecephys', 'Electrodes']) - // element.schema.properties[name] = {} // Ensure property is present in the schema now - // element.data.forEach(row => name in row ? undefined : row[name] = '') // Set column value as blank if not existent on row - // } + const commonPath = path.slice(0, -2) + + const electrodesSchema = getSchema([ ...commonPath, 'Electrodes'], this.schema) + + if (!name) return true // Only set when name is actually present + + if (!(name in electrodesSchema.items.properties)) { + const electrodesTable = this.getFormElement([ ...commonPath, 'Electrodes']) + // electrodesTable.schema.properties[name] = {} // Ensure property is present in the schema now + electrodesSchema.items.properties[name] = {} + electrodesTable.data.forEach(row => name in row ? undefined : row[name] = '') // Set column value as blank if not existent on row + electrodesTable.requestUpdate() + } + } } } From 4ddadf2fc2b0d135d40fe96c96fc6aac155af353 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 5 Mar 2024 23:40:18 +0000 Subject: [PATCH 22/24] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/renderer/src/validation/validation.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/renderer/src/validation/validation.ts b/src/renderer/src/validation/validation.ts index c83b5d961..33e665d01 100644 --- a/src/renderer/src/validation/validation.ts +++ b/src/renderer/src/validation/validation.ts @@ -245,7 +245,7 @@ schema.Ecephys.Electrodes = { const { value: electrodeColumns } = get(this.results, colPath) // NOTE: this.results is out of sync with the actual row contents at the moment of validation // console.log('Electrode Columns', electrodeColumns, colPath, name, path, this.results.Ecephys.ElectrodeColumns) - + if (electrodeColumns && !electrodeColumns.find((row: any) => row.name === name)) { // console.error('Not a valid column', name, electrodeColumns, path, this.results.Ecephys.ElectrodeColumns) return [ @@ -294,7 +294,7 @@ schema.Ecephys.Electrodes = { electrodesTable.data.forEach(row => name in row ? undefined : row[name] = '') // Set column value as blank if not existent on row electrodesTable.requestUpdate() } - + } } } From 922b17f3064e8009722d95d86c97715cd7541251 Mon Sep 17 00:00:00 2001 From: Garrett Michael Flynn Date: Wed, 6 Mar 2024 10:22:32 -0800 Subject: [PATCH 23/24] Fix validation for electrode columns and electrodes --- pyflask/manageNeuroconv/manage_neuroconv.py | 22 ++- src/renderer/src/stories/BasicTable.js | 129 +++++++++++------- src/renderer/src/stories/SimpleTable.js | 5 + .../pages/guided-mode/data/GuidedMetadata.js | 2 + src/renderer/src/validation/validation.ts | 55 +++++--- 5 files changed, 141 insertions(+), 72 deletions(-) diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index c5f127f66..1fe7f7f35 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -19,6 +19,10 @@ announcer = MessageAnnouncer() +EXCLUDED_RECORDING_INTERFACE_PROPERTIES = ["contact_vector", "contact_shapes"] + + + def is_path_contained(child, parent): parent = Path(parent) child = Path(child) @@ -393,9 +397,11 @@ def on_recording_interface(name, recording_interface): defs["ElectrodeColumn"] = electrode_def defs["ElectrodeColumn"]["required"] = list(electrode_def["properties"].keys()) + new_electrodes_properties = { properties["name"]: {key: value for key, value in properties.items() if key != "name"} for properties in original_electrodes_schema["default"] + if properties["name"] not in EXCLUDED_RECORDING_INTERFACE_PROPERTIES } defs["Electrode"] = { @@ -937,8 +943,12 @@ def generate_test_data(output_path: str): waveform_extractor=waveform_extractor, output_folder=phy_output_folder, remove_if_exists=True, copy_binary=False ) - -dtype_map = {" str: + if ' Dict[str, Any]: """A convenience function for uniformly excluding certain properties of the provided recording extractor.""" property_names = list(recording_interface.recording_extractor.get_property_keys()) - excluded_properties = ["contact_vector", "contact_shapes"] - properties = { property_name: recording_interface.recording_extractor.get_property(key=property_name) for property_name in property_names - if property_name not in excluded_properties + if property_name not in EXCLUDED_RECORDING_INTERFACE_PROPERTIES } return properties @@ -1006,7 +1014,7 @@ def get_electrode_columns_json(interface) -> List[Dict[str, Any]]: # electrode_columns.append( # dict( # name=property_name, - # description=property_descriptions.get(property_name, "No description."), + # description=property_descriptions.get(property_name, ""), # data_type=str(contact_vector.dtype.fields[property_name][0]), # ) # ) diff --git a/src/renderer/src/stories/BasicTable.js b/src/renderer/src/stories/BasicTable.js index f93d7d59f..84a292cac 100644 --- a/src/renderer/src/stories/BasicTable.js +++ b/src/renderer/src/stories/BasicTable.js @@ -1,8 +1,8 @@ -import { LitElement, css, html } from "lit"; +import { LitElement, css, html, unsafeCSS } from "lit"; import { styleMap } from "lit/directives/style-map.js"; import { header } from "./forms/utils"; import { checkStatus } from "../validation"; -import { errorHue, warningHue } from "./globals"; +import { emojiFontFamily, errorHue, warningHue } from "./globals"; import * as promises from "../promises"; @@ -66,6 +66,12 @@ export class BasicTable extends LitElement { user-select: none; } + .relative .info { + margin: 0px 5px; + font-size: 80%; + font-family: ${unsafeCSS(emojiFontFamily)}; + } + th span { display: inline-block; } @@ -159,9 +165,32 @@ export class BasicTable extends LitElement { return html`
${header(str)}
`; }; + #renderHeader = (str, { description }) => { - if (description) return html`${this.#renderHeaderContent(str)}`; - return html`${this.#renderHeaderContent(str)}`; + const th = document.createElement("th"); + + + const required = this.#itemSchema.required ? this.#itemSchema.required.includes(str) : false; + const container = document.createElement("div"); + container.classList.add("relative"); + const span = document.createElement("span"); + span.textContent = header(str); + if (required) span.setAttribute("required", ""); + container.appendChild(span); + + // Add Description Tooltip + if (description) { + const span = document.createElement("span"); + span.classList.add("info"); + span.innerText = "ℹ️"; + container.append(span); + tippy(span, { content: `${description[0].toUpperCase() + description.slice(1)}`, allowHTML: true }); + } + + + th.appendChild(container); + + return th }; #getRowData(row, cols = this.colHeaders) { @@ -170,13 +199,12 @@ export class BasicTable extends LitElement { let value; if (col === this.keyColumn) { if (hasRow) value = row; - else return ""; + else return; } else value = (hasRow ? this.data[row][col] : undefined) ?? // this.globals[col] ?? - this.#itemSchema.properties[col].default ?? - ""; + this.#itemSchema.properties[col]?.default; return value; }); } @@ -208,10 +236,10 @@ export class BasicTable extends LitElement { }; status; - onStatusChange = () => {}; - onLoaded = () => {}; + onStatusChange = () => { }; + onLoaded = () => { }; - #validateCell = (value, col, parent) => { + #validateCell = (value, col, row, parent) => { if (!value && !this.validateEmptyCells) return true; // Empty cells are valid if (!this.validateOnChange) return true; @@ -244,11 +272,12 @@ export class BasicTable extends LitElement { else if (value !== "" && type && inferredType !== type) { result = [{ message: `${col} is expected to be of type ${ogType}, not ${inferredType}`, type: "error" }]; } + // Otherwise validate using the specified onChange function - else result = this.validateOnChange([col], parent, value, this.#itemProps[col]); + else result = this.validateOnChange([row, col], parent, value, this.#itemProps[col]); // Will run synchronously if not a promise result - return promises.resolve(result, () => { + return promises.resolve(result, (result) => { let info = { title: undefined, warning: undefined, @@ -279,7 +308,7 @@ export class BasicTable extends LitElement { const results = this.#data.map((v, i) => { return v.map((vv, j) => { - const info = this.#validateCell(vv, this.colHeaders[j], { ...this.data[rows[i]] }); // Could be a promise or a basic response + const info = this.#validateCell(vv, this.colHeaders[j], i, { ...this.data[rows[i]] }); // Could be a promise or a basic response return promises.resolve(info, (info) => { if (info === true) return; const td = this.shadowRoot.getElementById(`i${i}_j${j}`); @@ -368,6 +397,10 @@ export class BasicTable extends LitElement { render() { this.#updateRendered(); + this.schema = this.schema // Always update the schema + + console.warn('RERENDERING') + const entries = this.#itemProps; // Add existing additional properties to the entries variable if necessary @@ -376,8 +409,8 @@ export class BasicTable extends LitElement { Object.keys(v).forEach((k) => !(k in entries) ? (entries[k] = { - type: typeof v[k], - }) + type: typeof v[k], + }) : "" ); return acc; @@ -391,7 +424,7 @@ export class BasicTable extends LitElement { // Sort Columns by Key Column and Requirement const keys = (this.#keys = - this.colHeaders = + this.colHeaders = sortTable( { ...this.#itemSchema, @@ -420,13 +453,13 @@ export class BasicTable extends LitElement { ${data.map( - (row, i) => - html` + (row, i) => + html` ${row.map( - (col, j) => html`
${JSON.stringify(col)}
` - )} + (col, j) => html`
${JSON.stringify(col)}
` + )} ` - )} + )} @@ -435,38 +468,38 @@ export class BasicTable extends LitElement { primary size="small" @click=${() => { - const input = document.createElement("input"); - input.type = "file"; - input.accept = "text/tab-separated-values"; - input.click(); - input.onchange = () => { - const file = input.files[0]; - const reader = new FileReader(); - reader.onload = () => { - this.#readTSV(reader.result); - this.requestUpdate(); - }; - reader.readAsText(file); - }; - }} + const input = document.createElement("input"); + input.type = "file"; + input.accept = "text/tab-separated-values"; + input.click(); + input.onchange = () => { + const file = input.files[0]; + const reader = new FileReader(); + reader.onload = () => { + this.#readTSV(reader.result); + this.requestUpdate(); + }; + reader.readAsText(file); + }; + }} >Upload TSV File { - const tsv = this.#getTSV(); - - const element = document.createElement("a"); - element.setAttribute( - "href", - "data:text/tab-separated-values;charset=utf-8," + encodeURIComponent(tsv) - ); - element.setAttribute("download", `${this.name.split(" ").join("_")}.tsv`); - element.style.display = "none"; - document.body.appendChild(element); - element.click(); - document.body.removeChild(element); - }} + const tsv = this.#getTSV(); + + const element = document.createElement("a"); + element.setAttribute( + "href", + "data:text/tab-separated-values;charset=utf-8," + encodeURIComponent(tsv) + ); + element.setAttribute("download", `${this.name.split(" ").join("_")}.tsv`); + element.style.display = "none"; + document.body.appendChild(element); + element.click(); + document.body.removeChild(element); + }} >Download TSV File diff --git a/src/renderer/src/stories/SimpleTable.js b/src/renderer/src/stories/SimpleTable.js index 9d63d3e86..81fa25125 100644 --- a/src/renderer/src/stories/SimpleTable.js +++ b/src/renderer/src/stories/SimpleTable.js @@ -118,9 +118,14 @@ export class SimpleTable extends LitElement { left: 0; z-index: 1; } + + table tr:first-child td { + border-top: 0px; + } th { border-right: 1px solid gray; + border-bottom: 1px solid gray; color: #222; font-weight: 400; text-align: center; diff --git a/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js b/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js index 65e8cbd07..a660589a1 100644 --- a/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js +++ b/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js @@ -208,6 +208,8 @@ export class GuidedMetadataPage extends ManagedPage { ); } + + console.log("schema", structuredClone(schema), structuredClone(results)); // Create the form const form = new JSONSchemaForm({ identifier: instanceId, diff --git a/src/renderer/src/validation/validation.ts b/src/renderer/src/validation/validation.ts index c83b5d961..e2de20633 100644 --- a/src/renderer/src/validation/validation.ts +++ b/src/renderer/src/validation/validation.ts @@ -237,17 +237,15 @@ schema.Ecephys.Electrodes = { Electrodes: { // All other column - ['*']: function (this: JSONSchemaForm, name, parent, path) { + ['*']: function (this: JSONSchemaForm, name, _, path) { + + const commonPath = path.slice(0, -2) - const commonPath = path.slice(0, -1) // NOTE: Path does not account for the row index const colPath = [...commonPath, 'ElectrodeColumns'] const { value: electrodeColumns } = get(this.results, colPath) // NOTE: this.results is out of sync with the actual row contents at the moment of validation - - // console.log('Electrode Columns', electrodeColumns, colPath, name, path, this.results.Ecephys.ElectrodeColumns) if (electrodeColumns && !electrodeColumns.find((row: any) => row.name === name)) { - // console.error('Not a valid column', name, electrodeColumns, path, this.results.Ecephys.ElectrodeColumns) return [ { message: 'Not a valid column', @@ -258,7 +256,7 @@ schema.Ecephys.Electrodes = { }, // Group name column - group_name: function (this: JSONSchemaForm, _, __, path, value) { + group_name: function (this: JSONSchemaForm, _, __, ___, value) { const groups = this.results.Ecephys.ElectrodeGroup.map(({ name }) => name) // Groups are validated across all interfaces @@ -277,26 +275,49 @@ schema.Ecephys.Electrodes = { // Update the columns available on the Electrodes table when there is a new name in the ElectrodeColumns table ElectrodeColumns: { ['*']: { - name: function (this: JSONSchemaForm, _, __, path, value) { - - const name = value + '*': function (this: JSONSchemaForm, propName, __, path, value) { const commonPath = path.slice(0, -2) + const electrodesTablePath = [ ...commonPath, 'Electrodes'] + const electrodesTable = this.getFormElement(electrodesTablePath) + const electrodesSchema = electrodesTable.schema // Manipulate the schema that is on the table + const globalElectrodeSchema = getSchema(electrodesTablePath, this.schema) + + const { value: row } = get(this.results, path) + + const currentName = row?.['name'] + + const hasNameUpdate = propName == 'name' && !(value in electrodesSchema.items.properties) - const electrodesSchema = getSchema([ ...commonPath, 'Electrodes'], this.schema) + const resolvedName = hasNameUpdate ? value : currentName - if (!name) return true // Only set when name is actually present + if (value === currentName) return true // No change + if (!resolvedName) return true // Only set when name is actually present - if (!(name in electrodesSchema.items.properties)) { + const schemaToEdit = [electrodesSchema, globalElectrodeSchema] + schemaToEdit.forEach(schema => { + + if (row) delete schema.items.properties[currentName] // Delete previous name from schema + + schema.items.properties[resolvedName] = { + description: propName === 'description' ? value : row?.description, + data_type: propName === 'data_type' ? value : row?.data_type + } + }) + + // Swap the new and current name information + if (hasNameUpdate) { const electrodesTable = this.getFormElement([ ...commonPath, 'Electrodes']) - // electrodesTable.schema.properties[name] = {} // Ensure property is present in the schema now - electrodesSchema.items.properties[name] = {} - electrodesTable.data.forEach(row => name in row ? undefined : row[name] = '') // Set column value as blank if not existent on row - electrodesTable.requestUpdate() + electrodesTable.data.forEach(row => { + if (!(value in row)) row[value] = row[currentName] // Initialize new column with old values + delete row[currentName] // Delete old column + }) } + // Always re-render the Electrodes table on column changes + electrodesTable.requestUpdate() } - } + }, } } } From 018c2447f393902f6ee1b44b2ebe21ea0a82932a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 6 Mar 2024 18:23:10 +0000 Subject: [PATCH 24/24] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pyflask/manageNeuroconv/manage_neuroconv.py | 8 +- src/renderer/src/stories/BasicTable.js | 83 +++++++++---------- src/renderer/src/stories/SimpleTable.js | 2 +- .../pages/guided-mode/data/GuidedMetadata.js | 1 - src/renderer/src/validation/validation.ts | 6 +- 5 files changed, 47 insertions(+), 53 deletions(-) diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index 1fe7f7f35..d8b8459dc 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -22,7 +22,6 @@ EXCLUDED_RECORDING_INTERFACE_PROPERTIES = ["contact_vector", "contact_shapes"] - def is_path_contained(child, parent): parent = Path(parent) child = Path(child) @@ -397,7 +396,6 @@ def on_recording_interface(name, recording_interface): defs["ElectrodeColumn"] = electrode_def defs["ElectrodeColumn"]["required"] = list(electrode_def["properties"].keys()) - new_electrodes_properties = { properties["name"]: {key: value for key, value in properties.items() if key != "name"} for properties in original_electrodes_schema["default"] @@ -943,12 +941,12 @@ def generate_test_data(output_path: str): waveform_extractor=waveform_extractor, output_folder=phy_output_folder, remove_if_exists=True, copy_binary=False ) + def map_dtype(dtype: str) -> str: - if '${header(str)}`; }; - #renderHeader = (str, { description }) => { const th = document.createElement("th"); - const required = this.#itemSchema.required ? this.#itemSchema.required.includes(str) : false; const container = document.createElement("div"); container.classList.add("relative"); @@ -187,10 +185,9 @@ export class BasicTable extends LitElement { tippy(span, { content: `${description[0].toUpperCase() + description.slice(1)}`, allowHTML: true }); } - th.appendChild(container); - return th + return th; }; #getRowData(row, cols = this.colHeaders) { @@ -236,8 +233,8 @@ export class BasicTable extends LitElement { }; status; - onStatusChange = () => { }; - onLoaded = () => { }; + onStatusChange = () => {}; + onLoaded = () => {}; #validateCell = (value, col, row, parent) => { if (!value && !this.validateEmptyCells) return true; // Empty cells are valid @@ -397,9 +394,9 @@ export class BasicTable extends LitElement { render() { this.#updateRendered(); - this.schema = this.schema // Always update the schema + this.schema = this.schema; // Always update the schema - console.warn('RERENDERING') + console.warn("RERENDERING"); const entries = this.#itemProps; @@ -409,8 +406,8 @@ export class BasicTable extends LitElement { Object.keys(v).forEach((k) => !(k in entries) ? (entries[k] = { - type: typeof v[k], - }) + type: typeof v[k], + }) : "" ); return acc; @@ -424,7 +421,7 @@ export class BasicTable extends LitElement { // Sort Columns by Key Column and Requirement const keys = (this.#keys = - this.colHeaders = + this.colHeaders = sortTable( { ...this.#itemSchema, @@ -453,13 +450,13 @@ export class BasicTable extends LitElement { ${data.map( - (row, i) => - html` + (row, i) => + html` ${row.map( - (col, j) => html`
${JSON.stringify(col)}
` - )} + (col, j) => html`
${JSON.stringify(col)}
` + )} ` - )} + )} @@ -468,38 +465,38 @@ export class BasicTable extends LitElement { primary size="small" @click=${() => { - const input = document.createElement("input"); - input.type = "file"; - input.accept = "text/tab-separated-values"; - input.click(); - input.onchange = () => { - const file = input.files[0]; - const reader = new FileReader(); - reader.onload = () => { - this.#readTSV(reader.result); - this.requestUpdate(); - }; - reader.readAsText(file); - }; - }} + const input = document.createElement("input"); + input.type = "file"; + input.accept = "text/tab-separated-values"; + input.click(); + input.onchange = () => { + const file = input.files[0]; + const reader = new FileReader(); + reader.onload = () => { + this.#readTSV(reader.result); + this.requestUpdate(); + }; + reader.readAsText(file); + }; + }} >Upload TSV File { - const tsv = this.#getTSV(); - - const element = document.createElement("a"); - element.setAttribute( - "href", - "data:text/tab-separated-values;charset=utf-8," + encodeURIComponent(tsv) - ); - element.setAttribute("download", `${this.name.split(" ").join("_")}.tsv`); - element.style.display = "none"; - document.body.appendChild(element); - element.click(); - document.body.removeChild(element); - }} + const tsv = this.#getTSV(); + + const element = document.createElement("a"); + element.setAttribute( + "href", + "data:text/tab-separated-values;charset=utf-8," + encodeURIComponent(tsv) + ); + element.setAttribute("download", `${this.name.split(" ").join("_")}.tsv`); + element.style.display = "none"; + document.body.appendChild(element); + element.click(); + document.body.removeChild(element); + }} >Download TSV File diff --git a/src/renderer/src/stories/SimpleTable.js b/src/renderer/src/stories/SimpleTable.js index 81fa25125..7161f52a7 100644 --- a/src/renderer/src/stories/SimpleTable.js +++ b/src/renderer/src/stories/SimpleTable.js @@ -118,7 +118,7 @@ export class SimpleTable extends LitElement { left: 0; z-index: 1; } - + table tr:first-child td { border-top: 0px; } diff --git a/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js b/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js index a660589a1..d68dae20d 100644 --- a/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js +++ b/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js @@ -208,7 +208,6 @@ export class GuidedMetadataPage extends ManagedPage { ); } - console.log("schema", structuredClone(schema), structuredClone(results)); // Create the form const form = new JSONSchemaForm({ diff --git a/src/renderer/src/validation/validation.ts b/src/renderer/src/validation/validation.ts index e2de20633..5fcd40b0b 100644 --- a/src/renderer/src/validation/validation.ts +++ b/src/renderer/src/validation/validation.ts @@ -244,7 +244,7 @@ schema.Ecephys.Electrodes = { const colPath = [...commonPath, 'ElectrodeColumns'] const { value: electrodeColumns } = get(this.results, colPath) // NOTE: this.results is out of sync with the actual row contents at the moment of validation - + if (electrodeColumns && !electrodeColumns.find((row: any) => row.name === name)) { return [ { @@ -306,14 +306,14 @@ schema.Ecephys.Electrodes = { }) // Swap the new and current name information - if (hasNameUpdate) { + if (hasNameUpdate) { const electrodesTable = this.getFormElement([ ...commonPath, 'Electrodes']) electrodesTable.data.forEach(row => { if (!(value in row)) row[value] = row[currentName] // Initialize new column with old values delete row[currentName] // Delete old column }) } - + // Always re-render the Electrodes table on column changes electrodesTable.requestUpdate() }