From c9f5ab4543f6780555197bcae9aba3ad343551f8 Mon Sep 17 00:00:00 2001 From: Garrett Michael Flynn Date: Thu, 14 Mar 2024 12:24:00 -0700 Subject: [PATCH 01/31] Update with relevant demo --- guide_testing_suite.yml | 13 ++ pyflask/manageNeuroconv/manage_neuroconv.py | 178 ++++++++++++++++---- schemas/base-metadata.schema.ts | 4 +- 3 files changed, 156 insertions(+), 39 deletions(-) diff --git a/guide_testing_suite.yml b/guide_testing_suite.yml index e814ed56c..254d20d38 100644 --- a/guide_testing_suite.yml +++ b/guide_testing_suite.yml @@ -33,3 +33,16 @@ pipelines: TDT: TdtRecordingInterface: folder_path: ephy_testing_data/tdt/aep_05 + + CellExplorer: + CellExplorerRecordingInterface: + folder_path: ephy_testing_data/cellexplorer/dataset_4/Peter_MS22_180629_110319_concat_stubbed + + + CellExplorerUnits: + CellExplorerRecordingInterface: + folder_path: ephy_testing_data/cellexplorer/dataset_4/Peter_MS22_180629_110319_concat_stubbed + CellExplorerSortingInterface: + file_path: ephy_testing_data/cellexplorer/dataset_4/Peter_MS22_180629_110319_concat_stubbed/Peter_MS22_180629_110319_concat_stubbed.spikes.cellinfo.mat + + \ No newline at end of file diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index 8dadce731..568447c71 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -19,8 +19,8 @@ announcer = MessageAnnouncer() -EXCLUDED_RECORDING_INTERFACE_PROPERTIES = ["contact_vector", "contact_shapes", "group", "location"] -EXTRA_RECORDING_INTERFACE_PROPERTIES = { +EXCLUDED_RECORDING_INTERFACE_PROPERTIES = [ "contact_vector", "contact_shapes", "group", "location" ] +EXTRA_RECORDING_INTERFACE_PROPERTIES = EXTRA_SORTING_INTERFACE_PROPERTIES = { "brain_area": { "data_type": "str", "description": "The brain area where the electrode is located.", @@ -28,6 +28,8 @@ } } +EXCLUDED_SORTING_INTERFACE_PROPERTIES = [ "location" ] + def is_path_contained(child, parent): parent = Path(parent) @@ -300,8 +302,7 @@ def get_source_schema(interface_class_dict: dict) -> dict: return CustomNWBConverter.get_source_schema() -def map_recording_interfaces(callback, converter): - from neuroconv.datainterfaces.ecephys.baserecordingextractorinterface import BaseRecordingExtractorInterface +def map_interfaces(BaseRecordingExtractorInterface, callback, converter): output = [] @@ -327,6 +328,7 @@ def get_metadata_schema(source_data: Dict[str, dict], interfaces: dict) -> Dict[ # Clear the Electrodes information for being set as a collection of Interfaces has_ecephys = "Ecephys" in metadata + has_electrodes = False if has_ecephys: metadata["Ecephys"]["Electrodes"] = {} @@ -334,45 +336,65 @@ def get_metadata_schema(source_data: Dict[str, dict], interfaces: dict) -> Dict[ schema["properties"]["Ecephys"]["required"].append("Electrodes") ecephys_properties = schema["properties"]["Ecephys"]["properties"] original_electrodes_schema = ecephys_properties["Electrodes"] - - ecephys_properties["Electrodes"] = {"type": "object", "properties": {}, "required": []} - + has_electrodes = True # original_electrodes_schema.get('default') + + # Add Electrodes to the schema + if has_electrodes: + ecephys_properties["Electrodes"] = {"type": "object", "properties": {}, "required": []} + + + def on_sorting_interface(name, sorting_interface): + + # metadata["Ecephys"]["Units"][name] = dict( + # Units=get_unit_table_json(sorting_interface), + # UnitColumns=get_unit_columns_json(sorting_interface), + # ) + + return sorting_interface + def on_recording_interface(name, recording_interface): - metadata["Ecephys"]["Electrodes"][name] = dict( - Electrodes=get_electrode_table_json(recording_interface), - ElectrodeColumns=get_electrode_columns_json(recording_interface), - ) + if has_electrodes: - ecephys_properties["Electrodes"]["properties"][name] = dict( - type="object", - properties=dict( - Electrodes={ - "type": "array", - "minItems": 0, - "items": {"$ref": "#/properties/Ecephys/properties/definitions/Electrode"}, - }, - ElectrodeColumns={ - "type": "array", - "minItems": 0, - "items": {"$ref": "#/properties/Ecephys/properties/definitions/ElectrodeColumn"}, - }, - ), - required=["Electrodes", "ElectrodeColumns"], - ) + metadata["Ecephys"]["Electrodes"][name] = dict( + Electrodes=get_electrode_table_json(recording_interface), + ElectrodeColumns=get_electrode_columns_json(recording_interface), + ) - ecephys_properties["Electrodes"]["required"].append(name) + ecephys_properties["Electrodes"]["properties"][name] = dict( + type="object", + properties=dict( + Electrodes={ + "type": "array", + "minItems": 0, + "items": {"$ref": "#/properties/Ecephys/properties/definitions/Electrode"}, + }, + ElectrodeColumns={ + "type": "array", + "minItems": 0, + "items": {"$ref": "#/properties/Ecephys/properties/definitions/ElectrodeColumn"}, + }, + ), + required=["Electrodes", "ElectrodeColumns"], + ) + + ecephys_properties["Electrodes"]["required"].append(name) return recording_interface - recording_interfaces = map_recording_interfaces(on_recording_interface, converter) + from neuroconv.datainterfaces.ecephys.baserecordingextractorinterface import BaseRecordingExtractorInterface + from neuroconv.datainterfaces.ecephys.basesortingextractorinterface import BaseSortingExtractorInterface - # Delete Ecephys metadata if ElectrodeTable helper function is not available - if has_ecephys: - if len(recording_interfaces) == 0: - schema["properties"].pop("Ecephys", dict()) + # Map recording interfaces to metadata + map_interfaces(BaseRecordingExtractorInterface, on_recording_interface, converter) - else: + # Map sorting interfaces to metadata + map_interfaces(BaseSortingExtractorInterface, on_sorting_interface, converter) + + # Delete Ecephys metadata if no interfaces processed + if has_ecephys: + + if (has_electrodes): defs = ecephys_properties["definitions"] electrode_def = defs["Electrodes"] @@ -405,7 +427,7 @@ def on_recording_interface(name, recording_interface): new_electrodes_properties = { properties["name"]: {key: value for key, value in properties.items() if key != "name"} - for properties in original_electrodes_schema["default"] + for properties in original_electrodes_schema.get('default', {}) if properties["name"] not in EXCLUDED_RECORDING_INTERFACE_PROPERTIES } @@ -984,6 +1006,87 @@ def get_recording_interface_properties(recording_interface) -> Dict[str, Any]: return properties +def get_sorting_interface_properties(sorting_interface) -> Dict[str, Any]: + """A convenience function for uniformly excluding certain properties of the provided sorting extractor.""" + property_names = list(sorting_interface.sorting_extractor.get_property_keys()) + + properties = { + property_name: sorting_interface.sorting_extractor.get_property(key=property_name) + for property_name in property_names + if property_name not in EXCLUDED_SORTING_INTERFACE_PROPERTIES + } + + for property_name, property_info in EXTRA_SORTING_INTERFACE_PROPERTIES.items(): + if property_name not in properties: + properties[property_name] = property_info + + return properties + + +def get_unit_columns_json(interface) -> List[Dict[str, Any]]: + """A convenience function for collecting and organizing the properties of the underlying sorting extractor.""" + properties = get_sorting_interface_properties(interface) + + # Hardcoded for Phy (NOTE: Update for more interfaces) + property_descriptions = dict( + # spike_times="The times of the spikes, in seconds.", + # spike_clusters="The cluster IDs of the spikes.", + # spike_templates="The template IDs of the spikes.", + # spike_amplitudes="The amplitudes of the spikes.", + # spike_depths="The depths of the spikes.", + # spike_widths="The widths of the spikes.", + ) + + for property_name, property_info in EXTRA_SORTING_INTERFACE_PROPERTIES.items(): + description = property_info.get("description", None) + if description: + property_descriptions[property_name] = description + + sorting_extractor = interface.sorting_extractor + unit_ids = sorting_extractor.get_unit_ids() + + unit_columns = [ + dict( + name=property_name, + description=property_descriptions.get(property_name, "No description."), + data_type=get_property_dtype( + recording_extractor=sorting_extractor, property_name=property_name, channel_ids=[unit_ids[0]] + ), + ) + for property_name in properties.keys() + ] + + return json.loads(json.dumps(obj=unit_columns)) + +def get_unit_table_json(interface) -> List[Dict[str, Any]]: + """ + A convenience function for collecting and organizing the property values of the underlying sorting extractor. + """ + + from neuroconv.utils import NWBMetaDataEncoder + + sorting = interface.sorting_extractor + + properties = get_sorting_interface_properties(interface) + + unit_ids = sorting.get_unit_ids() + + table = list() + for unit_id in unit_ids: + unit_column = dict() + for property_name in properties: + if property_name in EXTRA_SORTING_INTERFACE_PROPERTIES: + sorting_property_value = properties[property_name]["default"] + else: + sorting_property_value = sorting.get_property(key=property_name, ids=[unit_id])[ + 0 # First axis is always units in SI + ] # Since only fetching one unit at a time, use trivial zero-index + unit_column.update({property_name: sorting_property_value}) + table.append(unit_column) + table_as_json = json.loads(json.dumps(table, cls=NWBMetaDataEncoder)) + + return table_as_json + def get_electrode_columns_json(interface) -> List[Dict[str, Any]]: """A convenience function for collecting and organizing the properties of the underlying recording extractor.""" @@ -1057,7 +1160,10 @@ def get_electrode_table_json(interface) -> List[Dict[str, Any]]: electrode_column = dict() for property_name in properties: if property_name in EXTRA_RECORDING_INTERFACE_PROPERTIES: - recording_property_value = properties[property_name]["default"] + try: + recording_property_value = properties[property_name].get('default') # Get default value + except: + recording_property_value = properties[property_name][0] # Get first value else: recording_property_value = recording.get_property(key=property_name, ids=[electrode_id])[ 0 # First axis is always electodes in SI diff --git a/schemas/base-metadata.schema.ts b/schemas/base-metadata.schema.ts index 02fd6a61f..d990c6796 100644 --- a/schemas/base-metadata.schema.ts +++ b/schemas/base-metadata.schema.ts @@ -109,9 +109,7 @@ export const preprocessMetadataSchema = (schema: any = baseMetadataSchema, globa const uvProperties = ["gain_to_uV", "offset_to_uV"] uvProperties.forEach(prop => { - electrodeItems[prop] = {} - electrodeItems[prop].title = prop.replace('uV', uvMathFormat) - console.log(electrodeItems[prop]) + if (electrodeItems[prop]) electrodeItems[prop].title = prop.replace('uV', uvMathFormat) }) interfaceProps["Electrodes"].items.order = ["channel_name", "group_name", "shank_electrode_number", ...uvProperties]; interfaceProps["ElectrodeColumns"].items.order = ["name", "description", "data_type"]; From 8b12e2802dbce82d6503ab416d9c53416e98a43f Mon Sep 17 00:00:00 2001 From: Garrett Michael Flynn Date: Thu, 14 Mar 2024 12:25:42 -0700 Subject: [PATCH 02/31] Swap suppressed keys --- .../src/stories/pages/guided-mode/data/GuidedMetadata.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js b/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js index a6231e903..c974fb491 100644 --- a/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js +++ b/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js @@ -55,7 +55,7 @@ const propsToIgnore = { }, }, Ecephys: { - UnitProperties: true, + ElectricalSeries: true, ElectricalSeriesLF: true, ElectricalSeriesAP: true, Electrodes: { From b2e8ed2a8890fa54f10f93471b230a09a980e53f Mon Sep 17 00:00:00 2001 From: Garrett Michael Flynn Date: Fri, 15 Mar 2024 09:22:42 -0700 Subject: [PATCH 03/31] Show units. No editing allowed --- pyflask/manageNeuroconv/manage_neuroconv.py | 143 ++++++++++++++---- schemas/base-metadata.schema.ts | 71 ++++++--- src/renderer/src/stories/BasicTable.js | 8 +- .../pages/guided-mode/data/GuidedMetadata.js | 20 ++- 4 files changed, 184 insertions(+), 58 deletions(-) diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index 568447c71..d2a51e0d1 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -28,7 +28,31 @@ } } -EXCLUDED_SORTING_INTERFACE_PROPERTIES = [ "location" ] +EXCLUDED_SORTING_INTERFACE_PROPERTIES = [ + "location", + "spike_times", "electrodes" # Not validated +] + +DTYPE_DESCRIPTIONS = { + "bool": "logical", + "str": "string", + "ndarray": "n-dimensional array", + "float8": "8-bit number", + "float16": "16-bit number", + "float32": "32-bit number", + "float64": "64-bit number", + "int8": "8-bit integer", + "int16": "16-bit integer", + "int32": "32-bit integer", + "int64": "64-bit integer", +} + +DTYPE_SCHEMA = { + "type": "string", + "strict": False, + "enum": list(DTYPE_DESCRIPTIONS.keys()), + "enumLabels": DTYPE_DESCRIPTIONS, +} def is_path_contained(child, parent): @@ -329,12 +353,16 @@ def get_metadata_schema(source_data: Dict[str, dict], interfaces: dict) -> Dict[ # Clear the Electrodes information for being set as a collection of Interfaces has_ecephys = "Ecephys" in metadata has_electrodes = False + has_units = False if has_ecephys: - metadata["Ecephys"]["Electrodes"] = {} - schema["properties"]["Ecephys"]["required"].append("Electrodes") ecephys_properties = schema["properties"]["Ecephys"]["properties"] + + + # Populate Electrodes metadata + metadata["Ecephys"]["Electrodes"] = {} + schema["properties"]["Ecephys"]["required"].append("Electrodes") original_electrodes_schema = ecephys_properties["Electrodes"] has_electrodes = True # original_electrodes_schema.get('default') @@ -343,30 +371,67 @@ def get_metadata_schema(source_data: Dict[str, dict], interfaces: dict) -> Dict[ ecephys_properties["Electrodes"] = {"type": "object", "properties": {}, "required": []} + # Populate Units metadata + metadata["Ecephys"]["Units"] = {} + schema["properties"]["Ecephys"]["required"].append("Units") + original_units_schema = ecephys_properties.get("UnitProperties") # NOTE: Not specific to interface + has_units = original_units_schema is not None + + if has_units: + metadata["Ecephys"].pop("UnitProperties") # Remove UnitProperties from metadata + ecephys_properties.pop("UnitProperties") # Remove UnitProperties from schema + ecephys_properties["Units"] = {"type": "object", "properties": {}, "required": []} + + def on_sorting_interface(name, sorting_interface): - # metadata["Ecephys"]["Units"][name] = dict( - # Units=get_unit_table_json(sorting_interface), - # UnitColumns=get_unit_columns_json(sorting_interface), - # ) - + units_data = metadata["Ecephys"]["Units"][name] = dict( + Units=get_unit_table_json(sorting_interface), + UnitColumns=get_unit_columns_json(sorting_interface), + ) + + n_units = len(units_data["Units"]) + + ecephys_properties["Units"]["properties"][name] = dict( + type="object", + properties=dict( + Units={ + "type": "array", + "minItems": n_units, + "maxItems": n_units, + "items": {"$ref": "#/properties/Ecephys/properties/definitions/Unit"}, + }, + UnitColumns={ + "type": "array", + "minItems": 0, + "items": {"$ref": "#/properties/Ecephys/properties/definitions/UnitColumn"}, + }, + ), + required=["Units", "UnitColumns"], + ) + + ecephys_properties["Units"]["required"].append(name) + return sorting_interface def on_recording_interface(name, recording_interface): if has_electrodes: - metadata["Ecephys"]["Electrodes"][name] = dict( + electrode_data = metadata["Ecephys"]["Electrodes"][name] = dict( Electrodes=get_electrode_table_json(recording_interface), ElectrodeColumns=get_electrode_columns_json(recording_interface), ) + n_electrodes = len(electrode_data["Electrodes"]) + ecephys_properties["Electrodes"]["properties"][name] = dict( type="object", properties=dict( Electrodes={ "type": "array", - "minItems": 0, + "minItems": n_electrodes, + "maxItems": n_electrodes, "items": {"$ref": "#/properties/Ecephys/properties/definitions/Electrode"}, }, ElectrodeColumns={ @@ -394,32 +459,14 @@ def on_recording_interface(name, recording_interface): # Delete Ecephys metadata if no interfaces processed if has_ecephys: + defs = ecephys_properties["definitions"] + if (has_electrodes): - defs = ecephys_properties["definitions"] electrode_def = defs["Electrodes"] - dtype_descriptions = { - "bool": "logical", - "str": "string", - "ndarray": "n-dimensional array", - "float8": "8-bit number", - "float16": "16-bit number", - "float32": "32-bit number", - "float64": "64-bit number", - "int8": "8-bit integer", - "int16": "16-bit integer", - "int32": "32-bit integer", - "int64": "64-bit integer", - } - # NOTE: Update to output from NeuroConv - electrode_def["properties"]["data_type"] = { - "type": "string", - "strict": False, - "enum": list(dtype_descriptions.keys()), - "enumLabels": dtype_descriptions, - } + electrode_def["properties"]["data_type"] = DTYPE_SCHEMA # Configure electrode columns defs["ElectrodeColumn"] = electrode_def @@ -437,6 +484,30 @@ def on_recording_interface(name, recording_interface): "additionalProperties": True, # Allow for new columns } + if has_units: + + unitprops_def = defs["UnitProperties"] + + # NOTE: Update to output from NeuroConv + unitprops_def["properties"]["data_type"] = DTYPE_SCHEMA + + # Configure electrode columns + defs["UnitColumn"] = unitprops_def + defs["UnitColumn"]["required"] = list(unitprops_def["properties"].keys()) + + new_units_properties = { + properties["name"]: {key: value for key, value in properties.items() if key != "name"} + for properties in original_units_schema.get('default', {}) + if properties["name"] not in EXCLUDED_SORTING_INTERFACE_PROPERTIES + } + + defs["Unit"] = { + "type": "object", + "properties": new_units_properties, + "additionalProperties": True, # Allow for new columns + } + + return json.loads(json.dumps(replace_nan_with_none(dict(results=metadata, schema=schema)), cls=NWBMetaDataEncoder)) @@ -593,6 +664,11 @@ def update_conversion_progress(**kwargs): if ecephys_metadata: + # Quick fix to remove units + has_units = "Units" in ecephys_metadata + if has_units: + del ecephys_metadata["Units"] + for interface_name, interface_electrode_results in ecephys_metadata["Electrodes"].items(): interface = converter.data_interface_objects[interface_name] @@ -1212,7 +1288,7 @@ def update_recording_properties_from_table_as_json( for entry_index, entry in enumerate(electrode_table_json): electrode_properties = dict(entry) # copy - channel_name = electrode_properties.pop("channel_name") + channel_name = electrode_properties.pop("channel_name", None) for property_name, property_value in electrode_properties.items(): if property_name not in electrode_column_data_types: # Skip data with missing column information continue @@ -1221,10 +1297,11 @@ def update_recording_properties_from_table_as_json( # property_index = contact_vector_property_names.index(property_name) # modified_contact_vector[entry_index][property_index] = property_value else: + ids = [stream_prefix + "#" + channel_name] if channel_name else [] # Correct for minimal metadata (e.g. CellExplorer) recording_extractor.set_property( key=property_name, values=np.array([property_value], dtype=electrode_column_data_types[property_name]), - ids=[stream_prefix + "#" + channel_name], + ids=ids, ) # TODO: uncomment when neuroconv supports contact vectors (probe interface) diff --git a/schemas/base-metadata.schema.ts b/schemas/base-metadata.schema.ts index d990c6796..8ab337e80 100644 --- a/schemas/base-metadata.schema.ts +++ b/schemas/base-metadata.schema.ts @@ -4,7 +4,11 @@ import { header, replaceRefsWithValue } from '../src/renderer/src/stories/forms/ import baseMetadataSchema from './json/base_metadata_schema.json' assert { type: "json" } -const uvMathFormat = `µV`; //`µV` +import { merge } from '../src/renderer/src/stories/pages/utils' + +const UV_MATH_FORMAT = `µV`; //`µV` +const UV_PROPERTIES = ["gain_to_uV", "offset_to_uV"] +const COLUMN_SCHEMA_ORDER = ["name", "description", "data_type"] function getSpeciesNameComponents(arr: any[]) { const split = arr[arr.length - 1].split(' - ') @@ -14,8 +18,6 @@ function getSpeciesNameComponents(arr: any[]) { } } - - function getSpeciesInfo(species: any[][] = []) { @@ -37,10 +39,34 @@ function getSpeciesInfo(species: any[][] = []) { } -const propsToInclude = { - ecephys: ["Device", "ElectrodeGroup", "Electrodes", "ElectrodeColumns", "definitions"] -} +function updateEcephysTable(propName, schema, schemaToMerge) { + + const ecephys = schema.properties.Ecephys + + // Change rendering order for electrode table columns + const electrodesProp = ecephys.properties[propName] + for (let name in electrodesProp.properties) { + const interfaceProps = electrodesProp.properties[name].properties + + for (let subProp in schemaToMerge) { + if (interfaceProps[subProp]) { + const itemSchema = interfaceProps[subProp].items + + // Do not add new items + const updateCopy = structuredClone(schemaToMerge[subProp]) + const updateProps = updateCopy.properties + for (let itemProp in updateProps) { + if (!itemSchema.properties[itemProp]) delete updateProps[itemProp] + } + + // Merge into existing items + merge(updateCopy, itemSchema) + } + } + } + +} export const preprocessMetadataSchema = (schema: any = baseMetadataSchema, global = false) => { @@ -101,20 +127,27 @@ export const preprocessMetadataSchema = (schema: any = baseMetadataSchema, globa if (ecephys) { - // Change rendering order for electrode table columns - const electrodesProp = ecephys.properties["Electrodes"] - for (let name in electrodesProp.properties) { - const interfaceProps = electrodesProp.properties[name].properties - const electrodeItems = interfaceProps["Electrodes"].items.properties - const uvProperties = ["gain_to_uV", "offset_to_uV"] - - uvProperties.forEach(prop => { - if (electrodeItems[prop]) electrodeItems[prop].title = prop.replace('uV', uvMathFormat) - }) - interfaceProps["Electrodes"].items.order = ["channel_name", "group_name", "shank_electrode_number", ...uvProperties]; - interfaceProps["ElectrodeColumns"].items.order = ["name", "description", "data_type"]; + updateEcephysTable("Electrodes", copy, { + Electrodes: { + properties: UV_PROPERTIES.reduce((acc, prop) => { + acc[prop] = { title: prop.replace('uV', UV_MATH_FORMAT) } + return acc + }, {}), + order: ["channel_name", "group_name", "shank_electrode_number", ...UV_PROPERTIES] + }, + ElectrodeColumns: { + order: COLUMN_SCHEMA_ORDER + } + }) - } + updateEcephysTable("Units", copy, { + Units: { + order: ["clu_id", "group_id"] + }, + UnitColumns: { + order: COLUMN_SCHEMA_ORDER + } + }) } diff --git a/src/renderer/src/stories/BasicTable.js b/src/renderer/src/stories/BasicTable.js index 30fa73566..3fb922956 100644 --- a/src/renderer/src/stories/BasicTable.js +++ b/src/renderer/src/stories/BasicTable.js @@ -123,6 +123,7 @@ export class BasicTable extends LitElement { onStatusChange, onLoaded, onUpdate, + editable = true, } = {}) { super(); this.name = name ?? "data_table"; @@ -138,6 +139,8 @@ export class BasicTable extends LitElement { if (onUpdate) this.onUpdate = onUpdate; if (onStatusChange) this.onStatusChange = onStatusChange; if (onLoaded) this.onLoaded = onLoaded; + + this.editable = editable } #schema = {}; @@ -472,7 +475,8 @@ export class BasicTable extends LitElement { -
+ ${this.editable ? + html`
Download TSV File -
+
` : ''} `; } } diff --git a/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js b/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js index c974fb491..cb3ed0b12 100644 --- a/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js +++ b/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js @@ -25,6 +25,19 @@ import { Button } from "../../../Button.js"; import globalIcon from "../../../assets/global.svg?raw"; +const tableRenderConfig = { + '*': (metadata) => new SimpleTable(metadata), + Electrodes: true, + Units: (metadata) => { + metadata.editable = false; + return true + }, + UnitColumns: (metadata) => { + metadata.editable = false; + return true + } +} + const imagingPlaneKey = "imaging_plane"; const propsToIgnore = { Ophys: { @@ -406,10 +419,9 @@ export class GuidedMetadataPage extends ManagedPage { const updatedSchema = structuredClone(metadata.schema); metadata.schema = updatedSchema; - // NOTE: Handsontable will occasionally have a context menu that doesn't actually trigger any behaviors - if (name !== "Electrodes") return new SimpleTable(metadata); - else return true; // All other tables are handled by the default behavior - // if (name !== "ElectrodeColumns" && name !== "Electrodes") return new Table(metadata); + const tableConfig = tableRenderConfig[name] ?? tableRenderConfig["*"] ?? true; + if (typeof tableConfig === "function") return tableConfig(metadata); + else return tableConfig }, onThrow, }); From 6854175c6cabd405434de52de002ba8e79a9905b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 15 Mar 2024 16:24:45 +0000 Subject: [PATCH 04/31] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- guide_testing_suite.yml | 4 +- pyflask/manageNeuroconv/manage_neuroconv.py | 71 ++++++++-------- src/renderer/src/stories/BasicTable.js | 85 ++++++++++--------- .../pages/guided-mode/data/GuidedMetadata.js | 12 +-- 4 files changed, 85 insertions(+), 87 deletions(-) diff --git a/guide_testing_suite.yml b/guide_testing_suite.yml index 254d20d38..041ed59ab 100644 --- a/guide_testing_suite.yml +++ b/guide_testing_suite.yml @@ -33,7 +33,7 @@ pipelines: TDT: TdtRecordingInterface: folder_path: ephy_testing_data/tdt/aep_05 - + CellExplorer: CellExplorerRecordingInterface: folder_path: ephy_testing_data/cellexplorer/dataset_4/Peter_MS22_180629_110319_concat_stubbed @@ -45,4 +45,4 @@ pipelines: CellExplorerSortingInterface: file_path: ephy_testing_data/cellexplorer/dataset_4/Peter_MS22_180629_110319_concat_stubbed/Peter_MS22_180629_110319_concat_stubbed.spikes.cellinfo.mat - \ No newline at end of file + diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index d2a51e0d1..666740298 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -19,7 +19,7 @@ announcer = MessageAnnouncer() -EXCLUDED_RECORDING_INTERFACE_PROPERTIES = [ "contact_vector", "contact_shapes", "group", "location" ] +EXCLUDED_RECORDING_INTERFACE_PROPERTIES = ["contact_vector", "contact_shapes", "group", "location"] EXTRA_RECORDING_INTERFACE_PROPERTIES = EXTRA_SORTING_INTERFACE_PROPERTIES = { "brain_area": { "data_type": "str", @@ -28,10 +28,7 @@ } } -EXCLUDED_SORTING_INTERFACE_PROPERTIES = [ - "location", - "spike_times", "electrodes" # Not validated -] +EXCLUDED_SORTING_INTERFACE_PROPERTIES = ["location", "spike_times", "electrodes"] # Not validated DTYPE_DESCRIPTIONS = { "bool": "logical", @@ -359,32 +356,29 @@ def get_metadata_schema(source_data: Dict[str, dict], interfaces: dict) -> Dict[ ecephys_properties = schema["properties"]["Ecephys"]["properties"] - # Populate Electrodes metadata metadata["Ecephys"]["Electrodes"] = {} schema["properties"]["Ecephys"]["required"].append("Electrodes") original_electrodes_schema = ecephys_properties["Electrodes"] - has_electrodes = True # original_electrodes_schema.get('default') + has_electrodes = True # original_electrodes_schema.get('default') # Add Electrodes to the schema if has_electrodes: ecephys_properties["Electrodes"] = {"type": "object", "properties": {}, "required": []} - # Populate Units metadata metadata["Ecephys"]["Units"] = {} schema["properties"]["Ecephys"]["required"].append("Units") - original_units_schema = ecephys_properties.get("UnitProperties") # NOTE: Not specific to interface + original_units_schema = ecephys_properties.get("UnitProperties") # NOTE: Not specific to interface has_units = original_units_schema is not None if has_units: - metadata["Ecephys"].pop("UnitProperties") # Remove UnitProperties from metadata - ecephys_properties.pop("UnitProperties") # Remove UnitProperties from schema + metadata["Ecephys"].pop("UnitProperties") # Remove UnitProperties from metadata + ecephys_properties.pop("UnitProperties") # Remove UnitProperties from schema ecephys_properties["Units"] = {"type": "object", "properties": {}, "required": []} - def on_sorting_interface(name, sorting_interface): - + units_data = metadata["Ecephys"]["Units"][name] = dict( Units=get_unit_table_json(sorting_interface), UnitColumns=get_unit_columns_json(sorting_interface), @@ -393,27 +387,27 @@ def on_sorting_interface(name, sorting_interface): n_units = len(units_data["Units"]) ecephys_properties["Units"]["properties"][name] = dict( - type="object", - properties=dict( - Units={ - "type": "array", - "minItems": n_units, - "maxItems": n_units, - "items": {"$ref": "#/properties/Ecephys/properties/definitions/Unit"}, - }, - UnitColumns={ - "type": "array", - "minItems": 0, - "items": {"$ref": "#/properties/Ecephys/properties/definitions/UnitColumn"}, - }, - ), - required=["Units", "UnitColumns"], + type="object", + properties=dict( + Units={ + "type": "array", + "minItems": n_units, + "maxItems": n_units, + "items": {"$ref": "#/properties/Ecephys/properties/definitions/Unit"}, + }, + UnitColumns={ + "type": "array", + "minItems": 0, + "items": {"$ref": "#/properties/Ecephys/properties/definitions/UnitColumn"}, + }, + ), + required=["Units", "UnitColumns"], ) ecephys_properties["Units"]["required"].append(name) return sorting_interface - + def on_recording_interface(name, recording_interface): if has_electrodes: @@ -458,10 +452,10 @@ def on_recording_interface(name, recording_interface): # Delete Ecephys metadata if no interfaces processed if has_ecephys: - + defs = ecephys_properties["definitions"] - if (has_electrodes): + if has_electrodes: electrode_def = defs["Electrodes"] @@ -474,7 +468,7 @@ def on_recording_interface(name, recording_interface): new_electrodes_properties = { properties["name"]: {key: value for key, value in properties.items() if key != "name"} - for properties in original_electrodes_schema.get('default', {}) + for properties in original_electrodes_schema.get("default", {}) if properties["name"] not in EXCLUDED_RECORDING_INTERFACE_PROPERTIES } @@ -497,7 +491,7 @@ def on_recording_interface(name, recording_interface): new_units_properties = { properties["name"]: {key: value for key, value in properties.items() if key != "name"} - for properties in original_units_schema.get('default', {}) + for properties in original_units_schema.get("default", {}) if properties["name"] not in EXCLUDED_SORTING_INTERFACE_PROPERTIES } @@ -507,7 +501,6 @@ def on_recording_interface(name, recording_interface): "additionalProperties": True, # Allow for new columns } - return json.loads(json.dumps(replace_nan_with_none(dict(results=metadata, schema=schema)), cls=NWBMetaDataEncoder)) @@ -1082,6 +1075,7 @@ def get_recording_interface_properties(recording_interface) -> Dict[str, Any]: return properties + def get_sorting_interface_properties(sorting_interface) -> Dict[str, Any]: """A convenience function for uniformly excluding certain properties of the provided sorting extractor.""" property_names = list(sorting_interface.sorting_extractor.get_property_keys()) @@ -1134,6 +1128,7 @@ def get_unit_columns_json(interface) -> List[Dict[str, Any]]: return json.loads(json.dumps(obj=unit_columns)) + def get_unit_table_json(interface) -> List[Dict[str, Any]]: """ A convenience function for collecting and organizing the property values of the underlying sorting extractor. @@ -1237,9 +1232,9 @@ def get_electrode_table_json(interface) -> List[Dict[str, Any]]: for property_name in properties: if property_name in EXTRA_RECORDING_INTERFACE_PROPERTIES: try: - recording_property_value = properties[property_name].get('default') # Get default value + recording_property_value = properties[property_name].get("default") # Get default value except: - recording_property_value = properties[property_name][0] # Get first value + recording_property_value = properties[property_name][0] # Get first value else: recording_property_value = recording.get_property(key=property_name, ids=[electrode_id])[ 0 # First axis is always electodes in SI @@ -1297,7 +1292,9 @@ def update_recording_properties_from_table_as_json( # property_index = contact_vector_property_names.index(property_name) # modified_contact_vector[entry_index][property_index] = property_value else: - ids = [stream_prefix + "#" + channel_name] if channel_name else [] # Correct for minimal metadata (e.g. CellExplorer) + ids = ( + [stream_prefix + "#" + channel_name] if channel_name else [] + ) # Correct for minimal metadata (e.g. CellExplorer) recording_extractor.set_property( key=property_name, values=np.array([property_value], dtype=electrode_column_data_types[property_name]), diff --git a/src/renderer/src/stories/BasicTable.js b/src/renderer/src/stories/BasicTable.js index 3fb922956..dc85d5a44 100644 --- a/src/renderer/src/stories/BasicTable.js +++ b/src/renderer/src/stories/BasicTable.js @@ -140,7 +140,7 @@ export class BasicTable extends LitElement { if (onStatusChange) this.onStatusChange = onStatusChange; if (onLoaded) this.onLoaded = onLoaded; - this.editable = editable + this.editable = editable; } #schema = {}; @@ -475,47 +475,48 @@ export class BasicTable extends LitElement { - ${this.editable ? - html`
- { - const input = document.createElement("input"); - input.type = "file"; - input.accept = "text/tab-separated-values"; - input.click(); - input.onchange = () => { - const file = input.files[0]; - const reader = new FileReader(); - reader.onload = () => { - this.#readTSV(reader.result); - this.requestUpdate(); - }; - reader.readAsText(file); - }; - }} - >Upload TSV File - { - const tsv = this.#getTSV(); - - const element = document.createElement("a"); - element.setAttribute( - "href", - "data:text/tab-separated-values;charset=utf-8," + encodeURIComponent(tsv) - ); - element.setAttribute("download", `${this.name.split(" ").join("_")}.tsv`); - element.style.display = "none"; - document.body.appendChild(element); - element.click(); - document.body.removeChild(element); - }} - >Download TSV File -
` : ''} + ${this.editable + ? html`
+ { + const input = document.createElement("input"); + input.type = "file"; + input.accept = "text/tab-separated-values"; + input.click(); + input.onchange = () => { + const file = input.files[0]; + const reader = new FileReader(); + reader.onload = () => { + this.#readTSV(reader.result); + this.requestUpdate(); + }; + reader.readAsText(file); + }; + }} + >Upload TSV File + { + const tsv = this.#getTSV(); + + const element = document.createElement("a"); + element.setAttribute( + "href", + "data:text/tab-separated-values;charset=utf-8," + encodeURIComponent(tsv) + ); + element.setAttribute("download", `${this.name.split(" ").join("_")}.tsv`); + element.style.display = "none"; + document.body.appendChild(element); + element.click(); + document.body.removeChild(element); + }} + >Download TSV File +
` + : ""} `; } } diff --git a/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js b/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js index cb3ed0b12..a54f73665 100644 --- a/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js +++ b/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js @@ -26,17 +26,17 @@ import { Button } from "../../../Button.js"; import globalIcon from "../../../assets/global.svg?raw"; const tableRenderConfig = { - '*': (metadata) => new SimpleTable(metadata), + "*": (metadata) => new SimpleTable(metadata), Electrodes: true, Units: (metadata) => { metadata.editable = false; - return true + return true; }, UnitColumns: (metadata) => { metadata.editable = false; - return true - } -} + return true; + }, +}; const imagingPlaneKey = "imaging_plane"; const propsToIgnore = { @@ -421,7 +421,7 @@ export class GuidedMetadataPage extends ManagedPage { const tableConfig = tableRenderConfig[name] ?? tableRenderConfig["*"] ?? true; if (typeof tableConfig === "function") return tableConfig(metadata); - else return tableConfig + else return tableConfig; }, onThrow, }); From 0724d929ec45ef7dde04f844927a922412104ded Mon Sep 17 00:00:00 2001 From: Garrett Michael Flynn Date: Fri, 15 Mar 2024 10:10:23 -0700 Subject: [PATCH 05/31] Fix backend and hide empty tables --- pyflask/manageNeuroconv/manage_neuroconv.py | 102 +++++++++--------- schemas/base-metadata.schema.ts | 3 + .../pages/guided-mode/data/GuidedMetadata.js | 2 + 3 files changed, 55 insertions(+), 52 deletions(-) diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index 666740298..bd1347377 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -349,31 +349,33 @@ def get_metadata_schema(source_data: Dict[str, dict], interfaces: dict) -> Dict[ # Clear the Electrodes information for being set as a collection of Interfaces has_ecephys = "Ecephys" in metadata - has_electrodes = False has_units = False if has_ecephys: - ecephys_properties = schema["properties"]["Ecephys"]["properties"] + ecephys_schema = schema["properties"]["Ecephys"] + + if not ecephys_schema.get("required"): + ecephys_schema["required"] = [] + + ecephys_properties = ecephys_schema["properties"] # Populate Electrodes metadata - metadata["Ecephys"]["Electrodes"] = {} - schema["properties"]["Ecephys"]["required"].append("Electrodes") original_electrodes_schema = ecephys_properties["Electrodes"] - has_electrodes = True # original_electrodes_schema.get('default') # Add Electrodes to the schema - if has_electrodes: - ecephys_properties["Electrodes"] = {"type": "object", "properties": {}, "required": []} + metadata["Ecephys"]["Electrodes"] = {} + ecephys_schema["required"].append("Electrodes") + ecephys_properties["Electrodes"] = {"type": "object", "properties": {}, "required": []} # Populate Units metadata metadata["Ecephys"]["Units"] = {} schema["properties"]["Ecephys"]["required"].append("Units") - original_units_schema = ecephys_properties.get("UnitProperties") # NOTE: Not specific to interface + original_units_schema = ecephys_properties.get("UnitProperties", None) # NOTE: Not specific to interface has_units = original_units_schema is not None if has_units: - metadata["Ecephys"].pop("UnitProperties") # Remove UnitProperties from metadata + metadata["Ecephys"].pop("UnitProperties", None) # Remove UnitProperties from metadata (if exists) ecephys_properties.pop("UnitProperties") # Remove UnitProperties from schema ecephys_properties["Units"] = {"type": "object", "properties": {}, "required": []} @@ -410,34 +412,32 @@ def on_sorting_interface(name, sorting_interface): def on_recording_interface(name, recording_interface): - if has_electrodes: + electrode_data = metadata["Ecephys"]["Electrodes"][name] = dict( + Electrodes=get_electrode_table_json(recording_interface), + ElectrodeColumns=get_electrode_columns_json(recording_interface), + ) - electrode_data = metadata["Ecephys"]["Electrodes"][name] = dict( - Electrodes=get_electrode_table_json(recording_interface), - ElectrodeColumns=get_electrode_columns_json(recording_interface), - ) + n_electrodes = len(electrode_data["Electrodes"]) - n_electrodes = len(electrode_data["Electrodes"]) - - ecephys_properties["Electrodes"]["properties"][name] = dict( - type="object", - properties=dict( - Electrodes={ - "type": "array", - "minItems": n_electrodes, - "maxItems": n_electrodes, - "items": {"$ref": "#/properties/Ecephys/properties/definitions/Electrode"}, - }, - ElectrodeColumns={ - "type": "array", - "minItems": 0, - "items": {"$ref": "#/properties/Ecephys/properties/definitions/ElectrodeColumn"}, - }, - ), - required=["Electrodes", "ElectrodeColumns"], - ) + ecephys_properties["Electrodes"]["properties"][name] = dict( + type="object", + properties=dict( + Electrodes={ + "type": "array", + "minItems": n_electrodes, + "maxItems": n_electrodes, + "items": {"$ref": "#/properties/Ecephys/properties/definitions/Electrode"}, + }, + ElectrodeColumns={ + "type": "array", + "minItems": 0, + "items": {"$ref": "#/properties/Ecephys/properties/definitions/ElectrodeColumn"}, + }, + ), + required=["Electrodes", "ElectrodeColumns"], + ) - ecephys_properties["Electrodes"]["required"].append(name) + ecephys_properties["Electrodes"]["required"].append(name) return recording_interface @@ -455,28 +455,26 @@ def on_recording_interface(name, recording_interface): defs = ecephys_properties["definitions"] - if has_electrodes: + electrode_def = defs["Electrodes"] - electrode_def = defs["Electrodes"] + # NOTE: Update to output from NeuroConv + electrode_def["properties"]["data_type"] = DTYPE_SCHEMA - # NOTE: Update to output from NeuroConv - electrode_def["properties"]["data_type"] = DTYPE_SCHEMA + # Configure electrode columns + defs["ElectrodeColumn"] = electrode_def + defs["ElectrodeColumn"]["required"] = list(electrode_def["properties"].keys()) - # Configure electrode columns - defs["ElectrodeColumn"] = electrode_def - defs["ElectrodeColumn"]["required"] = list(electrode_def["properties"].keys()) - - new_electrodes_properties = { - properties["name"]: {key: value for key, value in properties.items() if key != "name"} - for properties in original_electrodes_schema.get("default", {}) - if properties["name"] not in EXCLUDED_RECORDING_INTERFACE_PROPERTIES - } + new_electrodes_properties = { + properties["name"]: {key: value for key, value in properties.items() if key != "name"} + for properties in original_electrodes_schema.get("default", {}) + if properties["name"] not in EXCLUDED_RECORDING_INTERFACE_PROPERTIES + } - defs["Electrode"] = { - "type": "object", - "properties": new_electrodes_properties, - "additionalProperties": True, # Allow for new columns - } + defs["Electrode"] = { + "type": "object", + "properties": new_electrodes_properties, + "additionalProperties": True, # Allow for new columns + } if has_units: diff --git a/schemas/base-metadata.schema.ts b/schemas/base-metadata.schema.ts index 8ab337e80..7f6944f68 100644 --- a/schemas/base-metadata.schema.ts +++ b/schemas/base-metadata.schema.ts @@ -45,6 +45,7 @@ function updateEcephysTable(propName, schema, schemaToMerge) { // Change rendering order for electrode table columns const electrodesProp = ecephys.properties[propName] + if (!electrodesProp) return false for (let name in electrodesProp.properties) { const interfaceProps = electrodesProp.properties[name].properties @@ -66,6 +67,8 @@ function updateEcephysTable(propName, schema, schemaToMerge) { } } + return true + } export const preprocessMetadataSchema = (schema: any = baseMetadataSchema, global = false) => { diff --git a/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js b/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js index a54f73665..2b81d7aeb 100644 --- a/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js +++ b/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js @@ -282,6 +282,8 @@ export class GuidedMetadataPage extends ManagedPage { renderCustomHTML: function (name, inputSchema, localPath, { onUpdate, onThrow }) { if (name === "TwoPhotonSeries" && (!this.value || !this.value.length)) return null; + if (name === "Device" && (!this.value || !this.value.length)) return null; + if (name === "ElectrodeGroup" && (!this.value || !this.value.length)) return null; const isAdditional = isAdditionalProperties(this.pattern); const isPattern = isPatternProperties(this.pattern); From e3aa813d1a7bd0cf4d522427c58c1cbd87e17671 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 15 Mar 2024 17:10:50 +0000 Subject: [PATCH 06/31] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- guide_testing_suite.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/guide_testing_suite.yml b/guide_testing_suite.yml index 041ed59ab..d10b00844 100644 --- a/guide_testing_suite.yml +++ b/guide_testing_suite.yml @@ -44,5 +44,3 @@ pipelines: folder_path: ephy_testing_data/cellexplorer/dataset_4/Peter_MS22_180629_110319_concat_stubbed CellExplorerSortingInterface: file_path: ephy_testing_data/cellexplorer/dataset_4/Peter_MS22_180629_110319_concat_stubbed/Peter_MS22_180629_110319_concat_stubbed.spikes.cellinfo.mat - - From 512b55e4d687003d65f1cce3ab737707c6fe5d50 Mon Sep 17 00:00:00 2001 From: Garrett Michael Flynn Date: Fri, 15 Mar 2024 10:15:59 -0700 Subject: [PATCH 07/31] Skip accordion if empty --- src/renderer/src/stories/JSONSchemaForm.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/renderer/src/stories/JSONSchemaForm.js b/src/renderer/src/stories/JSONSchemaForm.js index 70ea32652..a5f79020e 100644 --- a/src/renderer/src/stories/JSONSchemaForm.js +++ b/src/renderer/src/stories/JSONSchemaForm.js @@ -1242,9 +1242,12 @@ export class JSONSchemaForm extends LitElement { enableToggleContainer.append(enableToggle); Object.assign(enableToggle.style, { marginRight: "10px", pointerEvents: "all" }); + // Skip if accordion will be empty + if (!renderableInside.length) return + const accordion = (this.accordions[name] = new Accordion({ name: headerName, - toggleable: hasMany, + toggleable: hasMany, // Only show toggle if there are multiple siblings subtitle: html`
${explicitlyRequired ? "" : enableToggleContainer}
`, From a26ec043fdfa42c7af544fe1900590b69791f874 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 15 Mar 2024 17:20:20 +0000 Subject: [PATCH 08/31] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/renderer/src/stories/JSONSchemaForm.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/renderer/src/stories/JSONSchemaForm.js b/src/renderer/src/stories/JSONSchemaForm.js index a5f79020e..f0ef7fe5e 100644 --- a/src/renderer/src/stories/JSONSchemaForm.js +++ b/src/renderer/src/stories/JSONSchemaForm.js @@ -1243,8 +1243,8 @@ export class JSONSchemaForm extends LitElement { Object.assign(enableToggle.style, { marginRight: "10px", pointerEvents: "all" }); // Skip if accordion will be empty - if (!renderableInside.length) return - + if (!renderableInside.length) return; + const accordion = (this.accordions[name] = new Accordion({ name: headerName, toggleable: hasMany, // Only show toggle if there are multiple siblings From a5640f1b5b6b024cb5fa6ffafa58f19af3e83e23 Mon Sep 17 00:00:00 2001 From: Garrett Michael Flynn Date: Fri, 15 Mar 2024 13:09:38 -0700 Subject: [PATCH 09/31] Ensure original columns are required and cannot be edited --- pyflask/manageNeuroconv/manage_neuroconv.py | 21 +++++++++++++------ src/renderer/src/stories/BasicTable.js | 7 ++++--- src/renderer/src/stories/JSONSchemaForm.js | 1 - src/renderer/src/stories/SimpleTable.js | 7 ++++++- src/renderer/src/stories/Table.js | 11 +++++++++- src/renderer/src/stories/forms/utils.ts | 13 +++++++++++- .../pages/guided-mode/data/GuidedMetadata.js | 14 +++++++++++-- src/renderer/src/stories/table/Cell.ts | 10 +++++++-- src/renderer/src/stories/table/cells/base.ts | 21 ++++++++++++++----- 9 files changed, 83 insertions(+), 22 deletions(-) diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index bd1347377..61801fd10 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -371,12 +371,12 @@ def get_metadata_schema(source_data: Dict[str, dict], interfaces: dict) -> Dict[ # Populate Units metadata metadata["Ecephys"]["Units"] = {} schema["properties"]["Ecephys"]["required"].append("Units") - original_units_schema = ecephys_properties.get("UnitProperties", None) # NOTE: Not specific to interface + original_units_schema = ecephys_properties.pop("UnitProperties", None) # Remove UnitProperties from schema. NOTE: Not specific to interface has_units = original_units_schema is not None + metadata["Ecephys"].pop("UnitProperties", None) # Always remove top-level UnitProperties from metadata + if has_units: - metadata["Ecephys"].pop("UnitProperties", None) # Remove UnitProperties from metadata (if exists) - ecephys_properties.pop("UnitProperties") # Remove UnitProperties from schema ecephys_properties["Units"] = {"type": "object", "properties": {}, "required": []} def on_sorting_interface(name, sorting_interface): @@ -395,7 +395,12 @@ def on_sorting_interface(name, sorting_interface): "type": "array", "minItems": n_units, "maxItems": n_units, - "items": {"$ref": "#/properties/Ecephys/properties/definitions/Unit"}, + "items": { + "allOf": [ + { "$ref": "#/properties/Ecephys/properties/definitions/Unit" }, + {"required": list(map(lambda info: info["name"], units_data["UnitColumns"]))} + ] + }, }, UnitColumns={ "type": "array", @@ -426,7 +431,12 @@ def on_recording_interface(name, recording_interface): "type": "array", "minItems": n_electrodes, "maxItems": n_electrodes, - "items": {"$ref": "#/properties/Ecephys/properties/definitions/Electrode"}, + "items": { + "allOf": [ + { "$ref": "#/properties/Ecephys/properties/definitions/Electrode" }, + {"required": list(map(lambda info: info["name"], electrode_data["ElectrodeColumns"]))} + ] + }, }, ElectrodeColumns={ "type": "array", @@ -462,7 +472,6 @@ def on_recording_interface(name, recording_interface): # Configure electrode columns defs["ElectrodeColumn"] = electrode_def - defs["ElectrodeColumn"]["required"] = list(electrode_def["properties"].keys()) new_electrodes_properties = { properties["name"]: {key: value for key, value in properties.items() if key != "name"} diff --git a/src/renderer/src/stories/BasicTable.js b/src/renderer/src/stories/BasicTable.js index dc85d5a44..705d4a6ba 100644 --- a/src/renderer/src/stories/BasicTable.js +++ b/src/renderer/src/stories/BasicTable.js @@ -9,7 +9,6 @@ import * as promises from "../promises"; import "./Button"; import { sortTable } from "./Table"; import tippy from "tippy.js"; -import { getIgnore } from "./JSONSchemaForm"; export class BasicTable extends LitElement { static get styles() { @@ -276,11 +275,13 @@ export class BasicTable extends LitElement { let { type, original, inferred } = this.#getType(value, propInfo); + const isUndefined = value === undefined || value === '' + // Check if required - if (!value && "required" in this.#itemSchema && this.#itemSchema.required.includes(col)) + if (isUndefined && "required" in this.#itemSchema && this.#itemSchema.required.includes(col)) result = [{ message: `${col} is a required property`, type: "error" }]; // If not required, check matching types (if provided) for values that are defined - else if (value !== "" && type && inferred !== type) + else if (!isUndefined && type && inferred !== type) result = [{ message: `${col} is expected to be of type ${original}, not ${inferred}`, type: "error" }]; // Otherwise validate using the specified onChange function else result = this.validateOnChange([row, col], parent, value, this.#itemProps[col]); diff --git a/src/renderer/src/stories/JSONSchemaForm.js b/src/renderer/src/stories/JSONSchemaForm.js index a5f79020e..bba3ab43a 100644 --- a/src/renderer/src/stories/JSONSchemaForm.js +++ b/src/renderer/src/stories/JSONSchemaForm.js @@ -57,7 +57,6 @@ export const getSchema = (path, schema, base = []) => { // NOTE: Refs are now pre-resolved const resolved = get(path, schema, ["properties", "patternProperties"], ["patternProperties", "items"]); - // if (resolved?.["$ref"]) return this.getSchema(resolved["$ref"].split("/").slice(1)); // NOTE: This assumes reference to the root of the schema return resolved; }; diff --git a/src/renderer/src/stories/SimpleTable.js b/src/renderer/src/stories/SimpleTable.js index 7161f52a7..a47d945ae 100644 --- a/src/renderer/src/stories/SimpleTable.js +++ b/src/renderer/src/stories/SimpleTable.js @@ -11,7 +11,7 @@ import { styleMap } from "lit/directives/style-map.js"; import "./Button"; import tippy from "tippy.js"; -import { sortTable } from "./Table"; +import { sortTable, getEditable } from "./Table"; import { NestedInputCell } from "./table/cells/input"; import { getIgnore } from "./JSONSchemaForm"; @@ -24,6 +24,7 @@ const isVisible = function (ele, container) { return top <= containerRect.top ? containerRect.top - top <= height : bottom - containerRect.bottom <= height; }; + export class SimpleTable extends LitElement { validateOnChange; onUpdate = () => {}; @@ -201,6 +202,7 @@ export class SimpleTable extends LitElement { maxHeight, contextOptions = {}, ignore = {}, + editable = {} } = {}) { super(); this.schema = schema ?? {}; @@ -213,6 +215,7 @@ export class SimpleTable extends LitElement { this.maxHeight = maxHeight ?? ""; this.ignore = ignore; + this.editable = editable this.contextOptions = contextOptions; @@ -780,6 +783,7 @@ export class SimpleTable extends LitElement { const schema = this.#itemProps[fullInfo.col]; const ignore = getIgnore(this.ignore, [fullInfo.col]); + const isEditable = getEditable(value, this.editable, fullInfo.col) // Track the cell renderer const cell = new TableCell({ @@ -791,6 +795,7 @@ export class SimpleTable extends LitElement { ), col: this.colHeaders[info.j], }, + editable: isEditable, value, schema, ignore, diff --git a/src/renderer/src/stories/Table.js b/src/renderer/src/stories/Table.js index ae738b2ab..4a67723a3 100644 --- a/src/renderer/src/stories/Table.js +++ b/src/renderer/src/stories/Table.js @@ -16,6 +16,13 @@ const isRequired = (col, schema) => { return schema.required?.includes(col); }; +export const getEditable = (value, config, colName) => { + if (typeof config === 'boolean') return config + if (typeof config === 'function') return config(value) + return getEditable(value, config?.[colName] ?? true) +} + + export function sortTable(schema, keyColumn, order) { const cols = Object.keys(schema.properties) @@ -384,7 +391,9 @@ export class Table extends LitElement { return; } - if (!value && required) { + const isUndefined = value == '' + + if (isUndefined && required) { instanceThis.#handleValidationResult( [{ message: `${header(k)} is a required property.`, type: "error" }], this.row, diff --git a/src/renderer/src/stories/forms/utils.ts b/src/renderer/src/stories/forms/utils.ts index 529a39b95..43d09296e 100644 --- a/src/renderer/src/stories/forms/utils.ts +++ b/src/renderer/src/stories/forms/utils.ts @@ -1,6 +1,9 @@ +import { merge } from '../pages/utils' + const toCapitalizeAll = ['nwb', 'api', 'id'] const toCapitalizeNone = ['or', 'and'] + export const createRandomString = () => Math.random().toString(36).substring(7); export const tempPropertyKey = createRandomString(); export const tempPropertyValueKey = createRandomString(); @@ -33,7 +36,15 @@ export const textToArray = (value: string) => value.split("\n") if (prop && typeof prop === "object" && !Array.isArray(prop)) { const internalCopy = (copy[propName] = { ...prop }); const refValue = internalCopy["$ref"] - if (refValue) { + const allOfValue = internalCopy['allOf'] + if (allOfValue) { + copy [propName]= allOfValue.reduce((acc, curr) => { + const result = replaceRefsWithValue({ _temp: curr}, path, parent) + const resolved = result._temp + return merge(resolved, acc) + }, {}) + } + else if (refValue) { const refPath = refValue.split('/').slice(1) // NOTE: Assume from base const resolved = refPath.reduce((acc, key) => acc[key], parent) diff --git a/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js b/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js index 2b81d7aeb..2fc3e0ee7 100644 --- a/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js +++ b/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js @@ -1,4 +1,4 @@ -import { JSONSchemaForm } from "../../../JSONSchemaForm.js"; +import { JSONSchemaForm, getSchema } from "../../../JSONSchemaForm.js"; import { InstanceManager } from "../../../InstanceManager.js"; import { ManagedPage } from "./ManagedPage.js"; @@ -28,6 +28,16 @@ import globalIcon from "../../../assets/global.svg?raw"; const tableRenderConfig = { "*": (metadata) => new SimpleTable(metadata), Electrodes: true, + ElectrodeColumns: function (metadata, fullPath) { + + const electrodeSchema = getSchema([...fullPath.slice(0, -1), 'Electrodes'], this.schema) + return new SimpleTable({ + ...metadata, + editable: { + name: (value) => !electrodeSchema.items.required.includes(value) + } + }) + }, Units: (metadata) => { metadata.editable = false; return true; @@ -422,7 +432,7 @@ export class GuidedMetadataPage extends ManagedPage { metadata.schema = updatedSchema; const tableConfig = tableRenderConfig[name] ?? tableRenderConfig["*"] ?? true; - if (typeof tableConfig === "function") return tableConfig(metadata); + if (typeof tableConfig === "function") return tableConfig.call(form, metadata, [...fullPath, name]); else return tableConfig; }, onThrow, diff --git a/src/renderer/src/stories/table/Cell.ts b/src/renderer/src/stories/table/Cell.ts index 1cda87fa5..d4aa11b23 100644 --- a/src/renderer/src/stories/table/Cell.ts +++ b/src/renderer/src/stories/table/Cell.ts @@ -20,6 +20,7 @@ type OnValidateFunction = (info: ValidationResult) => void type TableCellProps = { value: string, + editable: boolean, info: { col: string } ignore: { [key: string]: boolean }, schema: {[key: string]: any}, @@ -33,6 +34,7 @@ export class TableCell extends LitElement { declare schema: TableCellProps['schema'] declare info: TableCellProps['info'] + declare editable: TableCellProps['editable'] static get styles() { return css` @@ -77,12 +79,13 @@ export class TableCell extends LitElement { type = 'text' - constructor({ info, value, schema, validateOnChange, ignore, onValidate }: TableCellProps) { + constructor({ info, value, editable = true, schema, validateOnChange, ignore, onValidate }: TableCellProps) { super() this.#value = value this.schema = schema this.info = info + this.editable = editable if (validateOnChange) this.validateOnChange = validateOnChange if (ignore) this.ignore = ignore @@ -102,7 +105,9 @@ export class TableCell extends LitElement { } - toggle = (v: boolean) => this.input.toggle(v) + toggle = (v: boolean) => { + if (this.editable) this.input.toggle(v) + } get value() { let v = this.input ? this.input.getValue() : this.#value @@ -216,6 +221,7 @@ export class TableCell extends LitElement { // Only actually rerender if new class type if (cls !== this.#cls) { this.input = new cls({ + editable: this.editable, onChange: async (v) => { if (this.input.interacted) this.interacted = true const result = await this.validate() diff --git a/src/renderer/src/stories/table/cells/base.ts b/src/renderer/src/stories/table/cells/base.ts index 1dca267cb..d815f6ed6 100644 --- a/src/renderer/src/stories/table/cells/base.ts +++ b/src/renderer/src/stories/table/cells/base.ts @@ -5,6 +5,7 @@ type BaseTableProps = { info: { col: string, }, + editable: boolean, toggle: (state?: boolean) => void, schema: any, onOpen: Function, @@ -31,6 +32,9 @@ export class TableCellBase extends LitElement { #editor?: HTMLElement #renderer?: HTMLElement + // Internal variables + #firstUpdated = false + #initialValue: undefined | any static get styles() { return css` @@ -64,6 +68,8 @@ export class TableCellBase extends LitElement { info: BaseTableProps['info']; editToggle: BaseTableProps['toggle'] + editable: BaseTableProps['editable']; + interacted = false constructor({ @@ -73,14 +79,14 @@ export class TableCellBase extends LitElement { onClose, onChange, toggle, - nestedProps + nestedProps, + editable = true }: Partial = {}) { super() this.info = info ?? {} this.schema = schema ?? {} - this.nestedProps = nestedProps ?? {} this.editToggle = toggle ?? (() => {}); @@ -89,6 +95,8 @@ export class TableCellBase extends LitElement { if (onChange) this.onChange = onChange if (onClose) this.onClose = onClose + this.editable = editable + this.#editable.addEventListener('input', (ev: InputEvent) => { this.interacted = true if (ev.inputType.includes('history')) this.setText(this.#editable.innerText) // Catch undo / redo} @@ -106,13 +114,14 @@ export class TableCellBase extends LitElement { onChange: BaseTableProps['onChange'] = () => {} #editableClose = () => this.editToggle(false) - #originalEditorValue = undefined toggle (state = !this.#active) { if (state === this.#active) return if (state) { + if (!this.editable && this.#initialValue) return // Non-editability does not apply to new rows + this.setAttribute('editing', '') const listenForEnter = (ev: KeyboardEvent) => { @@ -163,9 +172,11 @@ export class TableCellBase extends LitElement { getValue = (input: any = this.value) => input // Process inputs from the editor - #update(current: any, forceUpdate = false, runOnChange = true) { + #update = (current: any, forceUpdate = false, runOnChange = true) => { let value = this.getValue(current) + if (!this.#firstUpdated) this.#initialValue = value + // NOTE: Forcing change registration for all cells if (this.value !== value || forceUpdate) { this.value = value @@ -175,7 +186,6 @@ export class TableCellBase extends LitElement { setText(value: any, setOnInput = true, runOnChange = true) { if (setOnInput) [ this.#editor, this.#renderer ].forEach(element => this.setChild(element, value)) // RESETS HISTORY - if (this.schema.type === 'array' || this.schema.type === 'object') this.#update(value, true, runOnChange) // Ensure array values are not coerced else this.#update(`${value}`, undefined, runOnChange) // Coerce to string } @@ -208,6 +218,7 @@ export class TableCellBase extends LitElement { // Initialize values firstUpdated() { + this.#firstUpdated = true const elements = [ this.#editor, this.#renderer ] elements.forEach(element => this.setChild(element)) } From f0acf967712fd7cc60a5bac23d25ba2c79b5cdda Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 15 Mar 2024 20:09:58 +0000 Subject: [PATCH 10/31] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pyflask/manageNeuroconv/manage_neuroconv.py | 12 +++++++----- src/renderer/src/stories/BasicTable.js | 2 +- src/renderer/src/stories/SimpleTable.js | 7 +++---- src/renderer/src/stories/Table.js | 11 +++++------ .../stories/pages/guided-mode/data/GuidedMetadata.js | 9 ++++----- src/renderer/src/stories/table/cells/base.ts | 4 ++-- 6 files changed, 22 insertions(+), 23 deletions(-) diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index 61801fd10..ec3d28f7c 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -371,7 +371,9 @@ def get_metadata_schema(source_data: Dict[str, dict], interfaces: dict) -> Dict[ # Populate Units metadata metadata["Ecephys"]["Units"] = {} schema["properties"]["Ecephys"]["required"].append("Units") - original_units_schema = ecephys_properties.pop("UnitProperties", None) # Remove UnitProperties from schema. NOTE: Not specific to interface + original_units_schema = ecephys_properties.pop( + "UnitProperties", None + ) # Remove UnitProperties from schema. NOTE: Not specific to interface has_units = original_units_schema is not None metadata["Ecephys"].pop("UnitProperties", None) # Always remove top-level UnitProperties from metadata @@ -397,8 +399,8 @@ def on_sorting_interface(name, sorting_interface): "maxItems": n_units, "items": { "allOf": [ - { "$ref": "#/properties/Ecephys/properties/definitions/Unit" }, - {"required": list(map(lambda info: info["name"], units_data["UnitColumns"]))} + {"$ref": "#/properties/Ecephys/properties/definitions/Unit"}, + {"required": list(map(lambda info: info["name"], units_data["UnitColumns"]))}, ] }, }, @@ -433,8 +435,8 @@ def on_recording_interface(name, recording_interface): "maxItems": n_electrodes, "items": { "allOf": [ - { "$ref": "#/properties/Ecephys/properties/definitions/Electrode" }, - {"required": list(map(lambda info: info["name"], electrode_data["ElectrodeColumns"]))} + {"$ref": "#/properties/Ecephys/properties/definitions/Electrode"}, + {"required": list(map(lambda info: info["name"], electrode_data["ElectrodeColumns"]))}, ] }, }, diff --git a/src/renderer/src/stories/BasicTable.js b/src/renderer/src/stories/BasicTable.js index 705d4a6ba..85e06b125 100644 --- a/src/renderer/src/stories/BasicTable.js +++ b/src/renderer/src/stories/BasicTable.js @@ -275,7 +275,7 @@ export class BasicTable extends LitElement { let { type, original, inferred } = this.#getType(value, propInfo); - const isUndefined = value === undefined || value === '' + const isUndefined = value === undefined || value === ""; // Check if required if (isUndefined && "required" in this.#itemSchema && this.#itemSchema.required.includes(col)) diff --git a/src/renderer/src/stories/SimpleTable.js b/src/renderer/src/stories/SimpleTable.js index a47d945ae..ab01b34c1 100644 --- a/src/renderer/src/stories/SimpleTable.js +++ b/src/renderer/src/stories/SimpleTable.js @@ -24,7 +24,6 @@ const isVisible = function (ele, container) { return top <= containerRect.top ? containerRect.top - top <= height : bottom - containerRect.bottom <= height; }; - export class SimpleTable extends LitElement { validateOnChange; onUpdate = () => {}; @@ -202,7 +201,7 @@ export class SimpleTable extends LitElement { maxHeight, contextOptions = {}, ignore = {}, - editable = {} + editable = {}, } = {}) { super(); this.schema = schema ?? {}; @@ -215,7 +214,7 @@ export class SimpleTable extends LitElement { this.maxHeight = maxHeight ?? ""; this.ignore = ignore; - this.editable = editable + this.editable = editable; this.contextOptions = contextOptions; @@ -783,7 +782,7 @@ export class SimpleTable extends LitElement { const schema = this.#itemProps[fullInfo.col]; const ignore = getIgnore(this.ignore, [fullInfo.col]); - const isEditable = getEditable(value, this.editable, fullInfo.col) + const isEditable = getEditable(value, this.editable, fullInfo.col); // Track the cell renderer const cell = new TableCell({ diff --git a/src/renderer/src/stories/Table.js b/src/renderer/src/stories/Table.js index 4a67723a3..cf39f23d1 100644 --- a/src/renderer/src/stories/Table.js +++ b/src/renderer/src/stories/Table.js @@ -17,11 +17,10 @@ const isRequired = (col, schema) => { }; export const getEditable = (value, config, colName) => { - if (typeof config === 'boolean') return config - if (typeof config === 'function') return config(value) - return getEditable(value, config?.[colName] ?? true) -} - + if (typeof config === "boolean") return config; + if (typeof config === "function") return config(value); + return getEditable(value, config?.[colName] ?? true); +}; export function sortTable(schema, keyColumn, order) { const cols = Object.keys(schema.properties) @@ -391,7 +390,7 @@ export class Table extends LitElement { return; } - const isUndefined = value == '' + const isUndefined = value == ""; if (isUndefined && required) { instanceThis.#handleValidationResult( diff --git a/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js b/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js index 2fc3e0ee7..24b93830d 100644 --- a/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js +++ b/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js @@ -29,14 +29,13 @@ const tableRenderConfig = { "*": (metadata) => new SimpleTable(metadata), Electrodes: true, ElectrodeColumns: function (metadata, fullPath) { - - const electrodeSchema = getSchema([...fullPath.slice(0, -1), 'Electrodes'], this.schema) + const electrodeSchema = getSchema([...fullPath.slice(0, -1), "Electrodes"], this.schema); return new SimpleTable({ ...metadata, editable: { - name: (value) => !electrodeSchema.items.required.includes(value) - } - }) + name: (value) => !electrodeSchema.items.required.includes(value), + }, + }); }, Units: (metadata) => { metadata.editable = false; diff --git a/src/renderer/src/stories/table/cells/base.ts b/src/renderer/src/stories/table/cells/base.ts index d815f6ed6..14098c568 100644 --- a/src/renderer/src/stories/table/cells/base.ts +++ b/src/renderer/src/stories/table/cells/base.ts @@ -121,7 +121,7 @@ export class TableCellBase extends LitElement { if (state) { if (!this.editable && this.#initialValue) return // Non-editability does not apply to new rows - + this.setAttribute('editing', '') const listenForEnter = (ev: KeyboardEvent) => { @@ -175,7 +175,7 @@ export class TableCellBase extends LitElement { #update = (current: any, forceUpdate = false, runOnChange = true) => { let value = this.getValue(current) - if (!this.#firstUpdated) this.#initialValue = value + if (!this.#firstUpdated) this.#initialValue = value // NOTE: Forcing change registration for all cells if (this.value !== value || forceUpdate) { From b4075e2f50cb38e705563167bac2da9f31c5b753 Mon Sep 17 00:00:00 2001 From: Garrett Michael Flynn Date: Mon, 18 Mar 2024 10:54:07 -0700 Subject: [PATCH 11/31] Show unit name and fix issues from review --- pyflask/manageNeuroconv/manage_neuroconv.py | 89 +++++++++++++++---- schemas/base-metadata.schema.ts | 2 +- src/renderer/src/stories/JSONSchemaForm.js | 15 +++- src/renderer/src/stories/JSONSchemaInput.js | 3 +- .../pages/guided-mode/data/GuidedMetadata.js | 27 +++--- .../guided-mode/data/GuidedSourceData.js | 4 - src/renderer/src/validation/validation.ts | 80 +++++++++-------- 7 files changed, 147 insertions(+), 73 deletions(-) diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index ec3d28f7c..cd6ab9689 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -20,14 +20,32 @@ EXCLUDED_RECORDING_INTERFACE_PROPERTIES = ["contact_vector", "contact_shapes", "group", "location"] -EXTRA_RECORDING_INTERFACE_PROPERTIES = EXTRA_SORTING_INTERFACE_PROPERTIES = { - "brain_area": { + +EXTRA_INTERFACE_PROPERTIES = { + "brain_area": { "data_type": "str", - "description": "The brain area where the electrode is located.", "default": "unknown", } } +EXTRA_RECORDING_INTERFACE_PROPERTIES = { + "brain_area": { + "description": "The brain area where the electrode is located.", + **EXTRA_INTERFACE_PROPERTIES["brain_area"] + } +} + +EXTRA_SORTING_INTERFACE_PROPERTIES = { + "name": { + "description": "The unique name for the unit", + "data_type": "string" + }, + "brain_area": { + "description": "The brain area where the unit is located.", + **EXTRA_INTERFACE_PROPERTIES["brain_area"] + } +} + EXCLUDED_SORTING_INTERFACE_PROPERTIES = ["location", "spike_times", "electrodes"] # Not validated DTYPE_DESCRIPTIONS = { @@ -668,9 +686,20 @@ def update_conversion_progress(**kwargs): # Quick fix to remove units has_units = "Units" in ecephys_metadata + if has_units: + for interface_name, interface_unit_results in ecephys_metadata["Units"].items(): + interface = converter.data_interface_objects[interface_name] + + update_sorting_properties_from_table_as_json( + interface, + unit_table_json=interface_unit_results["Units"], + unit_column_info=interface_unit_results["UnitColumns"], + ) + del ecephys_metadata["Units"] + for interface_name, interface_electrode_results in ecephys_metadata["Electrodes"].items(): interface = converter.data_interface_objects[interface_name] @@ -1056,11 +1085,11 @@ def map_dtype(dtype: str) -> str: return dtype -def get_property_dtype(recording_extractor, property_name: str, channel_ids: list): - if property_name in EXTRA_RECORDING_INTERFACE_PROPERTIES: - dtype = EXTRA_RECORDING_INTERFACE_PROPERTIES[property_name]["data_type"] +def get_property_dtype(extractor, property_name: str, ids: list, extra_props: dict): + if property_name in extra_props: + dtype = extra_props[property_name]["data_type"] else: - dtype = str(recording_extractor.get_property(key=property_name, ids=channel_ids).dtype) + dtype = str(extractor.get_property(key=property_name, ids=ids).dtype) # return type(recording.get_property(key=property_name)[0]).__name__.replace("_", "") # return dtype @@ -1106,14 +1135,9 @@ def get_unit_columns_json(interface) -> List[Dict[str, Any]]: """A convenience function for collecting and organizing the properties of the underlying sorting extractor.""" properties = get_sorting_interface_properties(interface) - # Hardcoded for Phy (NOTE: Update for more interfaces) property_descriptions = dict( - # spike_times="The times of the spikes, in seconds.", - # spike_clusters="The cluster IDs of the spikes.", - # spike_templates="The template IDs of the spikes.", - # spike_amplitudes="The amplitudes of the spikes.", - # spike_depths="The depths of the spikes.", - # spike_widths="The widths of the spikes.", + clu_id="The cluster ID for the unit", + group_id="The group ID for the unit" ) for property_name, property_info in EXTRA_SORTING_INTERFACE_PROPERTIES.items(): @@ -1129,7 +1153,7 @@ def get_unit_columns_json(interface) -> List[Dict[str, Any]]: name=property_name, description=property_descriptions.get(property_name, "No description."), data_type=get_property_dtype( - recording_extractor=sorting_extractor, property_name=property_name, channel_ids=[unit_ids[0]] + extractor=sorting_extractor, property_name=property_name, ids=[unit_ids[0]], extra_props=EXTRA_SORTING_INTERFACE_PROPERTIES ), ) for property_name in properties.keys() @@ -1153,10 +1177,14 @@ def get_unit_table_json(interface) -> List[Dict[str, Any]]: table = list() for unit_id in unit_ids: + unit_column = dict() + for property_name in properties: - if property_name in EXTRA_SORTING_INTERFACE_PROPERTIES: - sorting_property_value = properties[property_name]["default"] + if property_name is 'name': + sorting_property_value = unit_id + elif property_name in EXTRA_SORTING_INTERFACE_PROPERTIES: + sorting_property_value = properties[property_name].get('default') else: sorting_property_value = sorting.get_property(key=property_name, ids=[unit_id])[ 0 # First axis is always units in SI @@ -1199,7 +1227,7 @@ def get_electrode_columns_json(interface) -> List[Dict[str, Any]]: name=property_name, description=property_descriptions.get(property_name, "No description."), data_type=get_property_dtype( - recording_extractor=recording_extractor, property_name=property_name, channel_ids=[channel_ids[0]] + extractor=recording_extractor, property_name=property_name, ids=[channel_ids[0]], extra_props=EXTRA_RECORDING_INTERFACE_PROPERTIES ), ) for property_name in properties.keys() @@ -1313,3 +1341,28 @@ def update_recording_properties_from_table_as_json( # TODO: uncomment when neuroconv supports contact vectors (probe interface) # if "contact_vector" in property_names: # recording_extractor.set_property(key="contact_vector", values=modified_contact_vector) + + +def update_sorting_properties_from_table_as_json( + sorting_interface, unit_column_info: dict, unit_table_json: List[Dict[str, Any]] +): + import numpy as np + unit_column_data_types = {column["name"]: column["data_type"] for column in unit_column_info} + unit_column_descriptions = {column["name"]: column["description"] for column in unit_column_info} + sorting_extractor = sorting_interface.sorting_extractor + + for entry_index, entry in enumerate(unit_table_json): + unit_properties = dict(entry) # copy + + unit_id = unit_properties.pop("name", None) # NOTE: Is called unit_name in the actual units table + + for property_name, property_value in unit_properties.items(): + + if (property_name is "name"): + continue + + sorting_extractor.set_property( + key=property_name, + values=np.array([property_value], dtype=unit_column_data_types[property_name]), #, description=unit_column_descriptions[property_name]), + ids=[unit_id] + ) \ No newline at end of file diff --git a/schemas/base-metadata.schema.ts b/schemas/base-metadata.schema.ts index 7f6944f68..2176e298c 100644 --- a/schemas/base-metadata.schema.ts +++ b/schemas/base-metadata.schema.ts @@ -145,7 +145,7 @@ export const preprocessMetadataSchema = (schema: any = baseMetadataSchema, globa updateEcephysTable("Units", copy, { Units: { - order: ["clu_id", "group_id"] + order: ["name", "clu_id", "group_id"] }, UnitColumns: { order: COLUMN_SCHEMA_ORDER diff --git a/src/renderer/src/stories/JSONSchemaForm.js b/src/renderer/src/stories/JSONSchemaForm.js index 568a03c9b..3c6d51404 100644 --- a/src/renderer/src/stories/JSONSchemaForm.js +++ b/src/renderer/src/stories/JSONSchemaForm.js @@ -80,9 +80,15 @@ export const getIgnore = (o, path) => { return path.reduce((acc, key) => { const info = acc[key] ?? {}; + const accWildcard = acc["*"] ?? {} + const infoWildcard = info["*"] ?? {} + const mergedWildcards = { ...accWildcard, ...infoWildcard } + + if (key in mergedWildcards) return {...info, ...mergedWildcards[key]} + return { ...info, - "*": { ...(acc["*"] ?? {}), ...(info["*"] ?? {}) }, // Accumulate ignore values + "*": mergedWildcards, // Accumulate ignore values }; }, o); }; @@ -768,12 +774,15 @@ export class JSONSchemaForm extends LitElement { else return [key, value]; }; + const res = entries .map(([key, value]) => { if (!value.properties && key === "definitions") return false; // Skip definitions - if (this.ignore["*"]?.[key]) - return false; // Skip all properties with this name + + // If conclusively ignored + if (this.ignore["*"]?.[key] === true) return false; // Skip all properties with this name else if (this.ignore[key] === true) return false; // Skip this property + if (this.showLevelOverride >= path.length) return isRenderable(key, value); if (required[key]) return isRenderable(key, value); if (this.#getLink([...this.base, ...path, key])) return isRenderable(key, value); diff --git a/src/renderer/src/stories/JSONSchemaInput.js b/src/renderer/src/stories/JSONSchemaInput.js index a2b388697..4d84193a0 100644 --- a/src/renderer/src/stories/JSONSchemaInput.js +++ b/src/renderer/src/stories/JSONSchemaInput.js @@ -174,6 +174,8 @@ export function createTable(fullPath, { onUpdate, onThrow, overrides = {} }) { merge(overrides.schema, schemaCopy, { arrays: true }); + console.log(schemaPath, nestedIgnore) + const tableMetadata = { keyColumn: tempPropertyKey, schema: schemaCopy, @@ -254,7 +256,6 @@ export function createTable(fullPath, { onUpdate, onThrow, overrides = {} }) { } const nestedIgnore = getIgnore(ignore, fullPath); - Object.assign(nestedIgnore, overrides.ignore ?? {}); merge(overrides.ignore, nestedIgnore); diff --git a/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js b/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js index 24b93830d..9a73184a8 100644 --- a/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js +++ b/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js @@ -41,9 +41,14 @@ const tableRenderConfig = { metadata.editable = false; return true; }, - UnitColumns: (metadata) => { - metadata.editable = false; - return true; + UnitColumns: function (metadata, fullPath) { + const unitSchema = getSchema([...fullPath.slice(0, -1), "Units"], this.schema); + return new SimpleTable({ + ...metadata, + editable: { + name: (value) => !unitSchema.items.required.includes(value), + }, + }); }, }; @@ -80,13 +85,15 @@ const propsToIgnore = { ElectricalSeries: true, ElectricalSeriesLF: true, ElectricalSeriesAP: true, - Electrodes: { - "*": { - location: true, - group: true, - contact_vector: true, - }, - }, + Units: { + '*': { + UnitColumns: { + '*': { + data_type: true // Do not show data_type + } + } + } + } }, Icephys: true, // Always ignore icephys metadata (for now) Behavior: true, // Always ignore behavior metadata (for now) diff --git a/src/renderer/src/stories/pages/guided-mode/data/GuidedSourceData.js b/src/renderer/src/stories/pages/guided-mode/data/GuidedSourceData.js index 6bd47bc30..2d67ff619 100644 --- a/src/renderer/src/stories/pages/guided-mode/data/GuidedSourceData.js +++ b/src/renderer/src/stories/pages/guided-mode/data/GuidedSourceData.js @@ -222,10 +222,6 @@ export class GuidedSourceDataPage extends ManagedPage { updated() { const dashboard = document.querySelector("nwb-dashboard"); const page = dashboard.page; - setTimeout(() => { - console.log(page.forms[0].form.accordions["SpikeGLX Recording"]); - }); - console.log(page.forms[0].form.accordions); } render() { diff --git a/src/renderer/src/validation/validation.ts b/src/renderer/src/validation/validation.ts index 3a0eaae33..2c55435d9 100644 --- a/src/renderer/src/validation/validation.ts +++ b/src/renderer/src/validation/validation.ts @@ -229,23 +229,20 @@ schema.Ecephys.ElectrodeGroup = { // Label columns as invalid if not registered on the ElectrodeColumns table // NOTE: If not present in the schema, these are not being rendered... -schema.Ecephys.Electrodes = { - - // All interfaces - ["*"]: { - - Electrodes: { +const generateLinkedTableInteractions = (tableName, colTableName, additionalColumnValidation = {}) => { + return { + [ tableName ]: { // All other column ['*']: function (this: JSONSchemaForm, name, _, path) { const commonPath = path.slice(0, -2) - const colPath = [...commonPath, 'ElectrodeColumns'] + const colPath = [...commonPath, colTableName] - const { value: electrodeColumns } = get(this.results, colPath) // NOTE: this.results is out of sync with the actual row contents at the moment of validation + const { value } = get(this.results, colPath) // NOTE: this.results is out of sync with the actual row contents at the moment of validation - if (electrodeColumns && !electrodeColumns.find((row: any) => row.name === name)) { + if (value && !value.find((row: any) => row.name === name)) { return [ { message: 'Not a valid column', @@ -255,46 +252,32 @@ schema.Ecephys.Electrodes = { } }, - // Group name column - group_name: function (this: JSONSchemaForm, _, __, ___, value) { - - const groups = this.results.Ecephys.ElectrodeGroup.map(({ name }) => name) // Groups are validated across all interfaces - - if (groups.includes(value)) return true - else { - return [ - { - message: 'Not a valid group name', - type: 'error' - } - ] - } - } + ...additionalColumnValidation }, - // Update the columns available on the Electrodes table when there is a new name in the ElectrodeColumns table - ElectrodeColumns: { + // Update the columns available on the table when there is a new name in the columns table + [ colTableName ]: { ['*']: { '*': function (this: JSONSchemaForm, propName, __, path, value) { const commonPath = path.slice(0, -2) - const electrodesTablePath = [ ...commonPath, 'Electrodes'] - const electrodesTable = this.getFormElement(electrodesTablePath) - const electrodesSchema = electrodesTable.schema // Manipulate the schema that is on the table - const globalElectrodeSchema = getSchema(electrodesTablePath, this.schema) + const tablePath = [ ...commonPath, tableName] + const table = this.getFormElement(tablePath) + const tableSchema = table.schema // Manipulate the schema that is on the table + const globalSchema = getSchema(tablePath, this.schema) const { value: row } = get(this.results, path) const currentName = row?.['name'] - const hasNameUpdate = propName == 'name' && !(value in electrodesSchema.items.properties) + const hasNameUpdate = propName == 'name' && !(value in tableSchema.items.properties) const resolvedName = hasNameUpdate ? value : currentName if (value === currentName) return true // No change if (!resolvedName) return true // Only set when name is actually present - const schemaToEdit = [electrodesSchema, globalElectrodeSchema] + const schemaToEdit = [tableSchema, globalSchema] schemaToEdit.forEach(schema => { const properties = schema.items.properties @@ -311,21 +294,46 @@ schema.Ecephys.Electrodes = { // Swap the new and current name information if (hasNameUpdate) { - const electrodesTable = this.getFormElement([ ...commonPath, 'Electrodes']) - electrodesTable.data.forEach(row => { + const table = this.getFormElement([ ...commonPath, tableName]) + table.data.forEach(row => { if (!(value in row)) row[value] = row[currentName] // Initialize new column with old values delete row[currentName] // Delete old column }) } - // Always re-render the Electrodes table on column changes - electrodesTable.requestUpdate() + // Always re-render the table on column changes + table.requestUpdate() } }, } } } +schema.Ecephys.Units = { + ["*"]: generateLinkedTableInteractions('Units', 'UnitColumns', {}) +} + +schema.Ecephys.Electrodes = { + + // All interfaces + ["*"]: generateLinkedTableInteractions('Electrodes', 'ElectrodeColumns', { + group_name: function (this: JSONSchemaForm, _, __, ___, value) { + + const groups = this.results.Ecephys.ElectrodeGroup.map(({ name }) => name) // Groups are validated across all interfaces + + if (groups.includes(value)) return true + else { + return [ + { + message: 'Not a valid group name', + type: 'error' + } + ] + } + } + }) +} + // ----------------- Ophys Validation ----------------- // schema.Ophys.ImagingPlane = { From 06c3f18a15b8f8a4d43782b3f9940e425611dcac Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 18 Mar 2024 17:54:29 +0000 Subject: [PATCH 12/31] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pyflask/manageNeuroconv/manage_neuroconv.py | 50 ++++++++++--------- src/renderer/src/stories/JSONSchemaForm.js | 16 +++--- src/renderer/src/stories/JSONSchemaInput.js | 2 +- .../pages/guided-mode/data/GuidedMetadata.js | 14 +++--- 4 files changed, 42 insertions(+), 40 deletions(-) diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index cd4bb50fa..038491a78 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -22,7 +22,7 @@ EXCLUDED_RECORDING_INTERFACE_PROPERTIES = ["contact_vector", "contact_shapes", "group", "location"] EXTRA_INTERFACE_PROPERTIES = { - "brain_area": { + "brain_area": { "data_type": "str", "default": "unknown", } @@ -31,19 +31,16 @@ EXTRA_RECORDING_INTERFACE_PROPERTIES = { "brain_area": { "description": "The brain area where the electrode is located.", - **EXTRA_INTERFACE_PROPERTIES["brain_area"] + **EXTRA_INTERFACE_PROPERTIES["brain_area"], } } EXTRA_SORTING_INTERFACE_PROPERTIES = { - "name": { - "description": "The unique name for the unit", - "data_type": "string" - }, - "brain_area": { + "name": {"description": "The unique name for the unit", "data_type": "string"}, + "brain_area": { "description": "The brain area where the unit is located.", - **EXTRA_INTERFACE_PROPERTIES["brain_area"] - } + **EXTRA_INTERFACE_PROPERTIES["brain_area"], + }, } EXCLUDED_SORTING_INTERFACE_PROPERTIES = ["location", "spike_times", "electrodes"] # Not validated @@ -701,7 +698,6 @@ def update_conversion_progress(**kwargs): del ecephys_metadata["Units"] - for interface_name, interface_electrode_results in ecephys_metadata["Electrodes"].items(): interface = converter.data_interface_objects[interface_name] @@ -1137,10 +1133,7 @@ def get_unit_columns_json(interface) -> List[Dict[str, Any]]: """A convenience function for collecting and organizing the properties of the underlying sorting extractor.""" properties = get_sorting_interface_properties(interface) - property_descriptions = dict( - clu_id="The cluster ID for the unit", - group_id="The group ID for the unit" - ) + property_descriptions = dict(clu_id="The cluster ID for the unit", group_id="The group ID for the unit") for property_name, property_info in EXTRA_SORTING_INTERFACE_PROPERTIES.items(): description = property_info.get("description", None) @@ -1155,7 +1148,10 @@ def get_unit_columns_json(interface) -> List[Dict[str, Any]]: name=property_name, description=property_descriptions.get(property_name, "No description."), data_type=get_property_dtype( - extractor=sorting_extractor, property_name=property_name, ids=[unit_ids[0]], extra_props=EXTRA_SORTING_INTERFACE_PROPERTIES + extractor=sorting_extractor, + property_name=property_name, + ids=[unit_ids[0]], + extra_props=EXTRA_SORTING_INTERFACE_PROPERTIES, ), ) for property_name in properties.keys() @@ -1183,10 +1179,10 @@ def get_unit_table_json(interface) -> List[Dict[str, Any]]: unit_column = dict() for property_name in properties: - if property_name is 'name': + if property_name is "name": sorting_property_value = unit_id elif property_name in EXTRA_SORTING_INTERFACE_PROPERTIES: - sorting_property_value = properties[property_name].get('default') + sorting_property_value = properties[property_name].get("default") else: sorting_property_value = sorting.get_property(key=property_name, ids=[unit_id])[ 0 # First axis is always units in SI @@ -1229,7 +1225,10 @@ def get_electrode_columns_json(interface) -> List[Dict[str, Any]]: name=property_name, description=property_descriptions.get(property_name, "No description."), data_type=get_property_dtype( - extractor=recording_extractor, property_name=property_name, ids=[channel_ids[0]], extra_props=EXTRA_RECORDING_INTERFACE_PROPERTIES + extractor=recording_extractor, + property_name=property_name, + ids=[channel_ids[0]], + extra_props=EXTRA_RECORDING_INTERFACE_PROPERTIES, ), ) for property_name in properties.keys() @@ -1349,22 +1348,25 @@ def update_sorting_properties_from_table_as_json( sorting_interface, unit_column_info: dict, unit_table_json: List[Dict[str, Any]] ): import numpy as np + unit_column_data_types = {column["name"]: column["data_type"] for column in unit_column_info} unit_column_descriptions = {column["name"]: column["description"] for column in unit_column_info} sorting_extractor = sorting_interface.sorting_extractor - for entry_index, entry in enumerate(unit_table_json): + for entry_index, entry in enumerate(unit_table_json): unit_properties = dict(entry) # copy - unit_id = unit_properties.pop("name", None) # NOTE: Is called unit_name in the actual units table + unit_id = unit_properties.pop("name", None) # NOTE: Is called unit_name in the actual units table for property_name, property_value in unit_properties.items(): - if (property_name is "name"): + if property_name is "name": continue sorting_extractor.set_property( key=property_name, - values=np.array([property_value], dtype=unit_column_data_types[property_name]), #, description=unit_column_descriptions[property_name]), - ids=[unit_id] - ) \ No newline at end of file + values=np.array( + [property_value], dtype=unit_column_data_types[property_name] + ), # , description=unit_column_descriptions[property_name]), + ids=[unit_id], + ) diff --git a/src/renderer/src/stories/JSONSchemaForm.js b/src/renderer/src/stories/JSONSchemaForm.js index 3c6d51404..a58e5a79f 100644 --- a/src/renderer/src/stories/JSONSchemaForm.js +++ b/src/renderer/src/stories/JSONSchemaForm.js @@ -80,11 +80,11 @@ export const getIgnore = (o, path) => { return path.reduce((acc, key) => { const info = acc[key] ?? {}; - const accWildcard = acc["*"] ?? {} - const infoWildcard = info["*"] ?? {} - const mergedWildcards = { ...accWildcard, ...infoWildcard } - - if (key in mergedWildcards) return {...info, ...mergedWildcards[key]} + const accWildcard = acc["*"] ?? {}; + const infoWildcard = info["*"] ?? {}; + const mergedWildcards = { ...accWildcard, ...infoWildcard }; + + if (key in mergedWildcards) return { ...info, ...mergedWildcards[key] }; return { ...info, @@ -774,15 +774,15 @@ export class JSONSchemaForm extends LitElement { else return [key, value]; }; - const res = entries .map(([key, value]) => { if (!value.properties && key === "definitions") return false; // Skip definitions // If conclusively ignored - if (this.ignore["*"]?.[key] === true) return false; // Skip all properties with this name + if (this.ignore["*"]?.[key] === true) + return false; // Skip all properties with this name else if (this.ignore[key] === true) return false; // Skip this property - + if (this.showLevelOverride >= path.length) return isRenderable(key, value); if (required[key]) return isRenderable(key, value); if (this.#getLink([...this.base, ...path, key])) return isRenderable(key, value); diff --git a/src/renderer/src/stories/JSONSchemaInput.js b/src/renderer/src/stories/JSONSchemaInput.js index 4d84193a0..252f2f79d 100644 --- a/src/renderer/src/stories/JSONSchemaInput.js +++ b/src/renderer/src/stories/JSONSchemaInput.js @@ -174,7 +174,7 @@ export function createTable(fullPath, { onUpdate, onThrow, overrides = {} }) { merge(overrides.schema, schemaCopy, { arrays: true }); - console.log(schemaPath, nestedIgnore) + console.log(schemaPath, nestedIgnore); const tableMetadata = { keyColumn: tempPropertyKey, diff --git a/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js b/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js index 9a73184a8..d8ad718de 100644 --- a/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js +++ b/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js @@ -86,14 +86,14 @@ const propsToIgnore = { ElectricalSeriesLF: true, ElectricalSeriesAP: true, Units: { - '*': { + "*": { UnitColumns: { - '*': { - data_type: true // Do not show data_type - } - } - } - } + "*": { + data_type: true, // Do not show data_type + }, + }, + }, + }, }, Icephys: true, // Always ignore icephys metadata (for now) Behavior: true, // Always ignore behavior metadata (for now) From 93c9d57cdf910c72785499d26ff00afa4873ffbe Mon Sep 17 00:00:00 2001 From: Garrett Michael Flynn Date: Thu, 21 Mar 2024 14:41:26 -0700 Subject: [PATCH 13/31] Update ElectrodeColumns and UnitColumns to be global Ecephys properties --- pyflask/manageNeuroconv/manage_neuroconv.py | 146 +++++++++-------- schemas/base-metadata.schema.ts | 52 +++--- src/renderer/src/stories/BasicTable.js | 12 +- src/renderer/src/stories/JSONSchemaForm.js | 1 + .../pages/guided-mode/data/GuidedMetadata.js | 46 ++++-- src/renderer/src/validation/validation.ts | 148 +++++++++--------- 6 files changed, 222 insertions(+), 183 deletions(-) diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index 038491a78..82189799a 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -36,7 +36,7 @@ } EXTRA_SORTING_INTERFACE_PROPERTIES = { - "name": {"description": "The unique name for the unit", "data_type": "string"}, + "name": {"description": "The unique name for the unit", "data_type": "str"}, "brain_area": { "description": "The brain area where the unit is located.", **EXTRA_INTERFACE_PROPERTIES["brain_area"], @@ -45,6 +45,7 @@ EXCLUDED_SORTING_INTERFACE_PROPERTIES = ["location", "spike_times", "electrodes"] # Not validated +# NOTE: These are the only accepted dtypes... DTYPE_DESCRIPTIONS = { "bool": "logical", "str": "string", @@ -61,7 +62,7 @@ DTYPE_SCHEMA = { "type": "string", - "strict": False, + # "strict": False, "enum": list(DTYPE_DESCRIPTIONS.keys()), "enumLabels": DTYPE_DESCRIPTIONS, } @@ -383,92 +384,110 @@ def get_metadata_schema(source_data: Dict[str, dict], interfaces: dict) -> Dict[ # Add Electrodes to the schema metadata["Ecephys"]["Electrodes"] = {} ecephys_schema["required"].append("Electrodes") + + ecephys_properties["ElectrodeColumns"] = { + "type": "array", + "minItems": 0, + "items": {"$ref": "#/properties/Ecephys/properties/definitions/ElectrodeColumn"}, + } + + ecephys_schema["required"].append("ElectrodeColumns") + + ecephys_properties["Electrodes"] = {"type": "object", "properties": {}, "required": []} # Populate Units metadata metadata["Ecephys"]["Units"] = {} schema["properties"]["Ecephys"]["required"].append("Units") - original_units_schema = ecephys_properties.pop( - "UnitProperties", None - ) # Remove UnitProperties from schema. NOTE: Not specific to interface - has_units = original_units_schema is not None - + original_units_schema = ecephys_properties.pop("UnitProperties", None) metadata["Ecephys"].pop("UnitProperties", None) # Always remove top-level UnitProperties from metadata + has_units = original_units_schema is not None + if has_units: + metadata["Ecephys"] + ecephys_properties["UnitColumns"] = { + "type": "array", + "minItems": 0, + "items": {"$ref": "#/properties/Ecephys/properties/definitions/UnitColumn"}, + } + + schema["properties"]["Ecephys"]["required"].append("UnitColumns") ecephys_properties["Units"] = {"type": "object", "properties": {}, "required": []} def on_sorting_interface(name, sorting_interface): - units_data = metadata["Ecephys"]["Units"][name] = dict( - Units=get_unit_table_json(sorting_interface), - UnitColumns=get_unit_columns_json(sorting_interface), - ) + unit_columns = get_unit_columns_json(sorting_interface) - n_units = len(units_data["Units"]) - - ecephys_properties["Units"]["properties"][name] = dict( - type="object", - properties=dict( - Units={ - "type": "array", - "minItems": n_units, - "maxItems": n_units, - "items": { - "allOf": [ - {"$ref": "#/properties/Ecephys/properties/definitions/Unit"}, - {"required": list(map(lambda info: info["name"], units_data["UnitColumns"]))}, - ] - }, - }, - UnitColumns={ - "type": "array", - "minItems": 0, - "items": {"$ref": "#/properties/Ecephys/properties/definitions/UnitColumn"}, - }, - ), - required=["Units", "UnitColumns"], - ) + # Aggregate electrode column information across recording interfaces + existing_unit_columns = metadata["Ecephys"].get('UnitColumns') + if existing_unit_columns: + for entry in unit_columns: + if any(obj["name"] == entry["name"] for obj in existing_unit_columns): + continue + else: + existing_unit_columns.append(entry) + else: + metadata["Ecephys"]["UnitColumns"] = unit_columns + + units_data = metadata["Ecephys"]["Units"][name] = get_unit_table_json(sorting_interface) + + n_units = len(units_data) + + ecephys_properties["Units"]["properties"][name] = { + "type": "array", + "minItems": n_units, + "maxItems": n_units, + "items": { + "allOf": [ + {"$ref": "#/properties/Ecephys/properties/definitions/Unit"}, + {"required": list(map(lambda info: info["name"], unit_columns))}, + ] + }, + } ecephys_properties["Units"]["required"].append(name) return sorting_interface + aggregate_electrode_info = {} + def on_recording_interface(name, recording_interface): + global aggregate_electrode_columns + + electrode_columns = get_electrode_columns_json(recording_interface) + + # Aggregate electrode column information across recording interfaces + existing_electrode_columns = metadata["Ecephys"].get('ElectrodeColumns') + if existing_electrode_columns: + for entry in electrode_columns: + if any(obj["name"] == entry["name"] for obj in existing_electrode_columns): + continue + else: + existing_electrode_columns.append(entry) + else: + metadata["Ecephys"]["ElectrodeColumns"] = electrode_columns - electrode_data = metadata["Ecephys"]["Electrodes"][name] = dict( - Electrodes=get_electrode_table_json(recording_interface), - ElectrodeColumns=get_electrode_columns_json(recording_interface), - ) + electrode_data = metadata["Ecephys"]["Electrodes"][name] = get_electrode_table_json(recording_interface) - n_electrodes = len(electrode_data["Electrodes"]) - - ecephys_properties["Electrodes"]["properties"][name] = dict( - type="object", - properties=dict( - Electrodes={ - "type": "array", - "minItems": n_electrodes, - "maxItems": n_electrodes, - "items": { - "allOf": [ - {"$ref": "#/properties/Ecephys/properties/definitions/Electrode"}, - {"required": list(map(lambda info: info["name"], electrode_data["ElectrodeColumns"]))}, - ] - }, - }, - ElectrodeColumns={ - "type": "array", - "minItems": 0, - "items": {"$ref": "#/properties/Ecephys/properties/definitions/ElectrodeColumn"}, - }, - ), - required=["Electrodes", "ElectrodeColumns"], - ) + n_electrodes = len(electrode_data) + + ecephys_properties["Electrodes"]["properties"][name] = { + "type": "array", + "minItems": n_electrodes, + "maxItems": n_electrodes, + "items": { + "allOf": [ + {"$ref": "#/properties/Ecephys/properties/definitions/Electrode"}, + {"required": list(map(lambda info: info["name"], electrode_columns))}, + ] + } + } ecephys_properties["Electrodes"]["required"].append(name) return recording_interface + from neuroconv.datainterfaces.ecephys.baserecordingextractorinterface import BaseRecordingExtractorInterface from neuroconv.datainterfaces.ecephys.basesortingextractorinterface import BaseSortingExtractorInterface @@ -491,6 +510,7 @@ def on_recording_interface(name, recording_interface): # Configure electrode columns defs["ElectrodeColumn"] = electrode_def + defs["ElectrodeColumn"]["required"] = list(electrode_def["properties"].keys()) new_electrodes_properties = { properties["name"]: {key: value for key, value in properties.items() if key != "name"} diff --git a/schemas/base-metadata.schema.ts b/schemas/base-metadata.schema.ts index 2176e298c..6ab1bc836 100644 --- a/schemas/base-metadata.schema.ts +++ b/schemas/base-metadata.schema.ts @@ -47,29 +47,25 @@ function updateEcephysTable(propName, schema, schemaToMerge) { const electrodesProp = ecephys.properties[propName] if (!electrodesProp) return false for (let name in electrodesProp.properties) { - const interfaceProps = electrodesProp.properties[name].properties - for (let subProp in schemaToMerge) { - if (interfaceProps[subProp]) { - const itemSchema = interfaceProps[subProp].items - - // Do not add new items - const updateCopy = structuredClone(schemaToMerge[subProp]) - const updateProps = updateCopy.properties - for (let itemProp in updateProps) { - if (!itemSchema.properties[itemProp]) delete updateProps[itemProp] - } - - // Merge into existing items - merge(updateCopy, itemSchema) - } + const itemSchema = electrodesProp.properties[name].items + // Do not add new items + const updateCopy = structuredClone(schemaToMerge) + const updateProps = updateCopy.properties + for (let itemProp in updateProps) { + if (!itemSchema.properties[itemProp]) delete updateProps[itemProp] } + + // Merge into existing items + merge(updateCopy, itemSchema) } + return true } + export const preprocessMetadataSchema = (schema: any = baseMetadataSchema, global = false) => { @@ -130,26 +126,20 @@ export const preprocessMetadataSchema = (schema: any = baseMetadataSchema, globa if (ecephys) { + if (ecephys.properties.ElectrodeColumns) ecephys.properties.ElectrodeColumns.order = COLUMN_SCHEMA_ORDER + if (ecephys.properties.UnitProperties) ecephys.properties.UnitProperties.order = COLUMN_SCHEMA_ORDER + + updateEcephysTable("Electrodes", copy, { - Electrodes: { - properties: UV_PROPERTIES.reduce((acc, prop) => { - acc[prop] = { title: prop.replace('uV', UV_MATH_FORMAT) } - return acc - }, {}), - order: ["channel_name", "group_name", "shank_electrode_number", ...UV_PROPERTIES] - }, - ElectrodeColumns: { - order: COLUMN_SCHEMA_ORDER - } + properties: UV_PROPERTIES.reduce((acc, prop) => { + acc[prop] = { title: prop.replace('uV', UV_MATH_FORMAT) } + return acc + }, {}), + order: ["channel_name", "group_name", "shank_electrode_number", ...UV_PROPERTIES] }) updateEcephysTable("Units", copy, { - Units: { - order: ["name", "clu_id", "group_id"] - }, - UnitColumns: { - order: COLUMN_SCHEMA_ORDER - } + order: ["name", "clu_id", "group_id"] }) } diff --git a/src/renderer/src/stories/BasicTable.js b/src/renderer/src/stories/BasicTable.js index 05587a82e..d2de3440a 100644 --- a/src/renderer/src/stories/BasicTable.js +++ b/src/renderer/src/stories/BasicTable.js @@ -137,7 +137,7 @@ export class BasicTable extends LitElement { this.schema = schema ?? {}; this.data = data ?? []; this.keyColumn = keyColumn; - this.maxHeight = maxHeight ?? ""; + this.maxHeight = maxHeight ?? "unset"; this.validateEmptyCells = validateEmptyCells ?? true; this.ignore = ignore ?? {}; @@ -382,6 +382,7 @@ export class BasicTable extends LitElement { }; #readTSV(text) { + console.log(text, text.split("\n")) let data = text.split("\n").map((row) => row.split("\t").map((v) => { try { @@ -391,7 +392,7 @@ export class BasicTable extends LitElement { } }) ); // Map to actual values using JSON.parse - + console.log(data) let header = data.shift(); const structuredData = data.map((row) => @@ -402,18 +403,21 @@ export class BasicTable extends LitElement { ); Object.keys(this.data).forEach((row) => delete this.data[row]); // Delete all previous rows + Object.keys(data).forEach((row) => { const cols = structuredData[row]; const latest = (this.data[this.keyColumn ? cols[this.keyColumn] : row] = {}); Object.entries(cols).forEach(([key, value]) => { - if (key in this.#itemProps) { + // if (key in this.#itemProps) { const { type } = this.#getType(value, this.#itemProps[key]); if (type === "string") value = `${value}`; // Convert to string if necessary latest[key] = value; - } + // } }); // Only include data from schema }); + console.log(header, data, structuredData, this.data, this.#itemProps) + if (this.onUpdate) this.onUpdate([], data); // Update the whole table } diff --git a/src/renderer/src/stories/JSONSchemaForm.js b/src/renderer/src/stories/JSONSchemaForm.js index 16f4dbf9d..8a517f5d9 100644 --- a/src/renderer/src/stories/JSONSchemaForm.js +++ b/src/renderer/src/stories/JSONSchemaForm.js @@ -569,6 +569,7 @@ export class JSONSchemaForm extends LitElement { const allErrors = Array.from(flaggedInputs) .map((inputElement) => { + if (!inputElement.nextElementSibling) return // Skip tables return Array.from(inputElement.nextElementSibling.children).map((li) => li.message); }) .flat(); diff --git a/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js b/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js index d8ad718de..2f3c4be32 100644 --- a/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js +++ b/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js @@ -25,28 +25,48 @@ import { Button } from "../../../Button.js"; import globalIcon from "../../../assets/global.svg?raw"; +const parentTableRenderConfig = { + Electrodes: true, + Units: (metadata) => { + metadata.editable = false; + return true; + } +} + +function getAggregateRequirements(path) { + const electrodeSchema = getSchema(path, this.schema) + return Object.values(electrodeSchema.properties).reduce((set, schema) => { + schema.items.required.forEach(item => set.add(item)) + return set + }, new Set()) +} + const tableRenderConfig = { "*": (metadata) => new SimpleTable(metadata), - Electrodes: true, - ElectrodeColumns: function (metadata, fullPath) { - const electrodeSchema = getSchema([...fullPath.slice(0, -1), "Electrodes"], this.schema); + ElectrodeColumns: function (metadata) { + const aggregateRequirements = getAggregateRequirements.call(this, ["Ecephys", 'Electrodes']) + return new SimpleTable({ ...metadata, editable: { - name: (value) => !electrodeSchema.items.required.includes(value), + name: (value) => !aggregateRequirements.has(value), }, }); }, - Units: (metadata) => { - metadata.editable = false; - return true; - }, - UnitColumns: function (metadata, fullPath) { - const unitSchema = getSchema([...fullPath.slice(0, -1), "Units"], this.schema); + UnitColumns: function (metadata) { + + const aggregateRequirements = getAggregateRequirements.call(this, ["Ecephys", 'Units']) + return new SimpleTable({ ...metadata, + contextOptions: { + row: { + add: false, + remove: false + } + }, editable: { - name: (value) => !unitSchema.items.required.includes(value), + name: (value) => !aggregateRequirements.has(value), }, }); }, @@ -437,7 +457,9 @@ export class GuidedMetadataPage extends ManagedPage { const updatedSchema = structuredClone(metadata.schema); metadata.schema = updatedSchema; - const tableConfig = tableRenderConfig[name] ?? tableRenderConfig["*"] ?? true; + const parentName = fullPath[fullPath.length - 1] + + const tableConfig = tableRenderConfig[name] ?? parentTableRenderConfig[parentName] ?? tableRenderConfig["*"] ?? true; if (typeof tableConfig === "function") return tableConfig.call(form, metadata, [...fullPath, name]); else return tableConfig; }, diff --git a/src/renderer/src/validation/validation.ts b/src/renderer/src/validation/validation.ts index 722c5f5a8..aefef71ae 100644 --- a/src/renderer/src/validation/validation.ts +++ b/src/renderer/src/validation/validation.ts @@ -228,110 +228,112 @@ schema.Ecephys.ElectrodeGroup = { // Label columns as invalid if not registered on the ElectrodeColumns table // NOTE: If not present in the schema, these are not being rendered... -const generateLinkedTableInteractions = (tableName, colTableName, additionalColumnValidation = {}) => { - return { - [ tableName ]: { +const generateLinkedTableInteractions = (tablesPath, colTablePath, additionalColumnValidation = {}) => { - // All other column - ['*']: function (this: JSONSchemaForm, name, _, path) { + const tableConfiguration = { - const commonPath = path.slice(0, -2) + ['*']: function (this: JSONSchemaForm, name, _, path) { - const colPath = [...commonPath, colTableName] + const { value } = get(this.results, colTablePath) // NOTE: this.results is out of sync with the actual row contents at the moment of validation - const { value } = get(this.results, colPath) // NOTE: this.results is out of sync with the actual row contents at the moment of validation + if (value && !value.find((row: any) => row.name === name)) { + return [ + { + message: 'Not a valid column', + type: 'error' + } + ] + } + }, - if (value && !value.find((row: any) => row.name === name)) { - return [ - { - message: 'Not a valid column', - type: 'error' - } - ] - } - }, + ...additionalColumnValidation + } - ...additionalColumnValidation - }, - // Update the columns available on the table when there is a new name in the columns table - [ colTableName ]: { - ['*']: { - '*': function (this: JSONSchemaForm, propName, __, path, value) { + // Update the columns available on the table when there is a new name in the columns table + const columnTableConfig = { + '*': function (this: JSONSchemaForm, propName, __, path, value) { - const commonPath = path.slice(0, -2) - const tablePath = [ ...commonPath, tableName] - const table = this.getFormElement(tablePath) - const tableSchema = table.schema // Manipulate the schema that is on the table - const globalSchema = getSchema(tablePath, this.schema) + const form = this.getFormElement(tablesPath) + Object.entries(form.tables).forEach(([tableName, table]: [ string, any ]) => { - const { value: row } = get(this.results, path) + const fullPath = [...tablesPath, tableName] - const currentName = row?.['name'] + const tableSchema = table.schema // Manipulate the schema that is on the table + const globalSchema = getSchema(fullPath, this.schema) - const hasNameUpdate = propName == 'name' && !(value in tableSchema.items.properties) + const { value: row } = get(this.results, path) - const resolvedName = hasNameUpdate ? value : currentName + const currentName = row?.['name'] - if (value === currentName) return true // No change - if (!resolvedName) return true // Only set when name is actually present + const hasNameUpdate = propName == 'name' && !(value in tableSchema.items.properties) - const schemaToEdit = [tableSchema, globalSchema] - schemaToEdit.forEach(schema => { + const resolvedName = hasNameUpdate ? value : currentName - const properties = schema.items.properties - const oldRef = properties[currentName] + if (value === currentName) return true // No change + if (!resolvedName) return true // Only set when name is actually present - if (row) delete properties[currentName] // Delete previous name from schema + const schemaToEdit = [tableSchema, globalSchema] + schemaToEdit.forEach(schema => { - properties[resolvedName] = { - ...oldRef ?? {}, - description: propName === 'description' ? value : row?.description, - data_type: propName === 'data_type' ? value : row?.data_type, - } - }) + const properties = schema.items.properties + const oldRef = properties[currentName] - // Swap the new and current name information - if (hasNameUpdate) { - const table = this.getFormElement([ ...commonPath, tableName]) - table.data.forEach(row => { - if (!(value in row)) row[value] = row[currentName] // Initialize new column with old values - delete row[currentName] // Delete old column - }) - } + if (row) delete properties[currentName] // Delete previous name from schema - // Always re-render the table on column changes - table.requestUpdate() + properties[resolvedName] = { + ...oldRef ?? {}, + description: propName === 'description' ? value : row?.description, + data_type: propName === 'data_type' ? value : row?.data_type, + } + }) + + // Swap the new and current name information + if (hasNameUpdate) { + const table = this.getFormElement(fullPath) // NOTE: Must request the table this way to update properly + table.data.forEach(row => { + if (!(value in row)) row[value] = row[currentName] // Initialize new column with old values + delete row[currentName] // Delete old column + }) } - }, + + // Always re-render the table on column changes + table.requestUpdate() + }) } } + + + return { + main: tableConfiguration, + columns: columnTableConfig + } } schema.Ecephys.Units = { - ["*"]: generateLinkedTableInteractions('Units', 'UnitColumns', {}) + ["*"]: generateLinkedTableInteractions(['Ecephys','Units'], ['Ecephys', 'UnitColumns']) } -schema.Ecephys.Electrodes = { - // All interfaces - ["*"]: generateLinkedTableInteractions('Electrodes', 'ElectrodeColumns', { - group_name: function (this: JSONSchemaForm, _, __, ___, value) { +const linkedTableOutput = generateLinkedTableInteractions(['Ecephys', 'Electrodes'], ['Ecephys', 'ElectrodeColumns'], { + group_name: function (this: JSONSchemaForm, _, __, ___, value) { - const groups = this.results.Ecephys.ElectrodeGroup.map(({ name }) => name) // Groups are validated across all interfaces + const groups = this.results.Ecephys.ElectrodeGroup.map(({ name }) => name) // Groups are validated across all interfaces - if (groups.includes(value)) return true - else { - return [ - { - message: 'Not a valid group name', - type: 'error' - } - ] - } + if (groups.includes(value)) return true + else { + return [ + { + message: 'Not a valid group name', + type: 'error' + } + ] } - }) -} + } +}) + +schema.Ecephys.ElectrodeColumns = linkedTableOutput.columns +schema.Ecephys.Electrodes = { ["*"]: linkedTableOutput.main } // ----------------- Ophys Validation ----------------- // From 57283b3f724e6fa8c11498a77474d6ade76bf60d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 21 Mar 2024 21:41:43 +0000 Subject: [PATCH 14/31] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pyflask/manageNeuroconv/manage_neuroconv.py | 10 +++---- src/renderer/src/stories/BasicTable.js | 12 ++++----- src/renderer/src/stories/JSONSchemaForm.js | 2 +- .../pages/guided-mode/data/GuidedMetadata.js | 26 +++++++++---------- 4 files changed, 24 insertions(+), 26 deletions(-) diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index 82189799a..4f68e0ceb 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -393,13 +393,12 @@ def get_metadata_schema(source_data: Dict[str, dict], interfaces: dict) -> Dict[ ecephys_schema["required"].append("ElectrodeColumns") - ecephys_properties["Electrodes"] = {"type": "object", "properties": {}, "required": []} # Populate Units metadata metadata["Ecephys"]["Units"] = {} schema["properties"]["Ecephys"]["required"].append("Units") - original_units_schema = ecephys_properties.pop("UnitProperties", None) + original_units_schema = ecephys_properties.pop("UnitProperties", None) metadata["Ecephys"].pop("UnitProperties", None) # Always remove top-level UnitProperties from metadata has_units = original_units_schema is not None @@ -420,7 +419,7 @@ def on_sorting_interface(name, sorting_interface): unit_columns = get_unit_columns_json(sorting_interface) # Aggregate electrode column information across recording interfaces - existing_unit_columns = metadata["Ecephys"].get('UnitColumns') + existing_unit_columns = metadata["Ecephys"].get("UnitColumns") if existing_unit_columns: for entry in unit_columns: if any(obj["name"] == entry["name"] for obj in existing_unit_columns): @@ -458,7 +457,7 @@ def on_recording_interface(name, recording_interface): electrode_columns = get_electrode_columns_json(recording_interface) # Aggregate electrode column information across recording interfaces - existing_electrode_columns = metadata["Ecephys"].get('ElectrodeColumns') + existing_electrode_columns = metadata["Ecephys"].get("ElectrodeColumns") if existing_electrode_columns: for entry in electrode_columns: if any(obj["name"] == entry["name"] for obj in existing_electrode_columns): @@ -481,13 +480,12 @@ def on_recording_interface(name, recording_interface): {"$ref": "#/properties/Ecephys/properties/definitions/Electrode"}, {"required": list(map(lambda info: info["name"], electrode_columns))}, ] - } + }, } ecephys_properties["Electrodes"]["required"].append(name) return recording_interface - from neuroconv.datainterfaces.ecephys.baserecordingextractorinterface import BaseRecordingExtractorInterface from neuroconv.datainterfaces.ecephys.basesortingextractorinterface import BaseSortingExtractorInterface diff --git a/src/renderer/src/stories/BasicTable.js b/src/renderer/src/stories/BasicTable.js index d2de3440a..008d46a77 100644 --- a/src/renderer/src/stories/BasicTable.js +++ b/src/renderer/src/stories/BasicTable.js @@ -382,7 +382,7 @@ export class BasicTable extends LitElement { }; #readTSV(text) { - console.log(text, text.split("\n")) + console.log(text, text.split("\n")); let data = text.split("\n").map((row) => row.split("\t").map((v) => { try { @@ -392,7 +392,7 @@ export class BasicTable extends LitElement { } }) ); // Map to actual values using JSON.parse - console.log(data) + console.log(data); let header = data.shift(); const structuredData = data.map((row) => @@ -409,14 +409,14 @@ export class BasicTable extends LitElement { const latest = (this.data[this.keyColumn ? cols[this.keyColumn] : row] = {}); Object.entries(cols).forEach(([key, value]) => { // if (key in this.#itemProps) { - const { type } = this.#getType(value, this.#itemProps[key]); - if (type === "string") value = `${value}`; // Convert to string if necessary - latest[key] = value; + const { type } = this.#getType(value, this.#itemProps[key]); + if (type === "string") value = `${value}`; // Convert to string if necessary + latest[key] = value; // } }); // Only include data from schema }); - console.log(header, data, structuredData, this.data, this.#itemProps) + console.log(header, data, structuredData, this.data, this.#itemProps); if (this.onUpdate) this.onUpdate([], data); // Update the whole table } diff --git a/src/renderer/src/stories/JSONSchemaForm.js b/src/renderer/src/stories/JSONSchemaForm.js index 8a517f5d9..889974d2f 100644 --- a/src/renderer/src/stories/JSONSchemaForm.js +++ b/src/renderer/src/stories/JSONSchemaForm.js @@ -569,7 +569,7 @@ export class JSONSchemaForm extends LitElement { const allErrors = Array.from(flaggedInputs) .map((inputElement) => { - if (!inputElement.nextElementSibling) return // Skip tables + if (!inputElement.nextElementSibling) return; // Skip tables return Array.from(inputElement.nextElementSibling.children).map((li) => li.message); }) .flat(); diff --git a/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js b/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js index 2f3c4be32..96888a7d5 100644 --- a/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js +++ b/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js @@ -30,21 +30,21 @@ const parentTableRenderConfig = { Units: (metadata) => { metadata.editable = false; return true; - } -} + }, +}; function getAggregateRequirements(path) { - const electrodeSchema = getSchema(path, this.schema) + const electrodeSchema = getSchema(path, this.schema); return Object.values(electrodeSchema.properties).reduce((set, schema) => { - schema.items.required.forEach(item => set.add(item)) - return set - }, new Set()) + schema.items.required.forEach((item) => set.add(item)); + return set; + }, new Set()); } const tableRenderConfig = { "*": (metadata) => new SimpleTable(metadata), ElectrodeColumns: function (metadata) { - const aggregateRequirements = getAggregateRequirements.call(this, ["Ecephys", 'Electrodes']) + const aggregateRequirements = getAggregateRequirements.call(this, ["Ecephys", "Electrodes"]); return new SimpleTable({ ...metadata, @@ -54,16 +54,15 @@ const tableRenderConfig = { }); }, UnitColumns: function (metadata) { - - const aggregateRequirements = getAggregateRequirements.call(this, ["Ecephys", 'Units']) + const aggregateRequirements = getAggregateRequirements.call(this, ["Ecephys", "Units"]); return new SimpleTable({ ...metadata, contextOptions: { row: { add: false, - remove: false - } + remove: false, + }, }, editable: { name: (value) => !aggregateRequirements.has(value), @@ -457,9 +456,10 @@ export class GuidedMetadataPage extends ManagedPage { const updatedSchema = structuredClone(metadata.schema); metadata.schema = updatedSchema; - const parentName = fullPath[fullPath.length - 1] + const parentName = fullPath[fullPath.length - 1]; - const tableConfig = tableRenderConfig[name] ?? parentTableRenderConfig[parentName] ?? tableRenderConfig["*"] ?? true; + const tableConfig = + tableRenderConfig[name] ?? parentTableRenderConfig[parentName] ?? tableRenderConfig["*"] ?? true; if (typeof tableConfig === "function") return tableConfig.call(form, metadata, [...fullPath, name]); else return tableConfig; }, From bfad2d7873af510541c1d2fcdace4aa85dc93199 Mon Sep 17 00:00:00 2001 From: Garrett Michael Flynn Date: Thu, 21 Mar 2024 14:44:45 -0700 Subject: [PATCH 15/31] Add new validation to units table as well --- src/renderer/src/validation/validation.ts | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/renderer/src/validation/validation.ts b/src/renderer/src/validation/validation.ts index aefef71ae..3ff0cacfe 100644 --- a/src/renderer/src/validation/validation.ts +++ b/src/renderer/src/validation/validation.ts @@ -310,12 +310,13 @@ const generateLinkedTableInteractions = (tablesPath, colTablePath, additionalCol } } -schema.Ecephys.Units = { - ["*"]: generateLinkedTableInteractions(['Ecephys','Units'], ['Ecephys', 'UnitColumns']) -} +const linkedUnitsTableOutput = generateLinkedTableInteractions(['Ecephys','Units'], ['Ecephys', 'UnitColumns']) + +schema.Ecephys.Units = { ["*"]: linkedUnitsTableOutput.main } +schema.Ecephys.UnitColumns = linkedUnitsTableOutput.columns -const linkedTableOutput = generateLinkedTableInteractions(['Ecephys', 'Electrodes'], ['Ecephys', 'ElectrodeColumns'], { +const linkedElectrodesTableOutput = generateLinkedTableInteractions(['Ecephys', 'Electrodes'], ['Ecephys', 'ElectrodeColumns'], { group_name: function (this: JSONSchemaForm, _, __, ___, value) { const groups = this.results.Ecephys.ElectrodeGroup.map(({ name }) => name) // Groups are validated across all interfaces @@ -332,8 +333,8 @@ const linkedTableOutput = generateLinkedTableInteractions(['Ecephys', 'Electrode } }) -schema.Ecephys.ElectrodeColumns = linkedTableOutput.columns -schema.Ecephys.Electrodes = { ["*"]: linkedTableOutput.main } +schema.Ecephys.ElectrodeColumns = linkedElectrodesTableOutput.columns +schema.Ecephys.Electrodes = { ["*"]: linkedElectrodesTableOutput.main } // ----------------- Ophys Validation ----------------- // From 91e0c158ca9b33e6d5ba33d55bd1c986e7afb30c Mon Sep 17 00:00:00 2001 From: Garrett Michael Flynn Date: Fri, 22 Mar 2024 17:36:45 -0700 Subject: [PATCH 16/31] Fix Ecephys property order --- schemas/base-metadata.schema.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/schemas/base-metadata.schema.ts b/schemas/base-metadata.schema.ts index 6ab1bc836..2ffe5b6bd 100644 --- a/schemas/base-metadata.schema.ts +++ b/schemas/base-metadata.schema.ts @@ -126,6 +126,10 @@ export const preprocessMetadataSchema = (schema: any = baseMetadataSchema, globa if (ecephys) { + ecephys.order = ["Device", "ElectrodeGroup"] + ecephys.properties.Device.title = 'Devices' + ecephys.properties.ElectrodeGroup.title = 'Electrode Groups' + if (ecephys.properties.ElectrodeColumns) ecephys.properties.ElectrodeColumns.order = COLUMN_SCHEMA_ORDER if (ecephys.properties.UnitProperties) ecephys.properties.UnitProperties.order = COLUMN_SCHEMA_ORDER From 3f35d2815cef5852f0924b3343117f603f9c05e2 Mon Sep 17 00:00:00 2001 From: Garrett Michael Flynn Date: Fri, 22 Mar 2024 18:29:35 -0700 Subject: [PATCH 17/31] Fix invalid table elements --- pyflask/manageNeuroconv/manage_neuroconv.py | 69 +++++++++++++++------ 1 file changed, 49 insertions(+), 20 deletions(-) diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index 4f68e0ceb..e74cef184 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -41,6 +41,9 @@ "description": "The brain area where the unit is located.", **EXTRA_INTERFACE_PROPERTIES["brain_area"], }, + "quality": { + "data_type": "str", + } } EXCLUDED_SORTING_INTERFACE_PROPERTIES = ["location", "spike_times", "electrodes"] # Not validated @@ -429,6 +432,7 @@ def on_sorting_interface(name, sorting_interface): else: metadata["Ecephys"]["UnitColumns"] = unit_columns + units_data = metadata["Ecephys"]["Units"][name] = get_unit_table_json(sorting_interface) n_units = len(units_data) @@ -449,8 +453,6 @@ def on_sorting_interface(name, sorting_interface): return sorting_interface - aggregate_electrode_info = {} - def on_recording_interface(name, recording_interface): global aggregate_electrode_columns @@ -705,27 +707,32 @@ def update_conversion_progress(**kwargs): has_units = "Units" in ecephys_metadata if has_units: + shared_units_columns = ecephys_metadata["UnitColumns"] for interface_name, interface_unit_results in ecephys_metadata["Units"].items(): interface = converter.data_interface_objects[interface_name] update_sorting_properties_from_table_as_json( interface, - unit_table_json=interface_unit_results["Units"], - unit_column_info=interface_unit_results["UnitColumns"], + unit_table_json=interface_unit_results, + unit_column_info=shared_units_columns, ) + del ecephys_metadata["UnitColumns"] del ecephys_metadata["Units"] + shared_electrode_columns = ecephys_metadata["ElectrodeColumns"] + for interface_name, interface_electrode_results in ecephys_metadata["Electrodes"].items(): interface = converter.data_interface_objects[interface_name] update_recording_properties_from_table_as_json( interface, - electrode_table_json=interface_electrode_results["Electrodes"], - electrode_column_info=interface_electrode_results["ElectrodeColumns"], + electrode_table_json=interface_electrode_results, + electrode_column_info=shared_electrode_columns, ) - del ecephys_metadata["Electrodes"] # NOTE: Not sure what this should be now... + del ecephys_metadata["ElectrodeColumns"] + del ecephys_metadata["Electrodes"] # Actually run the conversion converter.run_conversion( @@ -1152,25 +1159,30 @@ def get_unit_columns_json(interface) -> List[Dict[str, Any]]: properties = get_sorting_interface_properties(interface) property_descriptions = dict(clu_id="The cluster ID for the unit", group_id="The group ID for the unit") + property_data_types = dict() for property_name, property_info in EXTRA_SORTING_INTERFACE_PROPERTIES.items(): description = property_info.get("description", None) + data_type = property_info.get("data_type", None) if description: property_descriptions[property_name] = description + if data_type: + property_data_types[property_name] = data_type sorting_extractor = interface.sorting_extractor unit_ids = sorting_extractor.get_unit_ids() + unit_columns = [ dict( name=property_name, description=property_descriptions.get(property_name, "No description."), - data_type=get_property_dtype( + data_type=property_data_types.get(property_name, get_property_dtype( extractor=sorting_extractor, property_name=property_name, ids=[unit_ids[0]], extra_props=EXTRA_SORTING_INTERFACE_PROPERTIES, - ), + )), ) for property_name in properties.keys() ] @@ -1198,9 +1210,12 @@ def get_unit_table_json(interface) -> List[Dict[str, Any]]: for property_name in properties: if property_name is "name": - sorting_property_value = unit_id + sorting_property_value = str(unit_id) # Insert unit_id as name (str) elif property_name in EXTRA_SORTING_INTERFACE_PROPERTIES: - sorting_property_value = properties[property_name].get("default") + try: + sorting_property_value = properties[property_name].get("default") # Get default value + except: + sorting_property_value = properties[property_name][0] # Get first value else: sorting_property_value = sorting.get_property(key=property_name, ids=[unit_id])[ 0 # First axis is always units in SI @@ -1378,13 +1393,27 @@ def update_sorting_properties_from_table_as_json( for property_name, property_value in unit_properties.items(): - if property_name is "name": - continue + if property_name == "name": + continue # Already controlling unit_id with the above variable - sorting_extractor.set_property( - key=property_name, - values=np.array( - [property_value], dtype=unit_column_data_types[property_name] - ), # , description=unit_column_descriptions[property_name]), - ids=[unit_id], - ) + dtype = unit_column_data_types[property_name] + if property_name == 'quality': + property_value = [ property_value ] + dtype = 'object' # Should allow the array to go through + + try: + + sorting_extractor.set_property( + key=property_name, + values=np.array( + [property_value], dtype=dtype + ), + ids=[unit_id], + ) + + except Exception as e: + if property_name == 'original_cluster_id': + continue + + raise Exception(f"Error setting property {property_name} for unit {unit_id} ({property_value}): {e}", type(property_value)) + From 455f3feb21a1e6120e4ff912c751486bc1391e4f Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 23 Mar 2024 01:29:51 +0000 Subject: [PATCH 18/31] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pyflask/manageNeuroconv/manage_neuroconv.py | 39 +++++++++++---------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index e74cef184..c5d690f11 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -43,7 +43,7 @@ }, "quality": { "data_type": "str", - } + }, } EXCLUDED_SORTING_INTERFACE_PROPERTIES = ["location", "spike_times", "electrodes"] # Not validated @@ -432,7 +432,6 @@ def on_sorting_interface(name, sorting_interface): else: metadata["Ecephys"]["UnitColumns"] = unit_columns - units_data = metadata["Ecephys"]["Units"][name] = get_unit_table_json(sorting_interface) n_units = len(units_data) @@ -1172,17 +1171,19 @@ def get_unit_columns_json(interface) -> List[Dict[str, Any]]: sorting_extractor = interface.sorting_extractor unit_ids = sorting_extractor.get_unit_ids() - unit_columns = [ dict( name=property_name, description=property_descriptions.get(property_name, "No description."), - data_type=property_data_types.get(property_name, get_property_dtype( - extractor=sorting_extractor, - property_name=property_name, - ids=[unit_ids[0]], - extra_props=EXTRA_SORTING_INTERFACE_PROPERTIES, - )), + data_type=property_data_types.get( + property_name, + get_property_dtype( + extractor=sorting_extractor, + property_name=property_name, + ids=[unit_ids[0]], + extra_props=EXTRA_SORTING_INTERFACE_PROPERTIES, + ), + ), ) for property_name in properties.keys() ] @@ -1210,7 +1211,7 @@ def get_unit_table_json(interface) -> List[Dict[str, Any]]: for property_name in properties: if property_name is "name": - sorting_property_value = str(unit_id) # Insert unit_id as name (str) + sorting_property_value = str(unit_id) # Insert unit_id as name (str) elif property_name in EXTRA_SORTING_INTERFACE_PROPERTIES: try: sorting_property_value = properties[property_name].get("default") # Get default value @@ -1397,23 +1398,23 @@ def update_sorting_properties_from_table_as_json( continue # Already controlling unit_id with the above variable dtype = unit_column_data_types[property_name] - if property_name == 'quality': - property_value = [ property_value ] - dtype = 'object' # Should allow the array to go through + if property_name == "quality": + property_value = [property_value] + dtype = "object" # Should allow the array to go through try: sorting_extractor.set_property( key=property_name, - values=np.array( - [property_value], dtype=dtype - ), + values=np.array([property_value], dtype=dtype), ids=[unit_id], ) except Exception as e: - if property_name == 'original_cluster_id': + if property_name == "original_cluster_id": continue - raise Exception(f"Error setting property {property_name} for unit {unit_id} ({property_value}): {e}", type(property_value)) - + raise Exception( + f"Error setting property {property_name} for unit {unit_id} ({property_value}): {e}", + type(property_value), + ) From 205a3f7d64d4103ad991ed1bb78419a541c79022 Mon Sep 17 00:00:00 2001 From: Garrett Michael Flynn Date: Fri, 22 Mar 2024 18:43:10 -0700 Subject: [PATCH 19/31] Allow editing descriptions --- pyflask/manageNeuroconv/manage_neuroconv.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index e74cef184..a2f3766a3 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -717,8 +717,9 @@ def update_conversion_progress(**kwargs): unit_column_info=shared_units_columns, ) - del ecephys_metadata["UnitColumns"] + ecephys_metadata["UnitProperties"] = [ { "name": entry["name"], "description": entry["description"] } for entry in shared_units_columns ] del ecephys_metadata["Units"] + del ecephys_metadata["UnitColumns"] shared_electrode_columns = ecephys_metadata["ElectrodeColumns"] @@ -731,8 +732,8 @@ def update_conversion_progress(**kwargs): electrode_column_info=shared_electrode_columns, ) + ecephys_metadata["Electrodes"] = [ { "name": entry["name"], "description": entry["description"] } for entry in shared_electrode_columns ] del ecephys_metadata["ElectrodeColumns"] - del ecephys_metadata["Electrodes"] # Actually run the conversion converter.run_conversion( @@ -1375,7 +1376,7 @@ def update_recording_properties_from_table_as_json( # TODO: uncomment when neuroconv supports contact vectors (probe interface) # if "contact_vector" in property_names: # recording_extractor.set_property(key="contact_vector", values=modified_contact_vector) - + def update_sorting_properties_from_table_as_json( sorting_interface, unit_column_info: dict, unit_table_json: List[Dict[str, Any]] @@ -1383,7 +1384,7 @@ def update_sorting_properties_from_table_as_json( import numpy as np unit_column_data_types = {column["name"]: column["data_type"] for column in unit_column_info} - unit_column_descriptions = {column["name"]: column["description"] for column in unit_column_info} + sorting_extractor = sorting_interface.sorting_extractor for entry_index, entry in enumerate(unit_table_json): @@ -1415,5 +1416,4 @@ def update_sorting_properties_from_table_as_json( if property_name == 'original_cluster_id': continue - raise Exception(f"Error setting property {property_name} for unit {unit_id} ({property_value}): {e}", type(property_value)) - + raise Exception(f"Error setting property {property_name} for unit {unit_id} ({property_value}): {e}", type(property_value)) \ No newline at end of file From 53346ce3d9a3a8f9c8d7022b8ba31f53e6726990 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 23 Mar 2024 01:43:51 +0000 Subject: [PATCH 20/31] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pyflask/manageNeuroconv/manage_neuroconv.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index 08b91f825..62c939d63 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -716,7 +716,9 @@ def update_conversion_progress(**kwargs): unit_column_info=shared_units_columns, ) - ecephys_metadata["UnitProperties"] = [ { "name": entry["name"], "description": entry["description"] } for entry in shared_units_columns ] + ecephys_metadata["UnitProperties"] = [ + {"name": entry["name"], "description": entry["description"]} for entry in shared_units_columns + ] del ecephys_metadata["Units"] del ecephys_metadata["UnitColumns"] @@ -731,7 +733,9 @@ def update_conversion_progress(**kwargs): electrode_column_info=shared_electrode_columns, ) - ecephys_metadata["Electrodes"] = [ { "name": entry["name"], "description": entry["description"] } for entry in shared_electrode_columns ] + ecephys_metadata["Electrodes"] = [ + {"name": entry["name"], "description": entry["description"]} for entry in shared_electrode_columns + ] del ecephys_metadata["ElectrodeColumns"] # Actually run the conversion @@ -1377,7 +1381,7 @@ def update_recording_properties_from_table_as_json( # TODO: uncomment when neuroconv supports contact vectors (probe interface) # if "contact_vector" in property_names: # recording_extractor.set_property(key="contact_vector", values=modified_contact_vector) - + def update_sorting_properties_from_table_as_json( sorting_interface, unit_column_info: dict, unit_table_json: List[Dict[str, Any]] @@ -1385,7 +1389,7 @@ def update_sorting_properties_from_table_as_json( import numpy as np unit_column_data_types = {column["name"]: column["data_type"] for column in unit_column_info} - + sorting_extractor = sorting_interface.sorting_extractor for entry_index, entry in enumerate(unit_table_json): @@ -1415,4 +1419,7 @@ def update_sorting_properties_from_table_as_json( if property_name == "original_cluster_id": continue - raise Exception(f"Error setting property {property_name} for unit {unit_id} ({property_value}): {e}", type(property_value)) + raise Exception( + f"Error setting property {property_name} for unit {unit_id} ({property_value}): {e}", + type(property_value), + ) From 37464b20587a148ee77c62fd3e313762ecf5ba48 Mon Sep 17 00:00:00 2001 From: Garrett Michael Flynn Date: Sat, 23 Mar 2024 10:41:46 -0700 Subject: [PATCH 21/31] Fix unit_id type issue --- pyflask/manageNeuroconv/manage_neuroconv.py | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index 62c939d63..6adec0e3e 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -1407,19 +1407,10 @@ def update_sorting_properties_from_table_as_json( property_value = [property_value] dtype = "object" # Should allow the array to go through - try: - - sorting_extractor.set_property( - key=property_name, - values=np.array([property_value], dtype=dtype), - ids=[unit_id], - ) + sorting_extractor.set_property( + key=property_name, + values=np.array([property_value], dtype=dtype), + ids=[int(unit_id)], + ) - except Exception as e: - if property_name == "original_cluster_id": - continue - raise Exception( - f"Error setting property {property_name} for unit {unit_id} ({property_value}): {e}", - type(property_value), - ) From 79d2deb2aac821d5355f2fe20c83327dfb529e5d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 23 Mar 2024 17:43:10 +0000 Subject: [PATCH 22/31] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pyflask/manageNeuroconv/manage_neuroconv.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index 6adec0e3e..5e2c43186 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -1412,5 +1412,3 @@ def update_sorting_properties_from_table_as_json( values=np.array([property_value], dtype=dtype), ids=[int(unit_id)], ) - - From 3d2f704c734c0519ce796132f56df16138730a43 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Sat, 23 Mar 2024 15:51:16 -0400 Subject: [PATCH 23/31] Update pyflask/manageNeuroconv/manage_neuroconv.py --- pyflask/manageNeuroconv/manage_neuroconv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index 5e2c43186..48bf46bfc 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -1215,7 +1215,7 @@ def get_unit_table_json(interface) -> List[Dict[str, Any]]: unit_column = dict() for property_name in properties: - if property_name is "name": + if property_name == "name": sorting_property_value = str(unit_id) # Insert unit_id as name (str) elif property_name in EXTRA_SORTING_INTERFACE_PROPERTIES: try: From b7379b124cc1a64f7af6d349749370e2a44e1d3c Mon Sep 17 00:00:00 2001 From: Garrett Michael Flynn Date: Mon, 25 Mar 2024 11:26:27 -0700 Subject: [PATCH 24/31] Handle other array-to-string properties for Units --- pyflask/manageNeuroconv/manage_neuroconv.py | 54 ++++++++++++------- .../guided-mode/data/GuidedSourceData.js | 2 +- 2 files changed, 35 insertions(+), 21 deletions(-) diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index 48bf46bfc..db76e6376 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -28,22 +28,36 @@ } } -EXTRA_RECORDING_INTERFACE_PROPERTIES = { +EXTRA_RECORDING_INTERFACE_PROPERTIES = list(EXTRA_INTERFACE_PROPERTIES.keys()) + +RECORDING_INTERFACE_PROPERTY_OVERRIDES = { "brain_area": { "description": "The brain area where the electrode is located.", **EXTRA_INTERFACE_PROPERTIES["brain_area"], } } -EXTRA_SORTING_INTERFACE_PROPERTIES = { +EXTRA_SORTING_INTERFACE_PROPERTIES = ["name", *EXTRA_INTERFACE_PROPERTIES.keys()] + +SORTING_INTERFACE_PROPERTIES_TO_RECAST = { + "quality": { + "data_type": "str", + }, + "KSLabel": { + "data_type": "str", + }, + "KSLabel_repeat": { + "data_type": "str", + } +} + +SORTING_INTERFACE_PROPERTY_OVERRIDES = { "name": {"description": "The unique name for the unit", "data_type": "str"}, "brain_area": { "description": "The brain area where the unit is located.", **EXTRA_INTERFACE_PROPERTIES["brain_area"], }, - "quality": { - "data_type": "str", - }, + **SORTING_INTERFACE_PROPERTIES_TO_RECAST } EXCLUDED_SORTING_INTERFACE_PROPERTIES = ["location", "spike_times", "electrodes"] # Not validated @@ -1134,9 +1148,9 @@ def get_recording_interface_properties(recording_interface) -> Dict[str, Any]: if property_name not in EXCLUDED_RECORDING_INTERFACE_PROPERTIES } - for property_name, property_info in EXTRA_RECORDING_INTERFACE_PROPERTIES.items(): + for property_name in EXTRA_RECORDING_INTERFACE_PROPERTIES: if property_name not in properties: - properties[property_name] = property_info + properties[property_name] = {} return properties @@ -1151,9 +1165,9 @@ def get_sorting_interface_properties(sorting_interface) -> Dict[str, Any]: if property_name not in EXCLUDED_SORTING_INTERFACE_PROPERTIES } - for property_name, property_info in EXTRA_SORTING_INTERFACE_PROPERTIES.items(): + for property_name in EXTRA_SORTING_INTERFACE_PROPERTIES: if property_name not in properties: - properties[property_name] = property_info + properties[property_name] = {} return properties @@ -1165,7 +1179,7 @@ def get_unit_columns_json(interface) -> List[Dict[str, Any]]: property_descriptions = dict(clu_id="The cluster ID for the unit", group_id="The group ID for the unit") property_data_types = dict() - for property_name, property_info in EXTRA_SORTING_INTERFACE_PROPERTIES.items(): + for property_name, property_info in SORTING_INTERFACE_PROPERTY_OVERRIDES.items(): description = property_info.get("description", None) data_type = property_info.get("data_type", None) if description: @@ -1186,7 +1200,7 @@ def get_unit_columns_json(interface) -> List[Dict[str, Any]]: extractor=sorting_extractor, property_name=property_name, ids=[unit_ids[0]], - extra_props=EXTRA_SORTING_INTERFACE_PROPERTIES, + extra_props=SORTING_INTERFACE_PROPERTY_OVERRIDES, ), ), ) @@ -1217,11 +1231,11 @@ def get_unit_table_json(interface) -> List[Dict[str, Any]]: for property_name in properties: if property_name == "name": sorting_property_value = str(unit_id) # Insert unit_id as name (str) - elif property_name in EXTRA_SORTING_INTERFACE_PROPERTIES: + elif property_name in SORTING_INTERFACE_PROPERTY_OVERRIDES: try: - sorting_property_value = properties[property_name].get("default") # Get default value + sorting_property_value = SORTING_INTERFACE_PROPERTY_OVERRIDES[property_name]["default"] # Get default value except: - sorting_property_value = properties[property_name][0] # Get first value + sorting_property_value = sorting.get_property(key=property_name, ids=[unit_id])[0] else: sorting_property_value = sorting.get_property(key=property_name, ids=[unit_id])[ 0 # First axis is always units in SI @@ -1248,7 +1262,7 @@ def get_electrode_columns_json(interface) -> List[Dict[str, Any]]: offset_to_uV="The offset from the data type to microVolts, applied after the gain.", ) - for property_name, property_info in EXTRA_RECORDING_INTERFACE_PROPERTIES.items(): + for property_name, property_info in RECORDING_INTERFACE_PROPERTY_OVERRIDES.items(): description = property_info.get("description", None) if description: property_descriptions[property_name] = description @@ -1267,7 +1281,7 @@ def get_electrode_columns_json(interface) -> List[Dict[str, Any]]: extractor=recording_extractor, property_name=property_name, ids=[channel_ids[0]], - extra_props=EXTRA_RECORDING_INTERFACE_PROPERTIES, + extra_props=RECORDING_INTERFACE_PROPERTY_OVERRIDES, ), ) for property_name in properties.keys() @@ -1307,11 +1321,11 @@ def get_electrode_table_json(interface) -> List[Dict[str, Any]]: for electrode_id in electrode_ids: electrode_column = dict() for property_name in properties: - if property_name in EXTRA_RECORDING_INTERFACE_PROPERTIES: + if property_name in RECORDING_INTERFACE_PROPERTY_OVERRIDES: try: - recording_property_value = properties[property_name].get("default") # Get default value + recording_property_value = RECORDING_INTERFACE_PROPERTY_OVERRIDES[property_name]["default"] # Get default value except: - recording_property_value = properties[property_name][0] # Get first value + recording_property_value = recording.get_property(key=property_name, ids=[electrode_id])[0] else: recording_property_value = recording.get_property(key=property_name, ids=[electrode_id])[ 0 # First axis is always electodes in SI @@ -1403,7 +1417,7 @@ def update_sorting_properties_from_table_as_json( continue # Already controlling unit_id with the above variable dtype = unit_column_data_types[property_name] - if property_name == "quality": + if property_name in SORTING_INTERFACE_PROPERTIES_TO_RECAST: property_value = [property_value] dtype = "object" # Should allow the array to go through diff --git a/src/renderer/src/stories/pages/guided-mode/data/GuidedSourceData.js b/src/renderer/src/stories/pages/guided-mode/data/GuidedSourceData.js index 2d67ff619..be3f34d04 100644 --- a/src/renderer/src/stories/pages/guided-mode/data/GuidedSourceData.js +++ b/src/renderer/src/stories/pages/guided-mode/data/GuidedSourceData.js @@ -120,7 +120,7 @@ export class GuidedSourceDataPage extends ManagedPage { const [type, ...splitText] = result.message.split(":"); const text = splitText.length ? splitText.join(":").replaceAll("<", "<").replaceAll(">", ">") - : `
${result.traceback.trim().split("\n").slice(-2)[0].trim()}
`; + : result.traceback ? `
${result.traceback.trim().split("\n").slice(-2)[0].trim()}
` : ''; const message = `

Request Failed

${type}

${text}

`; this.notify(message, "error"); From 68c4c77cec1ee7969285817af61037a562440af8 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 25 Mar 2024 18:26:44 +0000 Subject: [PATCH 25/31] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pyflask/manageNeuroconv/manage_neuroconv.py | 12 ++++++++---- .../pages/guided-mode/data/GuidedSourceData.js | 4 +++- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index db76e6376..a198cc538 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -48,7 +48,7 @@ }, "KSLabel_repeat": { "data_type": "str", - } + }, } SORTING_INTERFACE_PROPERTY_OVERRIDES = { @@ -57,7 +57,7 @@ "description": "The brain area where the unit is located.", **EXTRA_INTERFACE_PROPERTIES["brain_area"], }, - **SORTING_INTERFACE_PROPERTIES_TO_RECAST + **SORTING_INTERFACE_PROPERTIES_TO_RECAST, } EXCLUDED_SORTING_INTERFACE_PROPERTIES = ["location", "spike_times", "electrodes"] # Not validated @@ -1233,7 +1233,9 @@ def get_unit_table_json(interface) -> List[Dict[str, Any]]: sorting_property_value = str(unit_id) # Insert unit_id as name (str) elif property_name in SORTING_INTERFACE_PROPERTY_OVERRIDES: try: - sorting_property_value = SORTING_INTERFACE_PROPERTY_OVERRIDES[property_name]["default"] # Get default value + sorting_property_value = SORTING_INTERFACE_PROPERTY_OVERRIDES[property_name][ + "default" + ] # Get default value except: sorting_property_value = sorting.get_property(key=property_name, ids=[unit_id])[0] else: @@ -1323,7 +1325,9 @@ def get_electrode_table_json(interface) -> List[Dict[str, Any]]: for property_name in properties: if property_name in RECORDING_INTERFACE_PROPERTY_OVERRIDES: try: - recording_property_value = RECORDING_INTERFACE_PROPERTY_OVERRIDES[property_name]["default"] # Get default value + recording_property_value = RECORDING_INTERFACE_PROPERTY_OVERRIDES[property_name][ + "default" + ] # Get default value except: recording_property_value = recording.get_property(key=property_name, ids=[electrode_id])[0] else: diff --git a/src/renderer/src/stories/pages/guided-mode/data/GuidedSourceData.js b/src/renderer/src/stories/pages/guided-mode/data/GuidedSourceData.js index be3f34d04..72a342512 100644 --- a/src/renderer/src/stories/pages/guided-mode/data/GuidedSourceData.js +++ b/src/renderer/src/stories/pages/guided-mode/data/GuidedSourceData.js @@ -120,7 +120,9 @@ export class GuidedSourceDataPage extends ManagedPage { const [type, ...splitText] = result.message.split(":"); const text = splitText.length ? splitText.join(":").replaceAll("<", "<").replaceAll(">", ">") - : result.traceback ? `
${result.traceback.trim().split("\n").slice(-2)[0].trim()}
` : ''; + : result.traceback + ? `
${result.traceback.trim().split("\n").slice(-2)[0].trim()}
` + : ""; const message = `

Request Failed

${type}

${text}

`; this.notify(message, "error"); From 5d6e337ddc6d4357425198089495f23b893bc655 Mon Sep 17 00:00:00 2001 From: Garrett Michael Flynn Date: Mon, 25 Mar 2024 14:02:33 -0700 Subject: [PATCH 26/31] Update to unit_name --- pyflask/manageNeuroconv/manage_neuroconv.py | 12 ++++++------ schemas/base-metadata.schema.ts | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index a198cc538..500382e4c 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -37,7 +37,7 @@ } } -EXTRA_SORTING_INTERFACE_PROPERTIES = ["name", *EXTRA_INTERFACE_PROPERTIES.keys()] +EXTRA_SORTING_INTERFACE_PROPERTIES = ["unit_name", *EXTRA_INTERFACE_PROPERTIES.keys()] SORTING_INTERFACE_PROPERTIES_TO_RECAST = { "quality": { @@ -52,7 +52,7 @@ } SORTING_INTERFACE_PROPERTY_OVERRIDES = { - "name": {"description": "The unique name for the unit", "data_type": "str"}, + "unit_name": {"description": "The unique name for the unit", "data_type": "str"}, "brain_area": { "description": "The brain area where the unit is located.", **EXTRA_INTERFACE_PROPERTIES["brain_area"], @@ -435,7 +435,7 @@ def on_sorting_interface(name, sorting_interface): unit_columns = get_unit_columns_json(sorting_interface) - # Aggregate electrode column information across recording interfaces + # Aggregate unit column information across sorting interfaces existing_unit_columns = metadata["Ecephys"].get("UnitColumns") if existing_unit_columns: for entry in unit_columns: @@ -1229,7 +1229,7 @@ def get_unit_table_json(interface) -> List[Dict[str, Any]]: unit_column = dict() for property_name in properties: - if property_name == "name": + if property_name == "unit_name": sorting_property_value = str(unit_id) # Insert unit_id as name (str) elif property_name in SORTING_INTERFACE_PROPERTY_OVERRIDES: try: @@ -1413,11 +1413,11 @@ def update_sorting_properties_from_table_as_json( for entry_index, entry in enumerate(unit_table_json): unit_properties = dict(entry) # copy - unit_id = unit_properties.pop("name", None) # NOTE: Is called unit_name in the actual units table + unit_id = unit_properties.pop("unit_name", None) # NOTE: Is called unit_name in the actual units table for property_name, property_value in unit_properties.items(): - if property_name == "name": + if property_name == "unit_name": continue # Already controlling unit_id with the above variable dtype = unit_column_data_types[property_name] diff --git a/schemas/base-metadata.schema.ts b/schemas/base-metadata.schema.ts index 2ffe5b6bd..30c4a9c4b 100644 --- a/schemas/base-metadata.schema.ts +++ b/schemas/base-metadata.schema.ts @@ -143,7 +143,7 @@ export const preprocessMetadataSchema = (schema: any = baseMetadataSchema, globa }) updateEcephysTable("Units", copy, { - order: ["name", "clu_id", "group_id"] + order: ["unit_name", "clu_id", "group_id"] }) } From 01521cdaa54d3265201f2fb03412632d0cb9c5ab Mon Sep 17 00:00:00 2001 From: Garrett Michael Flynn Date: Tue, 26 Mar 2024 18:12:15 -0700 Subject: [PATCH 27/31] First round of updates from GUIDE meeting --- schemas/base-metadata.schema.ts | 2 + src/renderer/src/stories/BasicTable.js | 15 +- src/renderer/src/stories/JSONSchemaInput.js | 9 +- src/renderer/src/stories/SimpleTable.js | 21 +- src/renderer/src/stories/Table.js | 6 +- .../pages/guided-mode/data/GuidedMetadata.js | 10 +- src/renderer/src/stories/table/Cell.ts | 23 +- src/renderer/src/stories/table/cells/base.ts | 4 + .../src/stories/table/cells/dropdown.ts | 199 ++++++++++++++++++ tests/metadata.test.ts | 89 +------- 10 files changed, 270 insertions(+), 108 deletions(-) create mode 100644 src/renderer/src/stories/table/cells/dropdown.ts diff --git a/schemas/base-metadata.schema.ts b/schemas/base-metadata.schema.ts index 30c4a9c4b..f3c37dbfc 100644 --- a/schemas/base-metadata.schema.ts +++ b/schemas/base-metadata.schema.ts @@ -142,6 +142,8 @@ export const preprocessMetadataSchema = (schema: any = baseMetadataSchema, globa order: ["channel_name", "group_name", "shank_electrode_number", ...UV_PROPERTIES] }) + // ecephys.properties["Units"].title = "Unit Summaries" + updateEcephysTable("Units", copy, { order: ["unit_name", "clu_id", "group_id"] }) diff --git a/src/renderer/src/stories/BasicTable.js b/src/renderer/src/stories/BasicTable.js index 008d46a77..a7e648c87 100644 --- a/src/renderer/src/stories/BasicTable.js +++ b/src/renderer/src/stories/BasicTable.js @@ -392,7 +392,7 @@ export class BasicTable extends LitElement { } }) ); // Map to actual values using JSON.parse - console.log(data); + let header = data.shift(); const structuredData = data.map((row) => @@ -410,7 +410,10 @@ export class BasicTable extends LitElement { Object.entries(cols).forEach(([key, value]) => { // if (key in this.#itemProps) { const { type } = this.#getType(value, this.#itemProps[key]); - if (type === "string") value = `${value}`; // Convert to string if necessary + if (type === "string") { + if (value === undefined) value = '' + else value = `${value}`; // Convert to string if necessary + } latest[key] = value; // } }); // Only include data from schema @@ -468,6 +471,8 @@ export class BasicTable extends LitElement { const data = (this.#data = this.#getData()); + const description = this.#schema.description; + return html`
@@ -530,6 +535,12 @@ export class BasicTable extends LitElement { > ` : ""} + + ${description ? + html`

+ ${description} +

` : '' + } `; } } diff --git a/src/renderer/src/stories/JSONSchemaInput.js b/src/renderer/src/stories/JSONSchemaInput.js index da09fcf71..dc99881df 100644 --- a/src/renderer/src/stories/JSONSchemaInput.js +++ b/src/renderer/src/stories/JSONSchemaInput.js @@ -531,6 +531,7 @@ export class JSONSchemaInput extends LitElement { // form, // pattern // showLabel + // description controls = []; // required; validateOnChange = true; @@ -646,6 +647,8 @@ export class JSONSchemaInput extends LitElement { if (input === null) return null; // Hide rendering + const description = this.description ?? schema.description; + return html`
${input}${this.controls ? html`
${this.controls}
` : ""}
${ - schema.description + description ? html`

- ${unsafeHTML(capitalize(schema.description))}${[".", "?", "!"].includes( - schema.description.slice(-1)[0] + ${unsafeHTML(capitalize(description))}${[".", "?", "!"].includes( + description.slice(-1)[0] ) ? "" : "."} diff --git a/src/renderer/src/stories/SimpleTable.js b/src/renderer/src/stories/SimpleTable.js index c5f8178ef..c96b4f2e9 100644 --- a/src/renderer/src/stories/SimpleTable.js +++ b/src/renderer/src/stories/SimpleTable.js @@ -68,6 +68,10 @@ export class SimpleTable extends LitElement { border: none; } + td[editable="false"] { + background: whitesmoke; + } + :host([loading]:not([waiting])) table { height: 250px; } @@ -490,10 +494,11 @@ export class SimpleTable extends LitElement { id: "add-row", label: "Add Row", onclick: (path) => { - const cell = this.#getCellFromPath(path); - if (!cell) return this.addRow(); // No cell selected - const { i } = cell.simpleTableInfo; - this.addRow(i); //2) // TODO: Support adding more than one row + // const cell = this.#getCellFromPath(path); + // if (!cell) return this.addRow(); // No cell selected + // const { i } = cell.simpleTableInfo; + const lastRow = this.#cells.length - 1; + this.addRow(lastRow); // Just insert row at the end }, }, remove: { @@ -504,6 +509,7 @@ export class SimpleTable extends LitElement { if (!cell) return; // No cell selected const { i, row } = cell.simpleTableInfo; // TODO: Support detecting when the user would like to remove more than one row + // Validate with empty values before removing (to trigger any dependent validations) const cols = this.#data[row]; @@ -782,7 +788,8 @@ export class SimpleTable extends LitElement { const schema = this.#itemProps[fullInfo.col]; const ignore = getIgnore(this.ignore, [fullInfo.col]); - const isEditable = getEditable(value, this.editable, fullInfo.col); + const rowData = this.#data[row]; + const isEditable = getEditable(value, rowData, this.editable, fullInfo.col); // Track the cell renderer const cell = new TableCell({ @@ -850,9 +857,13 @@ export class SimpleTable extends LitElement { #renderCell = (value, info) => { const td = document.createElement("td"); + const cell = value instanceof TableCell ? value : this.#createCell(value, info); cell.simpleTableInfo.td = td; + + td.setAttribute('editable', cell.editable); + td.onmouseover = () => { if (this.#selecting) this.#selectCells(cell); }; diff --git a/src/renderer/src/stories/Table.js b/src/renderer/src/stories/Table.js index 760639f0c..beb2c7376 100644 --- a/src/renderer/src/stories/Table.js +++ b/src/renderer/src/stories/Table.js @@ -16,10 +16,10 @@ const isRequired = (col, schema) => { return schema.required?.includes(col); }; -export const getEditable = (value, config, colName) => { +export const getEditable = (value, rowData = {}, config, colName) => { if (typeof config === "boolean") return config; - if (typeof config === "function") return config(value); - return getEditable(value, config?.[colName] ?? true); + if (typeof config === "function") return config(value, rowData); + return getEditable(value, rowData, config?.[colName] ?? true); }; export function sortTable(schema, keyColumn, order) { diff --git a/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js b/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js index 96888a7d5..f1878949d 100644 --- a/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js +++ b/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js @@ -26,9 +26,13 @@ import { Button } from "../../../Button.js"; import globalIcon from "../../../assets/global.svg?raw"; const parentTableRenderConfig = { - Electrodes: true, + Electrodes: (metadata) => { + metadata.schema.description = "Download, modify, and re-upload data to change the electrode information." + return true + }, Units: (metadata) => { metadata.editable = false; + metadata.schema.description = "Update unit information directly on your source data." return true; }, }; @@ -50,6 +54,7 @@ const tableRenderConfig = { ...metadata, editable: { name: (value) => !aggregateRequirements.has(value), + data_type: (_, row) => !aggregateRequirements.has(row.name) }, }); }, @@ -66,6 +71,7 @@ const tableRenderConfig = { }, editable: { name: (value) => !aggregateRequirements.has(value), + data_type: (_, row) => !aggregateRequirements.has(row.name) }, }); }, @@ -460,7 +466,7 @@ export class GuidedMetadataPage extends ManagedPage { const tableConfig = tableRenderConfig[name] ?? parentTableRenderConfig[parentName] ?? tableRenderConfig["*"] ?? true; - if (typeof tableConfig === "function") return tableConfig.call(form, metadata, [...fullPath, name]); + if (typeof tableConfig === "function") return tableConfig.call(form, metadata, [...fullPath, name], this); else return tableConfig; }, onThrow, diff --git a/src/renderer/src/stories/table/Cell.ts b/src/renderer/src/stories/table/Cell.ts index d4aa11b23..3027f6a5b 100644 --- a/src/renderer/src/stories/table/Cell.ts +++ b/src/renderer/src/stories/table/Cell.ts @@ -4,7 +4,7 @@ import { NestedInputCell } from "./cells/input" import { TableCellBase } from "./cells/base" import { DateTimeCell } from "./cells/date-time" - +import { DropdownCell } from "./cells/dropdown" import { getValue, renderValue } from './convert' @@ -71,11 +71,11 @@ export class TableCell extends LitElement { ` } - // static get properties() { - // return { - // value: { reflect: true } - // } - // } + static get properties() { + return { + editable: { reflect: true } + } + } type = 'text' @@ -115,6 +115,10 @@ export class TableCell extends LitElement { } set value(value) { + + + if (!this.editable && this.interacted === true) return // Don't set value if not editable + if (this.input) this.input.set(renderValue(value, this.schema)) // Allow null to be set directly this.#value = this.input ? this.input.getValue() // Ensure all operations are undoable / value is coerced @@ -173,6 +177,8 @@ export class TableCell extends LitElement { this.interacted = persistentInteraction // this.value = value + if (!this.editable) return // Don't set value if not editable + if (this.input) this.input.set(value) // Ensure all operations are undoable else this.#value = value // Silently set value if not rendered yet } @@ -218,6 +224,11 @@ export class TableCell extends LitElement { this.type = "table" } + else if (this.schema.enum) { + cls = DropdownCell + this.type = "dropdown" + } + // Only actually rerender if new class type if (cls !== this.#cls) { this.input = new cls({ diff --git a/src/renderer/src/stories/table/cells/base.ts b/src/renderer/src/stories/table/cells/base.ts index 14098c568..d4f3424be 100644 --- a/src/renderer/src/stories/table/cells/base.ts +++ b/src/renderer/src/stories/table/cells/base.ts @@ -116,6 +116,7 @@ export class TableCellBase extends LitElement { #editableClose = () => this.editToggle(false) toggle (state = !this.#active) { + if (state === this.#active) return if (state) { @@ -125,6 +126,8 @@ export class TableCellBase extends LitElement { this.setAttribute('editing', '') const listenForEnter = (ev: KeyboardEvent) => { + console.log(ev) + if (ev.key === 'Enter') { ev.preventDefault() ev.stopPropagation() @@ -158,6 +161,7 @@ export class TableCellBase extends LitElement { document.removeEventListener('click', this.#editableClose) } else { current = this.#editor.value + console.log('Editor value', current) this.interacted = true if (this.#editor && this.#editor.onEditEnd) this.#editor.onEditEnd() } diff --git a/src/renderer/src/stories/table/cells/dropdown.ts b/src/renderer/src/stories/table/cells/dropdown.ts new file mode 100644 index 000000000..a0e2d6f31 --- /dev/null +++ b/src/renderer/src/stories/table/cells/dropdown.ts @@ -0,0 +1,199 @@ + +import { TableCellBase } from "./base"; +import { BaseRenderer } from './renderers/base'; + +import { LitElement, html, css } from 'lit'; + +type DropdownProps = { + open: boolean, + items: any[] +} + +class Dropdown extends LitElement { + static styles = css` + + * { + box-sizing: border-box; + } + + :host([open]) { + display: block; + } + + :host { + display: none; + position: absolute; + z-index: 1000; + background-color: #f9f9f9; + min-width: 160px; + box-shadow: 0px 8px 16px 0px rgba(0,0,0,0.2); + } + + ul { + list-style-type: none; + padding: 0; + margin: 0; + max-height: 100px; + overflow-y: auto; + } + + ul li { + cursor: pointer; + } + + li { + padding: 10px; + } + + li:hover { + background-color: #f1f1f1; + } + + `; + + constructor(props: Partial) { + super(); + Object.assign(this, props); + } + + static get properties() { + return { + open: { type: Boolean, reflect: true }, + items: { type: Array } + }; + } + + #select = (value) => { + this.dispatchEvent(new CustomEvent("change", { detail: value })); + this.open = false + } + + render() { + return html` +

    ${this.items.map((item) => html`
  • this.#select(item)}>${item}
    • `)}
+ `; + } + + toggleDropdown = (state = !this.open) => this.open = state; +} + +customElements.get("nwb-dropdown") || customElements.define("nwb-dropdown", Dropdown); + +export class DropdownCell extends TableCellBase { + + constructor(props) { + super(props); + } + + + // renderer = new NestedRenderer({ value: this.value }) + + editor = new EnumEditor({ + schema: this.schema, + }) + +} + +customElements.get("nwb-dropdown-cell") || customElements.define("nwb-dropdown-cell", DropdownCell); + + +export class EnumEditor extends BaseRenderer { + + INPUT = document.createElement("input") + + static get styles() { + return css` + + * { + box-sizing: border-box; + } + + input { + background: transparent; + border: none; + width: 100%; + height: 100%; + padding: 7px 10px; + } + + input:focus { + outline: none; + } + ` + } + + __value = undefined + get value() { + return this.__value + } + + set value(value) { + this.__value = value + if (this.INPUT) this.INPUT.value = value + } + + constructor(props) { + super(props); + Object.assign(this, props); + + this.INPUT.setAttribute("size", "1") + + const dropdown = this.DROPDOWN = new Dropdown({ items: this.schema.enum }) + + document.body.appendChild(dropdown) + + const toResolve: { resolve?: Function } = {} + this.INPUT.addEventListener("blur", async (ev) => { + + if (toResolve.resolve) return toResolve.resolve() + + ev.stopPropagation() + + const promise = new Promise((resolve) => { + toResolve.resolve = () => { + delete toResolve.resolve + resolve(true) + } + + setTimeout(() => { + this.INPUT.focus() + this.INPUT.blur() + }, 100) + + }) + + await promise + }) + + dropdown.addEventListener("change", (e) => { + this.value = e.detail + if (toResolve.resolve) toResolve.resolve() + }) + } + + render() { + return html`${this.INPUT}` + } + + focus() { + this.INPUT.focus(); + } + + close() { + this.DROPDOWN.toggleDropdown(false) + } + + onEditStart () { + this.DROPDOWN.toggleDropdown(true) + const { top, left, height } = this.getBoundingClientRect() + this.DROPDOWN.style.top = `${top + height}px` + this.DROPDOWN.style.left = `${left}px` + this.focus();// Allow blur + } + + onEditEnd = () => { + this.close() + } +} + +customElements.get("nwb-enum-editor") || customElements.define("nwb-enum-editor", EnumEditor); diff --git a/tests/metadata.test.ts b/tests/metadata.test.ts index 650872e2c..fe4bc28cd 100644 --- a/tests/metadata.test.ts +++ b/tests/metadata.test.ts @@ -55,92 +55,6 @@ test('removing all existing sessions will maintain the related subject entry on expect(Object.keys(results)).toEqual(Object.keys(copy)) }) - -// TODO: Convert an integration -test('inter-table updates are triggered', async () => { - - const results = { - Ecephys: { // NOTE: This layer is required to place the properties at the right level for the hardcoded validation function - ElectrodeGroup: [{ name: 's1' }], - Electrodes: [{ group_name: 's1' }] - } - } - - const schema = { - properties: { - Ecephys: { - properties: { - ElectrodeGroup: { - type: "array", - items: { - required: ["name"], - properties: { - name: { - type: "string" - }, - }, - type: "object", - }, - }, - Electrodes: { - type: "array", - items: { - type: "object", - properties: { - group_name: { - type: "string", - }, - }, - } - }, - } - } - } - } - - - - // Add invalid electrode - const randomStringId = Math.random().toString(36).substring(7) - results.Ecephys.Electrodes.push({ group_name: randomStringId }) - - // Create the form - const form = new JSONSchemaForm({ - schema, - results, - validateOnChange, - renderTable: (name, metadata, path) => { - if (name !== "Electrodes") return new SimpleTable(metadata); - else return true - }, - }) - - document.body.append(form) - - await form.rendered - - // Validate that the results are incorrect - const errors = await form.validate().catch(() => true).catch(() => true) - expect(errors).toBe(true) // Is invalid - - // Update the table with the missing electrode group - const table = form.getFormElement(['Ecephys', 'ElectrodeGroup']) // This is a SimpleTable where rows can be added - const row = table.addRow() - - const baseRow = table.getRow(0) - row.forEach((cell, i) => { - if (cell.simpleTableInfo.col === 'name') cell.setInput(randomStringId) // Set name to random string id - else cell.setInput(baseRow[i].value) // Otherwise carry over info - }) - - // Wait a second for new row values to resolve as table data (async) - await new Promise((res) => setTimeout(() => res(true), 1000)) - - // Validate that the new structure is correct - const hasErrors = await form.validate().then(() => false).catch((e) => true) - expect(hasErrors).toBe(false) // Is valid -}) - const popupSchemas = { "type": "object", "required": ["keywords", "experimenter"], @@ -287,7 +201,8 @@ test('inter-table updates are triggered', async () => { await form.rendered // Validate that the results are incorrect - const errors = await form.validate().catch(() => true).catch(() => true) + const errors = await form.validate().catch(() => true).catch((e) => e) + console.log(errors) expect(errors).toBe(true) // Is invalid // Update the table with the missing electrode group From 768b2597b2673e793c0ca7c43b592d490a83d64b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 27 Mar 2024 01:12:31 +0000 Subject: [PATCH 28/31] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/renderer/src/stories/BasicTable.js | 13 ++++++------- src/renderer/src/stories/JSONSchemaInput.js | 4 +--- src/renderer/src/stories/SimpleTable.js | 3 +-- .../pages/guided-mode/data/GuidedMetadata.js | 13 +++++++------ src/renderer/src/stories/table/cells/dropdown.ts | 6 +++--- 5 files changed, 18 insertions(+), 21 deletions(-) diff --git a/src/renderer/src/stories/BasicTable.js b/src/renderer/src/stories/BasicTable.js index a7e648c87..993e99a81 100644 --- a/src/renderer/src/stories/BasicTable.js +++ b/src/renderer/src/stories/BasicTable.js @@ -411,7 +411,7 @@ export class BasicTable extends LitElement { // if (key in this.#itemProps) { const { type } = this.#getType(value, this.#itemProps[key]); if (type === "string") { - if (value === undefined) value = '' + if (value === undefined) value = ""; else value = `${value}`; // Convert to string if necessary } latest[key] = value; @@ -535,12 +535,11 @@ export class BasicTable extends LitElement { >
` : ""} - - ${description ? - html`

- ${description} -

` : '' - } + ${description + ? html`

+ ${description} +

` + : ""} `; } } diff --git a/src/renderer/src/stories/JSONSchemaInput.js b/src/renderer/src/stories/JSONSchemaInput.js index dc99881df..b623a188e 100644 --- a/src/renderer/src/stories/JSONSchemaInput.js +++ b/src/renderer/src/stories/JSONSchemaInput.js @@ -667,9 +667,7 @@ export class JSONSchemaInput extends LitElement { ${ description ? html`

- ${unsafeHTML(capitalize(description))}${[".", "?", "!"].includes( - description.slice(-1)[0] - ) + ${unsafeHTML(capitalize(description))}${[".", "?", "!"].includes(description.slice(-1)[0]) ? "" : "."}

` diff --git a/src/renderer/src/stories/SimpleTable.js b/src/renderer/src/stories/SimpleTable.js index c96b4f2e9..982ecfbd2 100644 --- a/src/renderer/src/stories/SimpleTable.js +++ b/src/renderer/src/stories/SimpleTable.js @@ -509,7 +509,6 @@ export class SimpleTable extends LitElement { if (!cell) return; // No cell selected const { i, row } = cell.simpleTableInfo; // TODO: Support detecting when the user would like to remove more than one row - // Validate with empty values before removing (to trigger any dependent validations) const cols = this.#data[row]; @@ -862,7 +861,7 @@ export class SimpleTable extends LitElement { cell.simpleTableInfo.td = td; - td.setAttribute('editable', cell.editable); + td.setAttribute("editable", cell.editable); td.onmouseover = () => { if (this.#selecting) this.#selectCells(cell); diff --git a/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js b/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js index f1878949d..81aaf1c43 100644 --- a/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js +++ b/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js @@ -27,12 +27,12 @@ import globalIcon from "../../../assets/global.svg?raw"; const parentTableRenderConfig = { Electrodes: (metadata) => { - metadata.schema.description = "Download, modify, and re-upload data to change the electrode information." - return true + metadata.schema.description = "Download, modify, and re-upload data to change the electrode information."; + return true; }, Units: (metadata) => { metadata.editable = false; - metadata.schema.description = "Update unit information directly on your source data." + metadata.schema.description = "Update unit information directly on your source data."; return true; }, }; @@ -54,7 +54,7 @@ const tableRenderConfig = { ...metadata, editable: { name: (value) => !aggregateRequirements.has(value), - data_type: (_, row) => !aggregateRequirements.has(row.name) + data_type: (_, row) => !aggregateRequirements.has(row.name), }, }); }, @@ -71,7 +71,7 @@ const tableRenderConfig = { }, editable: { name: (value) => !aggregateRequirements.has(value), - data_type: (_, row) => !aggregateRequirements.has(row.name) + data_type: (_, row) => !aggregateRequirements.has(row.name), }, }); }, @@ -466,7 +466,8 @@ export class GuidedMetadataPage extends ManagedPage { const tableConfig = tableRenderConfig[name] ?? parentTableRenderConfig[parentName] ?? tableRenderConfig["*"] ?? true; - if (typeof tableConfig === "function") return tableConfig.call(form, metadata, [...fullPath, name], this); + if (typeof tableConfig === "function") + return tableConfig.call(form, metadata, [...fullPath, name], this); else return tableConfig; }, onThrow, diff --git a/src/renderer/src/stories/table/cells/dropdown.ts b/src/renderer/src/stories/table/cells/dropdown.ts index a0e2d6f31..6835f941d 100644 --- a/src/renderer/src/stories/table/cells/dropdown.ts +++ b/src/renderer/src/stories/table/cells/dropdown.ts @@ -48,14 +48,14 @@ class Dropdown extends LitElement { li:hover { background-color: #f1f1f1; } - + `; constructor(props: Partial) { super(); Object.assign(this, props); } - + static get properties() { return { open: { type: Boolean, reflect: true }, @@ -136,7 +136,7 @@ export class EnumEditor extends BaseRenderer { super(props); Object.assign(this, props); - this.INPUT.setAttribute("size", "1") + this.INPUT.setAttribute("size", "1") const dropdown = this.DROPDOWN = new Dropdown({ items: this.schema.enum }) From 445b6f76b6fecf69bbfc8f6445443af487593233 Mon Sep 17 00:00:00 2001 From: Garrett Michael Flynn Date: Wed, 27 Mar 2024 15:57:37 -0700 Subject: [PATCH 29/31] Only allow removing non-required rows --- src/renderer/src/stories/SimpleTable.js | 37 +++++++++++++++++++ .../pages/guided-mode/data/GuidedMetadata.js | 3 +- src/renderer/src/stories/table/ContextMenu.ts | 9 ++++- 3 files changed, 47 insertions(+), 2 deletions(-) diff --git a/src/renderer/src/stories/SimpleTable.js b/src/renderer/src/stories/SimpleTable.js index 982ecfbd2..13ba7b8ba 100644 --- a/src/renderer/src/stories/SimpleTable.js +++ b/src/renderer/src/stories/SimpleTable.js @@ -565,6 +565,43 @@ export class SimpleTable extends LitElement { this.#context = new ContextMenu({ target: this.shadowRoot.querySelector("table"), items, + onOpen: (path) => { + + const checks = { + row_remove: { + check: this.editable.__row_remove, + element: this.#context.shadowRoot.querySelector("#remove-row") + }, + + row_add: { + check: this.editable.__row_add, + element: this.#context.shadowRoot.querySelector("#add-row") + } + } + + const hasChecks = Object.values(checks).some(({ check }) => check); + + if (hasChecks) { + + const cell = this.#getCellFromPath(path); + const info = cell.simpleTableInfo; + const rowNames = Object.keys(this.#data); + const row = Array.isArray(this.#data) ? info.i : rowNames[info.i]; + + const results = Object.values(checks).map(({ check, element }) => { + if (check) { + const canRemove = check(cell.value, this.#data[row]); + if (canRemove) element.removeAttribute("disabled"); + else element.setAttribute("disabled", ""); + return canRemove + } else return true + }) + + return !results.every((r) => r === false) // If all are hidden, don't show the context menu + } + + return true + } }); this.#context.updated = () => this.#updateContextMenuRendering(); // Run when done rendering diff --git a/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js b/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js index 81aaf1c43..85bb01a45 100644 --- a/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js +++ b/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js @@ -55,6 +55,7 @@ const tableRenderConfig = { editable: { name: (value) => !aggregateRequirements.has(value), data_type: (_, row) => !aggregateRequirements.has(row.name), + __row_remove: (_, row) => !aggregateRequirements.has(row.name), }, }); }, @@ -71,7 +72,7 @@ const tableRenderConfig = { }, editable: { name: (value) => !aggregateRequirements.has(value), - data_type: (_, row) => !aggregateRequirements.has(row.name), + data_type: (_, row) => !aggregateRequirements.has(row.name) }, }); }, diff --git a/src/renderer/src/stories/table/ContextMenu.ts b/src/renderer/src/stories/table/ContextMenu.ts index 4f382b78b..997662b91 100644 --- a/src/renderer/src/stories/table/ContextMenu.ts +++ b/src/renderer/src/stories/table/ContextMenu.ts @@ -66,12 +66,14 @@ export class ContextMenu extends LitElement{ declare target: Document | HTMLElement declare items: any[] + declare onOpen: () => boolean | void - constructor({ target, items }: any){ + constructor({ target, items, onOpen }: any){ super() this.target = target ?? document this.items = items ?? [] + this.onOpen = onOpen ?? (() => {}) document.addEventListener('click', () => this.#hide()) // Hide at the last step of any click document.addEventListener('contextmenu', () => this.#hide()) @@ -88,7 +90,12 @@ export class ContextMenu extends LitElement{ #open(mouseEvent: MouseEvent) { mouseEvent.preventDefault() mouseEvent.stopPropagation() + this.#activePath = mouseEvent.path || mouseEvent.composedPath() + + const result = this.onOpen(this.#activePath) + if (result === false) return + this.style.display = 'block'; this.style.left = mouseEvent.pageX + "px"; this.style.top = mouseEvent.pageY + "px"; From ebedb05cb8a6928962a0335dc6d8a9851eed8499 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 28 Mar 2024 15:11:22 +0000 Subject: [PATCH 30/31] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/renderer/src/stories/SimpleTable.js | 22 +++++++++---------- .../pages/guided-mode/data/GuidedMetadata.js | 2 +- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/src/renderer/src/stories/SimpleTable.js b/src/renderer/src/stories/SimpleTable.js index d3ff18703..2fd49fd85 100644 --- a/src/renderer/src/stories/SimpleTable.js +++ b/src/renderer/src/stories/SimpleTable.js @@ -566,23 +566,21 @@ export class SimpleTable extends LitElement { target: this.shadowRoot.querySelector("table"), items, onOpen: (path) => { - const checks = { row_remove: { check: this.editable.__row_remove, - element: this.#context.shadowRoot.querySelector("#remove-row") + element: this.#context.shadowRoot.querySelector("#remove-row"), }, row_add: { check: this.editable.__row_add, - element: this.#context.shadowRoot.querySelector("#add-row") - } - } + element: this.#context.shadowRoot.querySelector("#add-row"), + }, + }; const hasChecks = Object.values(checks).some(({ check }) => check); if (hasChecks) { - const cell = this.#getCellFromPath(path); const info = cell.simpleTableInfo; const rowNames = Object.keys(this.#data); @@ -593,15 +591,15 @@ export class SimpleTable extends LitElement { const canRemove = check(cell.value, this.#data[row]); if (canRemove) element.removeAttribute("disabled"); else element.setAttribute("disabled", ""); - return canRemove - } else return true - }) + return canRemove; + } else return true; + }); - return !results.every((r) => r === false) // If all are hidden, don't show the context menu + return !results.every((r) => r === false); // If all are hidden, don't show the context menu } - return true - } + return true; + }, }); this.#context.updated = () => this.#updateContextMenuRendering(); // Run when done rendering diff --git a/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js b/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js index 55f3223b3..1e35cf2b0 100644 --- a/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js +++ b/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js @@ -72,7 +72,7 @@ const tableRenderConfig = { }, editable: { name: (value) => !aggregateRequirements.has(value), - data_type: (_, row) => !aggregateRequirements.has(row.name) + data_type: (_, row) => !aggregateRequirements.has(row.name), }, }); }, From 7c3f0443bb45a0109c8ce73a1445eca01291b99e Mon Sep 17 00:00:00 2001 From: Garrett Michael Flynn Date: Thu, 28 Mar 2024 13:18:35 -0700 Subject: [PATCH 31/31] Fix tests --- tests/metadata.test.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/metadata.test.ts b/tests/metadata.test.ts index dd8b743a1..0cb3682f9 100644 --- a/tests/metadata.test.ts +++ b/tests/metadata.test.ts @@ -214,8 +214,7 @@ test('inter-table updates are triggered', async () => { else cell.setInput(baseRow[i].value) // Otherwise carry over info }) - // Wait a second for new row values to resolve as table data (async) - await new Promise((res) => setTimeout(() => res(true), 1000)) + form.requestUpdate() // Re-render the form to update the table // Validate that the new structure is correct const hasErrors = await form.validate().then(() => false).catch((e) => true)