diff --git a/guide_testing_suite.yml b/guide_testing_suite.yml index e814ed56c..d10b00844 100644 --- a/guide_testing_suite.yml +++ b/guide_testing_suite.yml @@ -33,3 +33,14 @@ pipelines: TDT: TdtRecordingInterface: folder_path: ephy_testing_data/tdt/aep_05 + + CellExplorer: + CellExplorerRecordingInterface: + folder_path: ephy_testing_data/cellexplorer/dataset_4/Peter_MS22_180629_110319_concat_stubbed + + + CellExplorerUnits: + CellExplorerRecordingInterface: + folder_path: ephy_testing_data/cellexplorer/dataset_4/Peter_MS22_180629_110319_concat_stubbed + CellExplorerSortingInterface: + file_path: ephy_testing_data/cellexplorer/dataset_4/Peter_MS22_180629_110319_concat_stubbed/Peter_MS22_180629_110319_concat_stubbed.spikes.cellinfo.mat diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index ee137a943..500382e4c 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -20,14 +20,70 @@ EXCLUDED_RECORDING_INTERFACE_PROPERTIES = ["contact_vector", "contact_shapes", "group", "location"] -EXTRA_RECORDING_INTERFACE_PROPERTIES = { + +EXTRA_INTERFACE_PROPERTIES = { "brain_area": { "data_type": "str", - "description": "The brain area where the electrode is located.", "default": "unknown", } } +EXTRA_RECORDING_INTERFACE_PROPERTIES = list(EXTRA_INTERFACE_PROPERTIES.keys()) + +RECORDING_INTERFACE_PROPERTY_OVERRIDES = { + "brain_area": { + "description": "The brain area where the electrode is located.", + **EXTRA_INTERFACE_PROPERTIES["brain_area"], + } +} + +EXTRA_SORTING_INTERFACE_PROPERTIES = ["unit_name", *EXTRA_INTERFACE_PROPERTIES.keys()] + +SORTING_INTERFACE_PROPERTIES_TO_RECAST = { + "quality": { + "data_type": "str", + }, + "KSLabel": { + "data_type": "str", + }, + "KSLabel_repeat": { + "data_type": "str", + }, +} + +SORTING_INTERFACE_PROPERTY_OVERRIDES = { + "unit_name": {"description": "The unique name for the unit", "data_type": "str"}, + "brain_area": { + "description": "The brain area where the unit is located.", + **EXTRA_INTERFACE_PROPERTIES["brain_area"], + }, + **SORTING_INTERFACE_PROPERTIES_TO_RECAST, +} + +EXCLUDED_SORTING_INTERFACE_PROPERTIES = ["location", "spike_times", "electrodes"] # Not validated + +# NOTE: These are the only accepted dtypes... +DTYPE_DESCRIPTIONS = { + "bool": "logical", + "str": "string", + "ndarray": "n-dimensional array", + "float8": "8-bit number", + "float16": "16-bit number", + "float32": "32-bit number", + "float64": "64-bit number", + "int8": "8-bit integer", + "int16": "16-bit integer", + "int32": "32-bit integer", + "int64": "64-bit integer", +} + +DTYPE_SCHEMA = { + "type": "string", + # "strict": False, + "enum": list(DTYPE_DESCRIPTIONS.keys()), + "enumLabels": DTYPE_DESCRIPTIONS, +} + def is_path_contained(child, parent): parent = Path(parent) @@ -302,8 +358,7 @@ def get_source_schema(interface_class_dict: dict) -> dict: return CustomNWBConverter.get_source_schema() -def map_recording_interfaces(callback, converter): - from neuroconv.datainterfaces.ecephys.baserecordingextractorinterface import BaseRecordingExtractorInterface +def map_interfaces(BaseRecordingExtractorInterface, callback, converter): output = [] @@ -329,91 +384,179 @@ def get_metadata_schema(source_data: Dict[str, dict], interfaces: dict) -> Dict[ # Clear the Electrodes information for being set as a collection of Interfaces has_ecephys = "Ecephys" in metadata + has_units = False if has_ecephys: - metadata["Ecephys"]["Electrodes"] = {} - schema["properties"]["Ecephys"]["required"].append("Electrodes") - ecephys_properties = schema["properties"]["Ecephys"]["properties"] + ecephys_schema = schema["properties"]["Ecephys"] + + if not ecephys_schema.get("required"): + ecephys_schema["required"] = [] + + ecephys_properties = ecephys_schema["properties"] + + # Populate Electrodes metadata original_electrodes_schema = ecephys_properties["Electrodes"] + # Add Electrodes to the schema + metadata["Ecephys"]["Electrodes"] = {} + ecephys_schema["required"].append("Electrodes") + + ecephys_properties["ElectrodeColumns"] = { + "type": "array", + "minItems": 0, + "items": {"$ref": "#/properties/Ecephys/properties/definitions/ElectrodeColumn"}, + } + + ecephys_schema["required"].append("ElectrodeColumns") + ecephys_properties["Electrodes"] = {"type": "object", "properties": {}, "required": []} - def on_recording_interface(name, recording_interface): + # Populate Units metadata + metadata["Ecephys"]["Units"] = {} + schema["properties"]["Ecephys"]["required"].append("Units") + original_units_schema = ecephys_properties.pop("UnitProperties", None) + metadata["Ecephys"].pop("UnitProperties", None) # Always remove top-level UnitProperties from metadata - metadata["Ecephys"]["Electrodes"][name] = dict( - Electrodes=get_electrode_table_json(recording_interface), - ElectrodeColumns=get_electrode_columns_json(recording_interface), - ) + has_units = original_units_schema is not None - ecephys_properties["Electrodes"]["properties"][name] = dict( - type="object", - properties=dict( - Electrodes={ - "type": "array", - "minItems": 0, - "items": {"$ref": "#/properties/Ecephys/properties/definitions/Electrode"}, - }, - ElectrodeColumns={ - "type": "array", - "minItems": 0, - "items": {"$ref": "#/properties/Ecephys/properties/definitions/ElectrodeColumn"}, - }, - ), - required=["Electrodes", "ElectrodeColumns"], - ) + if has_units: + metadata["Ecephys"] + ecephys_properties["UnitColumns"] = { + "type": "array", + "minItems": 0, + "items": {"$ref": "#/properties/Ecephys/properties/definitions/UnitColumn"}, + } + + schema["properties"]["Ecephys"]["required"].append("UnitColumns") + ecephys_properties["Units"] = {"type": "object", "properties": {}, "required": []} + + def on_sorting_interface(name, sorting_interface): + + unit_columns = get_unit_columns_json(sorting_interface) + + # Aggregate unit column information across sorting interfaces + existing_unit_columns = metadata["Ecephys"].get("UnitColumns") + if existing_unit_columns: + for entry in unit_columns: + if any(obj["name"] == entry["name"] for obj in existing_unit_columns): + continue + else: + existing_unit_columns.append(entry) + else: + metadata["Ecephys"]["UnitColumns"] = unit_columns + + units_data = metadata["Ecephys"]["Units"][name] = get_unit_table_json(sorting_interface) + + n_units = len(units_data) + + ecephys_properties["Units"]["properties"][name] = { + "type": "array", + "minItems": n_units, + "maxItems": n_units, + "items": { + "allOf": [ + {"$ref": "#/properties/Ecephys/properties/definitions/Unit"}, + {"required": list(map(lambda info: info["name"], unit_columns))}, + ] + }, + } + + ecephys_properties["Units"]["required"].append(name) + + return sorting_interface + + def on_recording_interface(name, recording_interface): + global aggregate_electrode_columns + + electrode_columns = get_electrode_columns_json(recording_interface) + + # Aggregate electrode column information across recording interfaces + existing_electrode_columns = metadata["Ecephys"].get("ElectrodeColumns") + if existing_electrode_columns: + for entry in electrode_columns: + if any(obj["name"] == entry["name"] for obj in existing_electrode_columns): + continue + else: + existing_electrode_columns.append(entry) + else: + metadata["Ecephys"]["ElectrodeColumns"] = electrode_columns + + electrode_data = metadata["Ecephys"]["Electrodes"][name] = get_electrode_table_json(recording_interface) + + n_electrodes = len(electrode_data) + + ecephys_properties["Electrodes"]["properties"][name] = { + "type": "array", + "minItems": n_electrodes, + "maxItems": n_electrodes, + "items": { + "allOf": [ + {"$ref": "#/properties/Ecephys/properties/definitions/Electrode"}, + {"required": list(map(lambda info: info["name"], electrode_columns))}, + ] + }, + } ecephys_properties["Electrodes"]["required"].append(name) return recording_interface - recording_interfaces = map_recording_interfaces(on_recording_interface, converter) + from neuroconv.datainterfaces.ecephys.baserecordingextractorinterface import BaseRecordingExtractorInterface + from neuroconv.datainterfaces.ecephys.basesortingextractorinterface import BaseSortingExtractorInterface + + # Map recording interfaces to metadata + map_interfaces(BaseRecordingExtractorInterface, on_recording_interface, converter) - # Delete Ecephys metadata if ElectrodeTable helper function is not available + # Map sorting interfaces to metadata + map_interfaces(BaseSortingExtractorInterface, on_sorting_interface, converter) + + # Delete Ecephys metadata if no interfaces processed if has_ecephys: - if len(recording_interfaces) == 0: - schema["properties"].pop("Ecephys", dict()) - else: + defs = ecephys_properties["definitions"] - defs = ecephys_properties["definitions"] - electrode_def = defs["Electrodes"] - - dtype_descriptions = { - "bool": "logical", - "str": "string", - "ndarray": "n-dimensional array", - "float8": "8-bit number", - "float16": "16-bit number", - "float32": "32-bit number", - "float64": "64-bit number", - "int8": "8-bit integer", - "int16": "16-bit integer", - "int32": "32-bit integer", - "int64": "64-bit integer", - } + electrode_def = defs["Electrodes"] + + # NOTE: Update to output from NeuroConv + electrode_def["properties"]["data_type"] = DTYPE_SCHEMA + + # Configure electrode columns + defs["ElectrodeColumn"] = electrode_def + defs["ElectrodeColumn"]["required"] = list(electrode_def["properties"].keys()) + + new_electrodes_properties = { + properties["name"]: {key: value for key, value in properties.items() if key != "name"} + for properties in original_electrodes_schema.get("default", {}) + if properties["name"] not in EXCLUDED_RECORDING_INTERFACE_PROPERTIES + } + + defs["Electrode"] = { + "type": "object", + "properties": new_electrodes_properties, + "additionalProperties": True, # Allow for new columns + } + + if has_units: + + unitprops_def = defs["UnitProperties"] # NOTE: Update to output from NeuroConv - electrode_def["properties"]["data_type"] = { - "type": "string", - "strict": False, - "enum": list(dtype_descriptions.keys()), - "enumLabels": dtype_descriptions, - } + unitprops_def["properties"]["data_type"] = DTYPE_SCHEMA # Configure electrode columns - defs["ElectrodeColumn"] = electrode_def - defs["ElectrodeColumn"]["required"] = list(electrode_def["properties"].keys()) + defs["UnitColumn"] = unitprops_def + defs["UnitColumn"]["required"] = list(unitprops_def["properties"].keys()) - new_electrodes_properties = { + new_units_properties = { properties["name"]: {key: value for key, value in properties.items() if key != "name"} - for properties in original_electrodes_schema["default"] - if properties["name"] not in EXCLUDED_RECORDING_INTERFACE_PROPERTIES + for properties in original_units_schema.get("default", {}) + if properties["name"] not in EXCLUDED_SORTING_INTERFACE_PROPERTIES } - defs["Electrode"] = { + defs["Unit"] = { "type": "object", - "properties": new_electrodes_properties, + "properties": new_units_properties, "additionalProperties": True, # Allow for new columns } @@ -573,16 +716,41 @@ def update_conversion_progress(**kwargs): if ecephys_metadata: + # Quick fix to remove units + has_units = "Units" in ecephys_metadata + + if has_units: + shared_units_columns = ecephys_metadata["UnitColumns"] + for interface_name, interface_unit_results in ecephys_metadata["Units"].items(): + interface = converter.data_interface_objects[interface_name] + + update_sorting_properties_from_table_as_json( + interface, + unit_table_json=interface_unit_results, + unit_column_info=shared_units_columns, + ) + + ecephys_metadata["UnitProperties"] = [ + {"name": entry["name"], "description": entry["description"]} for entry in shared_units_columns + ] + del ecephys_metadata["Units"] + del ecephys_metadata["UnitColumns"] + + shared_electrode_columns = ecephys_metadata["ElectrodeColumns"] + for interface_name, interface_electrode_results in ecephys_metadata["Electrodes"].items(): interface = converter.data_interface_objects[interface_name] update_recording_properties_from_table_as_json( interface, - electrode_table_json=interface_electrode_results["Electrodes"], - electrode_column_info=interface_electrode_results["ElectrodeColumns"], + electrode_table_json=interface_electrode_results, + electrode_column_info=shared_electrode_columns, ) - del ecephys_metadata["Electrodes"] # NOTE: Not sure what this should be now... + ecephys_metadata["Electrodes"] = [ + {"name": entry["name"], "description": entry["description"]} for entry in shared_electrode_columns + ] + del ecephys_metadata["ElectrodeColumns"] # Actually run the conversion converter.run_conversion( @@ -958,11 +1126,11 @@ def map_dtype(dtype: str) -> str: return dtype -def get_property_dtype(recording_extractor, property_name: str, channel_ids: list): - if property_name in EXTRA_RECORDING_INTERFACE_PROPERTIES: - dtype = EXTRA_RECORDING_INTERFACE_PROPERTIES[property_name]["data_type"] +def get_property_dtype(extractor, property_name: str, ids: list, extra_props: dict): + if property_name in extra_props: + dtype = extra_props[property_name]["data_type"] else: - dtype = str(recording_extractor.get_property(key=property_name, ids=channel_ids).dtype) + dtype = str(extractor.get_property(key=property_name, ids=ids).dtype) # return type(recording.get_property(key=property_name)[0]).__name__.replace("_", "") # return dtype @@ -980,13 +1148,107 @@ def get_recording_interface_properties(recording_interface) -> Dict[str, Any]: if property_name not in EXCLUDED_RECORDING_INTERFACE_PROPERTIES } - for property_name, property_info in EXTRA_RECORDING_INTERFACE_PROPERTIES.items(): + for property_name in EXTRA_RECORDING_INTERFACE_PROPERTIES: + if property_name not in properties: + properties[property_name] = {} + + return properties + + +def get_sorting_interface_properties(sorting_interface) -> Dict[str, Any]: + """A convenience function for uniformly excluding certain properties of the provided sorting extractor.""" + property_names = list(sorting_interface.sorting_extractor.get_property_keys()) + + properties = { + property_name: sorting_interface.sorting_extractor.get_property(key=property_name) + for property_name in property_names + if property_name not in EXCLUDED_SORTING_INTERFACE_PROPERTIES + } + + for property_name in EXTRA_SORTING_INTERFACE_PROPERTIES: if property_name not in properties: - properties[property_name] = property_info + properties[property_name] = {} return properties +def get_unit_columns_json(interface) -> List[Dict[str, Any]]: + """A convenience function for collecting and organizing the properties of the underlying sorting extractor.""" + properties = get_sorting_interface_properties(interface) + + property_descriptions = dict(clu_id="The cluster ID for the unit", group_id="The group ID for the unit") + property_data_types = dict() + + for property_name, property_info in SORTING_INTERFACE_PROPERTY_OVERRIDES.items(): + description = property_info.get("description", None) + data_type = property_info.get("data_type", None) + if description: + property_descriptions[property_name] = description + if data_type: + property_data_types[property_name] = data_type + + sorting_extractor = interface.sorting_extractor + unit_ids = sorting_extractor.get_unit_ids() + + unit_columns = [ + dict( + name=property_name, + description=property_descriptions.get(property_name, "No description."), + data_type=property_data_types.get( + property_name, + get_property_dtype( + extractor=sorting_extractor, + property_name=property_name, + ids=[unit_ids[0]], + extra_props=SORTING_INTERFACE_PROPERTY_OVERRIDES, + ), + ), + ) + for property_name in properties.keys() + ] + + return json.loads(json.dumps(obj=unit_columns)) + + +def get_unit_table_json(interface) -> List[Dict[str, Any]]: + """ + A convenience function for collecting and organizing the property values of the underlying sorting extractor. + """ + + from neuroconv.utils import NWBMetaDataEncoder + + sorting = interface.sorting_extractor + + properties = get_sorting_interface_properties(interface) + + unit_ids = sorting.get_unit_ids() + + table = list() + for unit_id in unit_ids: + + unit_column = dict() + + for property_name in properties: + if property_name == "unit_name": + sorting_property_value = str(unit_id) # Insert unit_id as name (str) + elif property_name in SORTING_INTERFACE_PROPERTY_OVERRIDES: + try: + sorting_property_value = SORTING_INTERFACE_PROPERTY_OVERRIDES[property_name][ + "default" + ] # Get default value + except: + sorting_property_value = sorting.get_property(key=property_name, ids=[unit_id])[0] + else: + sorting_property_value = sorting.get_property(key=property_name, ids=[unit_id])[ + 0 # First axis is always units in SI + ] # Since only fetching one unit at a time, use trivial zero-index + unit_column.update({property_name: sorting_property_value}) + table.append(unit_column) + table_as_json = json.loads(json.dumps(table, cls=NWBMetaDataEncoder)) + + return table_as_json + + def get_electrode_columns_json(interface) -> List[Dict[str, Any]]: """A convenience function for collecting and organizing the properties of the underlying recording extractor.""" properties = get_recording_interface_properties(interface) @@ -1002,7 +1264,7 @@ def get_electrode_columns_json(interface) -> List[Dict[str, Any]]: offset_to_uV="The offset from the data type to microVolts, applied after the gain.", ) - for property_name, property_info in EXTRA_RECORDING_INTERFACE_PROPERTIES.items(): + for property_name, property_info in RECORDING_INTERFACE_PROPERTY_OVERRIDES.items(): description = property_info.get("description", None) if description: property_descriptions[property_name] = description @@ -1018,7 +1280,10 @@ def get_electrode_columns_json(interface) -> List[Dict[str, Any]]: name=property_name, description=property_descriptions.get(property_name, "No description."), data_type=get_property_dtype( - recording_extractor=recording_extractor, property_name=property_name, channel_ids=[channel_ids[0]] + extractor=recording_extractor, + property_name=property_name, + ids=[channel_ids[0]], + extra_props=RECORDING_INTERFACE_PROPERTY_OVERRIDES, ), ) for property_name in properties.keys() @@ -1058,8 +1323,13 @@ def get_electrode_table_json(interface) -> List[Dict[str, Any]]: for electrode_id in electrode_ids: electrode_column = dict() for property_name in properties: - if property_name in EXTRA_RECORDING_INTERFACE_PROPERTIES: - recording_property_value = properties[property_name]["default"] + if property_name in RECORDING_INTERFACE_PROPERTY_OVERRIDES: + try: + recording_property_value = RECORDING_INTERFACE_PROPERTY_OVERRIDES[property_name][ + "default" + ] # Get default value + except: + recording_property_value = recording.get_property(key=property_name, ids=[electrode_id])[0] else: recording_property_value = recording.get_property(key=property_name, ids=[electrode_id])[ 0 # First axis is always electodes in SI @@ -1108,7 +1378,7 @@ def update_recording_properties_from_table_as_json( for entry_index, entry in enumerate(electrode_table_json): electrode_properties = dict(entry) # copy - channel_name = electrode_properties.pop("channel_name") + channel_name = electrode_properties.pop("channel_name", None) for property_name, property_value in electrode_properties.items(): if property_name not in electrode_column_data_types: # Skip data with missing column information continue @@ -1117,12 +1387,46 @@ def update_recording_properties_from_table_as_json( # property_index = contact_vector_property_names.index(property_name) # modified_contact_vector[entry_index][property_index] = property_value else: + ids = ( + [stream_prefix + "#" + channel_name] if channel_name else [] + ) # Correct for minimal metadata (e.g. CellExplorer) recording_extractor.set_property( key=property_name, values=np.array([property_value], dtype=electrode_column_data_types[property_name]), - ids=[stream_prefix + "#" + channel_name], + ids=ids, ) # TODO: uncomment when neuroconv supports contact vectors (probe interface) # if "contact_vector" in property_names: # recording_extractor.set_property(key="contact_vector", values=modified_contact_vector) + + +def update_sorting_properties_from_table_as_json( + sorting_interface, unit_column_info: dict, unit_table_json: List[Dict[str, Any]] +): + import numpy as np + + unit_column_data_types = {column["name"]: column["data_type"] for column in unit_column_info} + + sorting_extractor = sorting_interface.sorting_extractor + + for entry_index, entry in enumerate(unit_table_json): + unit_properties = dict(entry) # copy + + unit_id = unit_properties.pop("unit_name", None) # NOTE: Is called unit_name in the actual units table + + for property_name, property_value in unit_properties.items(): + + if property_name == "unit_name": + continue # Already controlling unit_id with the above variable + + dtype = unit_column_data_types[property_name] + if property_name in SORTING_INTERFACE_PROPERTIES_TO_RECAST: + property_value = [property_value] + dtype = "object" # Should allow the array to go through + + sorting_extractor.set_property( + key=property_name, + values=np.array([property_value], dtype=dtype), + ids=[int(unit_id)], + ) diff --git a/schemas/base-metadata.schema.ts b/schemas/base-metadata.schema.ts index 1fe054fd3..699a5f44b 100644 --- a/schemas/base-metadata.schema.ts +++ b/schemas/base-metadata.schema.ts @@ -4,7 +4,11 @@ import { header, replaceRefsWithValue } from '../src/renderer/src/stories/forms/ import baseMetadataSchema from './json/base_metadata_schema.json' assert { type: "json" } -const uvMathFormat = `µV`; //`` +import { merge } from '../src/renderer/src/stories/pages/utils' + +const UV_MATH_FORMAT = `µV`; //`` +const UV_PROPERTIES = ["gain_to_uV", "offset_to_uV"] +const COLUMN_SCHEMA_ORDER = ["name", "description", "data_type"] function getSpeciesNameComponents(arr: any[]) { const split = arr[arr.length - 1].split(' - ') @@ -14,8 +18,6 @@ function getSpeciesNameComponents(arr: any[]) { } } - - function getSpeciesInfo(species: any[][] = []) { @@ -37,8 +39,31 @@ function getSpeciesInfo(species: any[][] = []) { } -const propsToInclude = { - ecephys: ["Device", "ElectrodeGroup", "Electrodes", "ElectrodeColumns", "definitions"] +function updateEcephysTable(propName, schema, schemaToMerge) { + + const ecephys = schema.properties.Ecephys + + // Change rendering order for electrode table columns + const electrodesProp = ecephys.properties[propName] + if (!electrodesProp) return false + for (let name in electrodesProp.properties) { + + const itemSchema = electrodesProp.properties[name].items + + // Do not add new items + const updateCopy = structuredClone(schemaToMerge) + const updateProps = updateCopy.properties + for (let itemProp in updateProps) { + if (!itemSchema.properties[itemProp]) delete updateProps[itemProp] + } + + // Merge into existing items + merge(updateCopy, itemSchema) + } + + + return true + } export const preprocessMetadataSchema = (schema: any = baseMetadataSchema, global = false) => { @@ -102,22 +127,27 @@ export const preprocessMetadataSchema = (schema: any = baseMetadataSchema, globa if (ecephys) { - // Change rendering order for electrode table columns - const electrodesProp = ecephys.properties["Electrodes"] - for (let name in electrodesProp.properties) { - const interfaceProps = electrodesProp.properties[name].properties - const electrodeItems = interfaceProps["Electrodes"].items.properties - const uvProperties = ["gain_to_uV", "offset_to_uV"] + ecephys.order = ["Device", "ElectrodeGroup"] + ecephys.properties.Device.title = 'Devices' + ecephys.properties.ElectrodeGroup.title = 'Electrode Groups' - uvProperties.forEach(prop => { - electrodeItems[prop] = {} - electrodeItems[prop].title = prop.replace('uV', uvMathFormat) - }) + if (ecephys.properties.ElectrodeColumns) ecephys.properties.ElectrodeColumns.order = COLUMN_SCHEMA_ORDER + if (ecephys.properties.UnitProperties) ecephys.properties.UnitProperties.order = COLUMN_SCHEMA_ORDER - interfaceProps["Electrodes"].items.order = ["channel_name", "group_name", "shank_electrode_number", ...uvProperties]; - interfaceProps["ElectrodeColumns"].items.order = ["name", "description", "data_type"]; - } + updateEcephysTable("Electrodes", copy, { + properties: UV_PROPERTIES.reduce((acc, prop) => { + acc[prop] = { title: prop.replace('uV', UV_MATH_FORMAT) } + return acc + }, {}), + order: ["channel_name", "group_name", "shank_electrode_number", ...UV_PROPERTIES] + }) + + // ecephys.properties["Units"].title = "Unit Summaries" + + updateEcephysTable("Units", copy, { + order: ["unit_name", "clu_id", "group_id"] + }) } diff --git a/src/renderer/src/stories/BasicTable.js b/src/renderer/src/stories/BasicTable.js index ea5e6ee30..993e99a81 100644 --- a/src/renderer/src/stories/BasicTable.js +++ b/src/renderer/src/stories/BasicTable.js @@ -130,13 +130,14 @@ export class BasicTable extends LitElement { onStatusChange, onLoaded, onUpdate, + editable = true, } = {}) { super(); this.name = name ?? "data_table"; this.schema = schema ?? {}; this.data = data ?? []; this.keyColumn = keyColumn; - this.maxHeight = maxHeight ?? ""; + this.maxHeight = maxHeight ?? "unset"; this.validateEmptyCells = validateEmptyCells ?? true; this.ignore = ignore ?? {}; @@ -145,6 +146,8 @@ export class BasicTable extends LitElement { if (onUpdate) this.onUpdate = onUpdate; if (onStatusChange) this.onStatusChange = onStatusChange; if (onLoaded) this.onLoaded = onLoaded; + + this.editable = editable; } #schema = {}; @@ -280,11 +283,13 @@ export class BasicTable extends LitElement { let { type, original, inferred } = this.#getType(value, propInfo); + const isUndefined = value === undefined || value === ""; + // Check if required - if (!value && "required" in this.#itemSchema && this.#itemSchema.required.includes(col)) + if (isUndefined && "required" in this.#itemSchema && this.#itemSchema.required.includes(col)) result = [{ message: `${col} is a required property`, type: "error" }]; // If not required, check matching types (if provided) for values that are defined - else if (value !== "" && type && inferred !== type) + else if (!isUndefined && type && inferred !== type) result = [{ message: `${col} is expected to be of type ${original}, not ${inferred}`, type: "error" }]; // Otherwise validate using the specified onChange function else result = this.validateOnChange([row, col], parent, value, this.#itemProps[col]); @@ -377,6 +382,7 @@ export class BasicTable extends LitElement { }; #readTSV(text) { + console.log(text, text.split("\n")); let data = text.split("\n").map((row) => row.split("\t").map((v) => { try { @@ -397,18 +403,24 @@ export class BasicTable extends LitElement { ); Object.keys(this.data).forEach((row) => delete this.data[row]); // Delete all previous rows + Object.keys(data).forEach((row) => { const cols = structuredData[row]; const latest = (this.data[this.keyColumn ? cols[this.keyColumn] : row] = {}); Object.entries(cols).forEach(([key, value]) => { - if (key in this.#itemProps) { - const { type } = this.#getType(value, this.#itemProps[key]); - if (type === "string") value = `${value}`; // Convert to string if necessary - latest[key] = value; + // if (key in this.#itemProps) { + const { type } = this.#getType(value, this.#itemProps[key]); + if (type === "string") { + if (value === undefined) value = ""; + else value = `${value}`; // Convert to string if necessary } + latest[key] = value; + // } }); // Only include data from schema }); + console.log(header, data, structuredData, this.data, this.#itemProps); + if (this.onUpdate) this.onUpdate([], data); // Update the whole table } @@ -459,6 +471,8 @@ export class BasicTable extends LitElement { const data = (this.#data = this.#getData()); + const description = this.#schema.description; + return html`
+ ${description} +
` + : ""} `; } } diff --git a/src/renderer/src/stories/JSONSchemaForm.js b/src/renderer/src/stories/JSONSchemaForm.js index 50c37818d..9d4695b80 100644 --- a/src/renderer/src/stories/JSONSchemaForm.js +++ b/src/renderer/src/stories/JSONSchemaForm.js @@ -57,7 +57,6 @@ export const getSchema = (path, schema, base = []) => { // NOTE: Refs are now pre-resolved const resolved = get(path, schema, ["properties", "patternProperties"], ["patternProperties", "items"]); - // if (resolved?.["$ref"]) return this.getSchema(resolved["$ref"].split("/").slice(1)); // NOTE: This assumes reference to the root of the schema return resolved; }; @@ -81,9 +80,15 @@ export const getIgnore = (o, path) => { return path.reduce((acc, key) => { const info = acc[key] ?? {}; + const accWildcard = acc["*"] ?? {}; + const infoWildcard = info["*"] ?? {}; + const mergedWildcards = { ...accWildcard, ...infoWildcard }; + + if (key in mergedWildcards) return { ...info, ...mergedWildcards[key] }; + return { ...info, - "*": { ...(acc["*"] ?? {}), ...(info["*"] ?? {}) }, // Accumulate ignore values + "*": mergedWildcards, // Accumulate ignore values }; }, o); }; @@ -565,6 +570,7 @@ export class JSONSchemaForm extends LitElement { const allErrors = Array.from(flaggedInputs) .map((inputElement) => { + if (!inputElement.nextElementSibling) return; // Skip tables return Array.from(inputElement.nextElementSibling.children).map((li) => li.message); }) .flat(); @@ -778,9 +784,12 @@ export class JSONSchemaForm extends LitElement { const res = entries .map(([key, value]) => { if (!value.properties && key === "definitions") return false; // Skip definitions - if (this.ignore["*"]?.[key]) + + // If conclusively ignored + if (this.ignore["*"]?.[key] === true) return false; // Skip all properties with this name else if (this.ignore[key] === true) return false; // Skip this property + if (this.showLevelOverride >= path.length) return isRenderable(key, value); if (required[key]) return isRenderable(key, value); if (this.#getLink([...this.base, ...path, key])) return isRenderable(key, value); @@ -1261,9 +1270,12 @@ export class JSONSchemaForm extends LitElement { enableToggleContainer.append(enableToggle); Object.assign(enableToggle.style, { marginRight: "10px", pointerEvents: "all" }); + // Skip if accordion will be empty + if (!renderableInside.length) return; + const accordion = (this.accordions[name] = new Accordion({ name: headerName, - toggleable: hasMany, + toggleable: hasMany, // Only show toggle if there are multiple siblings subtitle: html`- ${unsafeHTML(capitalize(schema.description))}${[".", "?", "!"].includes( - schema.description.slice(-1)[0] - ) + ${unsafeHTML(capitalize(description))}${[".", "?", "!"].includes(description.slice(-1)[0]) ? "" : "."}
` diff --git a/src/renderer/src/stories/SimpleTable.js b/src/renderer/src/stories/SimpleTable.js index 18487401f..2fd49fd85 100644 --- a/src/renderer/src/stories/SimpleTable.js +++ b/src/renderer/src/stories/SimpleTable.js @@ -11,7 +11,7 @@ import { styleMap } from "lit/directives/style-map.js"; import "./Button"; import tippy from "tippy.js"; -import { sortTable } from "./Table"; +import { sortTable, getEditable } from "./Table"; import { NestedInputCell } from "./table/cells/input"; import { getIgnore } from "./JSONSchemaForm"; @@ -68,6 +68,10 @@ export class SimpleTable extends LitElement { border: none; } + td[editable="false"] { + background: whitesmoke; + } + :host([loading]:not([waiting])) table { height: 250px; } @@ -201,6 +205,7 @@ export class SimpleTable extends LitElement { maxHeight, contextOptions = {}, ignore = {}, + editable = {}, } = {}) { super(); this.schema = schema ?? {}; @@ -213,6 +218,7 @@ export class SimpleTable extends LitElement { this.maxHeight = maxHeight ?? ""; this.ignore = ignore; + this.editable = editable; this.contextOptions = contextOptions; @@ -488,10 +494,11 @@ export class SimpleTable extends LitElement { id: "add-row", label: "Add Row", onclick: (path) => { - const cell = this.#getCellFromPath(path); - if (!cell) return this.addRow(); // No cell selected - const { i } = cell.simpleTableInfo; - this.addRow(i); //2) // TODO: Support adding more than one row + // const cell = this.#getCellFromPath(path); + // if (!cell) return this.addRow(); // No cell selected + // const { i } = cell.simpleTableInfo; + const lastRow = this.#cells.length - 1; + this.addRow(lastRow); // Just insert row at the end }, }, remove: { @@ -558,6 +565,41 @@ export class SimpleTable extends LitElement { this.#context = new ContextMenu({ target: this.shadowRoot.querySelector("table"), items, + onOpen: (path) => { + const checks = { + row_remove: { + check: this.editable.__row_remove, + element: this.#context.shadowRoot.querySelector("#remove-row"), + }, + + row_add: { + check: this.editable.__row_add, + element: this.#context.shadowRoot.querySelector("#add-row"), + }, + }; + + const hasChecks = Object.values(checks).some(({ check }) => check); + + if (hasChecks) { + const cell = this.#getCellFromPath(path); + const info = cell.simpleTableInfo; + const rowNames = Object.keys(this.#data); + const row = Array.isArray(this.#data) ? info.i : rowNames[info.i]; + + const results = Object.values(checks).map(({ check, element }) => { + if (check) { + const canRemove = check(cell.value, this.#data[row]); + if (canRemove) element.removeAttribute("disabled"); + else element.setAttribute("disabled", ""); + return canRemove; + } else return true; + }); + + return !results.every((r) => r === false); // If all are hidden, don't show the context menu + } + + return true; + }, }); this.#context.updated = () => this.#updateContextMenuRendering(); // Run when done rendering @@ -780,6 +822,8 @@ export class SimpleTable extends LitElement { const schema = this.#itemProps[fullInfo.col]; const ignore = getIgnore(this.ignore, [fullInfo.col]); + const rowData = this.#data[row]; + const isEditable = getEditable(value, rowData, this.editable, fullInfo.col); // Track the cell renderer const cell = new TableCell({ @@ -791,6 +835,7 @@ export class SimpleTable extends LitElement { ), col: this.colHeaders[info.j], }, + editable: isEditable, value, schema, ignore, @@ -846,9 +891,13 @@ export class SimpleTable extends LitElement { #renderCell = (value, info) => { const td = document.createElement("td"); + const cell = value instanceof TableCell ? value : this.#createCell(value, info); cell.simpleTableInfo.td = td; + + td.setAttribute("editable", cell.editable); + td.onmouseover = () => { if (this.#selecting) this.#selectCells(cell); }; diff --git a/src/renderer/src/stories/Table.js b/src/renderer/src/stories/Table.js index 4f391bff2..e1a7e1713 100644 --- a/src/renderer/src/stories/Table.js +++ b/src/renderer/src/stories/Table.js @@ -16,6 +16,12 @@ const isRequired = (col, schema) => { return schema.required?.includes(col); }; +export const getEditable = (value, rowData = {}, config, colName) => { + if (typeof config === "boolean") return config; + if (typeof config === "function") return config(value, rowData); + return getEditable(value, rowData, config?.[colName] ?? true); +}; + export function sortTable(schema, keyColumn, order) { const cols = Object.keys(schema.properties) @@ -378,7 +384,9 @@ export class Table extends LitElement { return; } - if (!value && required) { + const isUndefined = value == ""; + + if (isUndefined && required) { instanceThis.#handleValidationResult( [{ message: `${header(k)} is a required property.`, type: "error" }], row, diff --git a/src/renderer/src/stories/forms/utils.ts b/src/renderer/src/stories/forms/utils.ts index 529a39b95..43d09296e 100644 --- a/src/renderer/src/stories/forms/utils.ts +++ b/src/renderer/src/stories/forms/utils.ts @@ -1,6 +1,9 @@ +import { merge } from '../pages/utils' + const toCapitalizeAll = ['nwb', 'api', 'id'] const toCapitalizeNone = ['or', 'and'] + export const createRandomString = () => Math.random().toString(36).substring(7); export const tempPropertyKey = createRandomString(); export const tempPropertyValueKey = createRandomString(); @@ -33,7 +36,15 @@ export const textToArray = (value: string) => value.split("\n") if (prop && typeof prop === "object" && !Array.isArray(prop)) { const internalCopy = (copy[propName] = { ...prop }); const refValue = internalCopy["$ref"] - if (refValue) { + const allOfValue = internalCopy['allOf'] + if (allOfValue) { + copy [propName]= allOfValue.reduce((acc, curr) => { + const result = replaceRefsWithValue({ _temp: curr}, path, parent) + const resolved = result._temp + return merge(resolved, acc) + }, {}) + } + else if (refValue) { const refPath = refValue.split('/').slice(1) // NOTE: Assume from base const resolved = refPath.reduce((acc, key) => acc[key], parent) diff --git a/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js b/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js index 4b5975e8d..1e35cf2b0 100644 --- a/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js +++ b/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js @@ -1,4 +1,4 @@ -import { JSONSchemaForm } from "../../../JSONSchemaForm.js"; +import { JSONSchemaForm, getSchema } from "../../../JSONSchemaForm.js"; import { InstanceManager } from "../../../InstanceManager.js"; import { ManagedPage } from "./ManagedPage.js"; @@ -25,6 +25,59 @@ import { Button } from "../../../Button.js"; import globalIcon from "../../../assets/global.svg?raw"; +const parentTableRenderConfig = { + Electrodes: (metadata) => { + metadata.schema.description = "Download, modify, and re-upload data to change the electrode information."; + return true; + }, + Units: (metadata) => { + metadata.editable = false; + metadata.schema.description = "Update unit information directly on your source data."; + return true; + }, +}; + +function getAggregateRequirements(path) { + const electrodeSchema = getSchema(path, this.schema); + return Object.values(electrodeSchema.properties).reduce((set, schema) => { + schema.items.required.forEach((item) => set.add(item)); + return set; + }, new Set()); +} + +const tableRenderConfig = { + "*": (metadata) => new SimpleTable(metadata), + ElectrodeColumns: function (metadata) { + const aggregateRequirements = getAggregateRequirements.call(this, ["Ecephys", "Electrodes"]); + + return new SimpleTable({ + ...metadata, + editable: { + name: (value) => !aggregateRequirements.has(value), + data_type: (_, row) => !aggregateRequirements.has(row.name), + __row_remove: (_, row) => !aggregateRequirements.has(row.name), + }, + }); + }, + UnitColumns: function (metadata) { + const aggregateRequirements = getAggregateRequirements.call(this, ["Ecephys", "Units"]); + + return new SimpleTable({ + ...metadata, + contextOptions: { + row: { + add: false, + remove: false, + }, + }, + editable: { + name: (value) => !aggregateRequirements.has(value), + data_type: (_, row) => !aggregateRequirements.has(row.name), + }, + }); + }, +}; + const imagingPlaneKey = "imaging_plane"; const propsToIgnore = { Ophys: { @@ -55,14 +108,16 @@ const propsToIgnore = { }, }, Ecephys: { - UnitProperties: true, + ElectricalSeries: true, ElectricalSeriesLF: true, ElectricalSeriesAP: true, - Electrodes: { + Units: { "*": { - location: true, - group: true, - contact_vector: true, + UnitColumns: { + "*": { + data_type: true, // Do not show data_type + }, + }, }, }, }, @@ -269,6 +324,8 @@ export class GuidedMetadataPage extends ManagedPage { renderCustomHTML: function (name, inputSchema, localPath, { onUpdate, onThrow }) { if (name === "TwoPhotonSeries" && (!this.value || !this.value.length)) return null; + if (name === "Device" && (!this.value || !this.value.length)) return null; + if (name === "ElectrodeGroup" && (!this.value || !this.value.length)) return null; const isAdditional = isAdditionalProperties(this.pattern); const isPattern = isPatternProperties(this.pattern); @@ -406,10 +463,13 @@ export class GuidedMetadataPage extends ManagedPage { const updatedSchema = structuredClone(metadata.schema); metadata.schema = updatedSchema; - // NOTE: Handsontable will occasionally have a context menu that doesn't actually trigger any behaviors - if (name !== "Electrodes") return new SimpleTable(metadata); - else return true; // All other tables are handled by the default behavior - // if (name !== "ElectrodeColumns" && name !== "Electrodes") return new Table(metadata); + const parentName = fullPath[fullPath.length - 1]; + + const tableConfig = + tableRenderConfig[name] ?? parentTableRenderConfig[parentName] ?? tableRenderConfig["*"] ?? true; + if (typeof tableConfig === "function") + return tableConfig.call(form, metadata, [...fullPath, name], this); + else return tableConfig; }, onThrow, }); diff --git a/src/renderer/src/stories/pages/guided-mode/data/GuidedSourceData.js b/src/renderer/src/stories/pages/guided-mode/data/GuidedSourceData.js index 438520b7b..12f5a3818 100644 --- a/src/renderer/src/stories/pages/guided-mode/data/GuidedSourceData.js +++ b/src/renderer/src/stories/pages/guided-mode/data/GuidedSourceData.js @@ -120,7 +120,9 @@ export class GuidedSourceDataPage extends ManagedPage { const [type, ...splitText] = result.message.split(":"); const text = splitText.length ? splitText.join(":").replaceAll("<", "<").replaceAll(">", ">") - : `${result.traceback.trim().split("\n").slice(-2)[0].trim()}`; + : result.traceback + ? `
${result.traceback.trim().split("\n").slice(-2)[0].trim()}` + : ""; const message = `
${text}
`; this.notify(message, "error"); diff --git a/src/renderer/src/stories/table/Cell.ts b/src/renderer/src/stories/table/Cell.ts index 1cda87fa5..3027f6a5b 100644 --- a/src/renderer/src/stories/table/Cell.ts +++ b/src/renderer/src/stories/table/Cell.ts @@ -4,7 +4,7 @@ import { NestedInputCell } from "./cells/input" import { TableCellBase } from "./cells/base" import { DateTimeCell } from "./cells/date-time" - +import { DropdownCell } from "./cells/dropdown" import { getValue, renderValue } from './convert' @@ -20,6 +20,7 @@ type OnValidateFunction = (info: ValidationResult) => void type TableCellProps = { value: string, + editable: boolean, info: { col: string } ignore: { [key: string]: boolean }, schema: {[key: string]: any}, @@ -33,6 +34,7 @@ export class TableCell extends LitElement { declare schema: TableCellProps['schema'] declare info: TableCellProps['info'] + declare editable: TableCellProps['editable'] static get styles() { return css` @@ -69,20 +71,21 @@ export class TableCell extends LitElement { ` } - // static get properties() { - // return { - // value: { reflect: true } - // } - // } + static get properties() { + return { + editable: { reflect: true } + } + } type = 'text' - constructor({ info, value, schema, validateOnChange, ignore, onValidate }: TableCellProps) { + constructor({ info, value, editable = true, schema, validateOnChange, ignore, onValidate }: TableCellProps) { super() this.#value = value this.schema = schema this.info = info + this.editable = editable if (validateOnChange) this.validateOnChange = validateOnChange if (ignore) this.ignore = ignore @@ -102,7 +105,9 @@ export class TableCell extends LitElement { } - toggle = (v: boolean) => this.input.toggle(v) + toggle = (v: boolean) => { + if (this.editable) this.input.toggle(v) + } get value() { let v = this.input ? this.input.getValue() : this.#value @@ -110,6 +115,10 @@ export class TableCell extends LitElement { } set value(value) { + + + if (!this.editable && this.interacted === true) return // Don't set value if not editable + if (this.input) this.input.set(renderValue(value, this.schema)) // Allow null to be set directly this.#value = this.input ? this.input.getValue() // Ensure all operations are undoable / value is coerced @@ -168,6 +177,8 @@ export class TableCell extends LitElement { this.interacted = persistentInteraction // this.value = value + if (!this.editable) return // Don't set value if not editable + if (this.input) this.input.set(value) // Ensure all operations are undoable else this.#value = value // Silently set value if not rendered yet } @@ -213,9 +224,15 @@ export class TableCell extends LitElement { this.type = "table" } + else if (this.schema.enum) { + cls = DropdownCell + this.type = "dropdown" + } + // Only actually rerender if new class type if (cls !== this.#cls) { this.input = new cls({ + editable: this.editable, onChange: async (v) => { if (this.input.interacted) this.interacted = true const result = await this.validate() diff --git a/src/renderer/src/stories/table/ContextMenu.ts b/src/renderer/src/stories/table/ContextMenu.ts index 4f382b78b..997662b91 100644 --- a/src/renderer/src/stories/table/ContextMenu.ts +++ b/src/renderer/src/stories/table/ContextMenu.ts @@ -66,12 +66,14 @@ export class ContextMenu extends LitElement{ declare target: Document | HTMLElement declare items: any[] + declare onOpen: () => boolean | void - constructor({ target, items }: any){ + constructor({ target, items, onOpen }: any){ super() this.target = target ?? document this.items = items ?? [] + this.onOpen = onOpen ?? (() => {}) document.addEventListener('click', () => this.#hide()) // Hide at the last step of any click document.addEventListener('contextmenu', () => this.#hide()) @@ -88,7 +90,12 @@ export class ContextMenu extends LitElement{ #open(mouseEvent: MouseEvent) { mouseEvent.preventDefault() mouseEvent.stopPropagation() + this.#activePath = mouseEvent.path || mouseEvent.composedPath() + + const result = this.onOpen(this.#activePath) + if (result === false) return + this.style.display = 'block'; this.style.left = mouseEvent.pageX + "px"; this.style.top = mouseEvent.pageY + "px"; diff --git a/src/renderer/src/stories/table/cells/base.ts b/src/renderer/src/stories/table/cells/base.ts index 1dca267cb..d4f3424be 100644 --- a/src/renderer/src/stories/table/cells/base.ts +++ b/src/renderer/src/stories/table/cells/base.ts @@ -5,6 +5,7 @@ type BaseTableProps = { info: { col: string, }, + editable: boolean, toggle: (state?: boolean) => void, schema: any, onOpen: Function, @@ -31,6 +32,9 @@ export class TableCellBase extends LitElement { #editor?: HTMLElement #renderer?: HTMLElement + // Internal variables + #firstUpdated = false + #initialValue: undefined | any static get styles() { return css` @@ -64,6 +68,8 @@ export class TableCellBase extends LitElement { info: BaseTableProps['info']; editToggle: BaseTableProps['toggle'] + editable: BaseTableProps['editable']; + interacted = false constructor({ @@ -73,14 +79,14 @@ export class TableCellBase extends LitElement { onClose, onChange, toggle, - nestedProps + nestedProps, + editable = true }: Partial