Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Render Units Table #679

Merged
merged 46 commits into from
Mar 29, 2024
Merged
Changes from 1 commit
Commits
Show all changes
46 commits
Select commit Hold shift + click to select a range
c9f5ab4
Update with relevant demo
garrettmflynn Mar 14, 2024
8b12e28
Swap suppressed keys
garrettmflynn Mar 14, 2024
b2e8ed2
Show units. No editing allowed
garrettmflynn Mar 15, 2024
6854175
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Mar 15, 2024
0724d92
Fix backend and hide empty tables
garrettmflynn Mar 15, 2024
e3aa813
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Mar 15, 2024
512b55e
Skip accordion if empty
garrettmflynn Mar 15, 2024
a4d15a3
Merge branch 'units-table' of https://github.com/NeurodataWithoutBord…
garrettmflynn Mar 15, 2024
a26ec04
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Mar 15, 2024
a5640f1
Ensure original columns are required and cannot be edited
garrettmflynn Mar 15, 2024
5c62e65
Merge branch 'units-table' of https://github.com/NeurodataWithoutBord…
garrettmflynn Mar 15, 2024
f0acf96
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Mar 15, 2024
5b41630
Merge branch 'main' into units-table
CodyCBakerPhD Mar 15, 2024
4046a92
Merge branch 'main' into units-table
CodyCBakerPhD Mar 18, 2024
b4075e2
Show unit name and fix issues from review
garrettmflynn Mar 18, 2024
9608a93
Merge branch 'units-table' of https://github.com/NeurodataWithoutBord…
garrettmflynn Mar 18, 2024
06c3f18
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Mar 18, 2024
186b2ce
Merge branch 'main' into units-table
CodyCBakerPhD Mar 18, 2024
e966a28
Merge branch 'main' into units-table
CodyCBakerPhD Mar 20, 2024
93c9d57
Update ElectrodeColumns and UnitColumns to be global Ecephys properties
garrettmflynn Mar 21, 2024
57283b3
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Mar 21, 2024
bfad2d7
Add new validation to units table as well
garrettmflynn Mar 21, 2024
c2e392c
Merge branch 'units-table' of https://github.com/NeurodataWithoutBord…
garrettmflynn Mar 21, 2024
91e0c15
Fix Ecephys property order
garrettmflynn Mar 23, 2024
3f35d28
Fix invalid table elements
garrettmflynn Mar 23, 2024
455f3fe
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Mar 23, 2024
205a3f7
Allow editing descriptions
garrettmflynn Mar 23, 2024
2fbe64a
Merge branch 'units-table' of https://github.com/NeurodataWithoutBord…
garrettmflynn Mar 23, 2024
53346ce
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Mar 23, 2024
37464b2
Fix unit_id type issue
garrettmflynn Mar 23, 2024
00fa24c
Merge branch 'main' into units-table
garrettmflynn Mar 23, 2024
79d2deb
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Mar 23, 2024
3d2f704
Update pyflask/manageNeuroconv/manage_neuroconv.py
CodyCBakerPhD Mar 23, 2024
92a0bd9
Merge branch 'main' into units-table
CodyCBakerPhD Mar 25, 2024
b7379b1
Handle other array-to-string properties for Units
garrettmflynn Mar 25, 2024
68c4c77
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Mar 25, 2024
4edca74
Merge branch 'main' into units-table
CodyCBakerPhD Mar 25, 2024
5d6e337
Update to unit_name
garrettmflynn Mar 25, 2024
01521cd
First round of updates from GUIDE meeting
garrettmflynn Mar 27, 2024
768b259
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Mar 27, 2024
445b6f7
Only allow removing non-required rows
garrettmflynn Mar 27, 2024
bd0e12a
Merge branch 'main' into units-table
garrettmflynn Mar 28, 2024
ebedb05
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Mar 28, 2024
114194b
Merge branch 'main' into units-table
garrettmflynn Mar 28, 2024
7c3f044
Fix tests
garrettmflynn Mar 28, 2024
e28e6b2
Merge branch 'main' into units-table
CodyCBakerPhD Mar 29, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Show unit name and fix issues from review
  • Loading branch information
garrettmflynn committed Mar 18, 2024
commit b4075e2f50cb38e705563167bac2da9f31c5b753
89 changes: 71 additions & 18 deletions pyflask/manageNeuroconv/manage_neuroconv.py
Original file line number Diff line number Diff line change
@@ -20,14 +20,32 @@


EXCLUDED_RECORDING_INTERFACE_PROPERTIES = ["contact_vector", "contact_shapes", "group", "location"]
EXTRA_RECORDING_INTERFACE_PROPERTIES = EXTRA_SORTING_INTERFACE_PROPERTIES = {
"brain_area": {

EXTRA_INTERFACE_PROPERTIES = {
"brain_area": {
"data_type": "str",
"description": "The brain area where the electrode is located.",
"default": "unknown",
}
}

EXTRA_RECORDING_INTERFACE_PROPERTIES = {
"brain_area": {
"description": "The brain area where the electrode is located.",
**EXTRA_INTERFACE_PROPERTIES["brain_area"]
}
}

EXTRA_SORTING_INTERFACE_PROPERTIES = {
"name": {
"description": "The unique name for the unit",
"data_type": "string"
},
"brain_area": {
"description": "The brain area where the unit is located.",
**EXTRA_INTERFACE_PROPERTIES["brain_area"]
}
}

EXCLUDED_SORTING_INTERFACE_PROPERTIES = ["location", "spike_times", "electrodes"] # Not validated

DTYPE_DESCRIPTIONS = {
@@ -668,9 +686,20 @@ def update_conversion_progress(**kwargs):

# Quick fix to remove units
has_units = "Units" in ecephys_metadata

if has_units:
for interface_name, interface_unit_results in ecephys_metadata["Units"].items():
interface = converter.data_interface_objects[interface_name]

update_sorting_properties_from_table_as_json(
interface,
unit_table_json=interface_unit_results["Units"],
unit_column_info=interface_unit_results["UnitColumns"],
)

del ecephys_metadata["Units"]


for interface_name, interface_electrode_results in ecephys_metadata["Electrodes"].items():
interface = converter.data_interface_objects[interface_name]

@@ -1056,11 +1085,11 @@ def map_dtype(dtype: str) -> str:
return dtype


def get_property_dtype(recording_extractor, property_name: str, channel_ids: list):
if property_name in EXTRA_RECORDING_INTERFACE_PROPERTIES:
dtype = EXTRA_RECORDING_INTERFACE_PROPERTIES[property_name]["data_type"]
def get_property_dtype(extractor, property_name: str, ids: list, extra_props: dict):
if property_name in extra_props:
dtype = extra_props[property_name]["data_type"]
else:
dtype = str(recording_extractor.get_property(key=property_name, ids=channel_ids).dtype)
dtype = str(extractor.get_property(key=property_name, ids=ids).dtype)

# return type(recording.get_property(key=property_name)[0]).__name__.replace("_", "")
# return dtype
@@ -1106,14 +1135,9 @@ def get_unit_columns_json(interface) -> List[Dict[str, Any]]:
"""A convenience function for collecting and organizing the properties of the underlying sorting extractor."""
properties = get_sorting_interface_properties(interface)

# Hardcoded for Phy (NOTE: Update for more interfaces)
property_descriptions = dict(
# spike_times="The times of the spikes, in seconds.",
# spike_clusters="The cluster IDs of the spikes.",
# spike_templates="The template IDs of the spikes.",
# spike_amplitudes="The amplitudes of the spikes.",
# spike_depths="The depths of the spikes.",
# spike_widths="The widths of the spikes.",
clu_id="The cluster ID for the unit",
group_id="The group ID for the unit"
)

for property_name, property_info in EXTRA_SORTING_INTERFACE_PROPERTIES.items():
@@ -1129,7 +1153,7 @@ def get_unit_columns_json(interface) -> List[Dict[str, Any]]:
name=property_name,
description=property_descriptions.get(property_name, "No description."),
data_type=get_property_dtype(
recording_extractor=sorting_extractor, property_name=property_name, channel_ids=[unit_ids[0]]
extractor=sorting_extractor, property_name=property_name, ids=[unit_ids[0]], extra_props=EXTRA_SORTING_INTERFACE_PROPERTIES
),
)
for property_name in properties.keys()
@@ -1153,10 +1177,14 @@ def get_unit_table_json(interface) -> List[Dict[str, Any]]:

table = list()
for unit_id in unit_ids:

unit_column = dict()

for property_name in properties:
if property_name in EXTRA_SORTING_INTERFACE_PROPERTIES:
sorting_property_value = properties[property_name]["default"]
if property_name is 'name':
sorting_property_value = unit_id
elif property_name in EXTRA_SORTING_INTERFACE_PROPERTIES:
sorting_property_value = properties[property_name].get('default')
else:
sorting_property_value = sorting.get_property(key=property_name, ids=[unit_id])[
0 # First axis is always units in SI
@@ -1199,7 +1227,7 @@ def get_electrode_columns_json(interface) -> List[Dict[str, Any]]:
name=property_name,
description=property_descriptions.get(property_name, "No description."),
data_type=get_property_dtype(
recording_extractor=recording_extractor, property_name=property_name, channel_ids=[channel_ids[0]]
extractor=recording_extractor, property_name=property_name, ids=[channel_ids[0]], extra_props=EXTRA_RECORDING_INTERFACE_PROPERTIES
),
)
for property_name in properties.keys()
@@ -1313,3 +1341,28 @@ def update_recording_properties_from_table_as_json(
# TODO: uncomment when neuroconv supports contact vectors (probe interface)
# if "contact_vector" in property_names:
# recording_extractor.set_property(key="contact_vector", values=modified_contact_vector)


def update_sorting_properties_from_table_as_json(
sorting_interface, unit_column_info: dict, unit_table_json: List[Dict[str, Any]]
):
import numpy as np
unit_column_data_types = {column["name"]: column["data_type"] for column in unit_column_info}
unit_column_descriptions = {column["name"]: column["description"] for column in unit_column_info}
sorting_extractor = sorting_interface.sorting_extractor

for entry_index, entry in enumerate(unit_table_json):
unit_properties = dict(entry) # copy

unit_id = unit_properties.pop("name", None) # NOTE: Is called unit_name in the actual units table

for property_name, property_value in unit_properties.items():

if (property_name is "name"):
continue

sorting_extractor.set_property(
key=property_name,
values=np.array([property_value], dtype=unit_column_data_types[property_name]), #, description=unit_column_descriptions[property_name]),
ids=[unit_id]
)
2 changes: 1 addition & 1 deletion schemas/base-metadata.schema.ts
Original file line number Diff line number Diff line change
@@ -145,7 +145,7 @@ export const preprocessMetadataSchema = (schema: any = baseMetadataSchema, globa

updateEcephysTable("Units", copy, {
Units: {
order: ["clu_id", "group_id"]
order: ["name", "clu_id", "group_id"]
},
UnitColumns: {
order: COLUMN_SCHEMA_ORDER
15 changes: 12 additions & 3 deletions src/renderer/src/stories/JSONSchemaForm.js
Original file line number Diff line number Diff line change
@@ -80,9 +80,15 @@ export const getIgnore = (o, path) => {
return path.reduce((acc, key) => {
const info = acc[key] ?? {};

const accWildcard = acc["*"] ?? {}
const infoWildcard = info["*"] ?? {}
const mergedWildcards = { ...accWildcard, ...infoWildcard }

if (key in mergedWildcards) return {...info, ...mergedWildcards[key]}

return {
...info,
"*": { ...(acc["*"] ?? {}), ...(info["*"] ?? {}) }, // Accumulate ignore values
"*": mergedWildcards, // Accumulate ignore values
};
}, o);
};
@@ -768,12 +774,15 @@ export class JSONSchemaForm extends LitElement {
else return [key, value];
};


const res = entries
.map(([key, value]) => {
if (!value.properties && key === "definitions") return false; // Skip definitions
if (this.ignore["*"]?.[key])
return false; // Skip all properties with this name

// If conclusively ignored
if (this.ignore["*"]?.[key] === true) return false; // Skip all properties with this name
else if (this.ignore[key] === true) return false; // Skip this property

if (this.showLevelOverride >= path.length) return isRenderable(key, value);
if (required[key]) return isRenderable(key, value);
if (this.#getLink([...this.base, ...path, key])) return isRenderable(key, value);
3 changes: 2 additions & 1 deletion src/renderer/src/stories/JSONSchemaInput.js
Original file line number Diff line number Diff line change
@@ -174,6 +174,8 @@ export function createTable(fullPath, { onUpdate, onThrow, overrides = {} }) {

merge(overrides.schema, schemaCopy, { arrays: true });

console.log(schemaPath, nestedIgnore)

const tableMetadata = {
keyColumn: tempPropertyKey,
schema: schemaCopy,
@@ -254,7 +256,6 @@ export function createTable(fullPath, { onUpdate, onThrow, overrides = {} }) {
}

const nestedIgnore = getIgnore(ignore, fullPath);

Object.assign(nestedIgnore, overrides.ignore ?? {});

merge(overrides.ignore, nestedIgnore);
27 changes: 17 additions & 10 deletions src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js
Original file line number Diff line number Diff line change
@@ -41,9 +41,14 @@ const tableRenderConfig = {
metadata.editable = false;
return true;
},
UnitColumns: (metadata) => {
metadata.editable = false;
return true;
UnitColumns: function (metadata, fullPath) {
const unitSchema = getSchema([...fullPath.slice(0, -1), "Units"], this.schema);
return new SimpleTable({
...metadata,
editable: {
name: (value) => !unitSchema.items.required.includes(value),
},
});
},
};

@@ -80,13 +85,15 @@ const propsToIgnore = {
ElectricalSeries: true,
ElectricalSeriesLF: true,
ElectricalSeriesAP: true,
Electrodes: {
"*": {
location: true,
group: true,
contact_vector: true,
},
},
Units: {
'*': {
UnitColumns: {
'*': {
data_type: true // Do not show data_type
}
}
}
}
},
Icephys: true, // Always ignore icephys metadata (for now)
Behavior: true, // Always ignore behavior metadata (for now)
Original file line number Diff line number Diff line change
@@ -222,10 +222,6 @@ export class GuidedSourceDataPage extends ManagedPage {
updated() {
const dashboard = document.querySelector("nwb-dashboard");
const page = dashboard.page;
setTimeout(() => {
console.log(page.forms[0].form.accordions["SpikeGLX Recording"]);
});
console.log(page.forms[0].form.accordions);
}

render() {
Loading