From 6759250d6a10df969424cab20d4fb96d39ceafa2 Mon Sep 17 00:00:00 2001 From: Garrett Date: Mon, 25 Sep 2023 17:30:20 -0700 Subject: [PATCH 1/4] Python-only fix for expecting NaN in JSON --- pyflask/manageNeuroconv/manage_neuroconv.py | 83 ++++++++++++++++++- .../pages/guided-mode/data/GuidedMetadata.js | 7 +- src/renderer/src/validation/validation.json | 4 + 3 files changed, 91 insertions(+), 3 deletions(-) diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index 1aaf6e1ce..089e7edcf 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -1,6 +1,7 @@ """Collection of utility functions used by the NeuroConv Flask API.""" import os import json +import math from datetime import datetime from typing import Dict, Optional # , List, Union # TODO: figure way to add these back in without importing other class from shutil import rmtree, copytree @@ -11,6 +12,82 @@ announcer = MessageAnnouncer() +def replace_nan_with_none(data): + if isinstance(data, dict): + # If it's a dictionary, iterate over its items and replace NaN values with None + return {key: replace_nan_with_none(value) for key, value in data.items()} + elif isinstance(data, list): + # If it's a list, iterate over its elements and replace NaN values with None + return [replace_nan_with_none(item) for item in data] + elif isinstance(data, (float, int)) and (data != data): + return None # Replace NaN with None + else: + return data + +def resolve_references(schema, root_schema = None): + + from jsonschema import RefResolver + """ + Recursively resolve references in a JSON schema based on the root schema. + + Args: + schema (dict): The JSON schema to resolve. + root_schema (dict): The root JSON schema. + + Returns: + dict: The resolved JSON schema. + """ + + if root_schema is None: + root_schema = schema + + if "$ref" in schema: + resolver = RefResolver.from_schema(root_schema) + return resolver.resolve(schema["$ref"])[1] + + + if "properties" in schema: + for key, prop_schema in schema["properties"].items(): + schema["properties"][key] = resolve_references(prop_schema, root_schema) + + if "items" in schema: + schema["items"] = resolve_references(schema["items"], root_schema) + + return schema + + +def replace_none_with_nan(json_object, json_schema): + + import math + import copy + + """ + Recursively search a JSON object and replace None values with NaN where appropriate. + + Args: + json_object (dict): The JSON object to search and modify. + json_schema (dict): The JSON schema to validate against. + + Returns: + dict: The modified JSON object with None values replaced by NaN. + """ + def replace_none_recursive(obj, schema): + if isinstance(obj, dict): + for key, value in obj.items(): + if key in schema.get("properties", {}): + prop_schema = schema["properties"][key] + if prop_schema.get("type") == "number" and value is None: + obj[key] = math.nan + else: + replace_none_recursive(value, prop_schema) + elif isinstance(obj, list): + for item in obj: + replace_none_recursive(item, schema.get("items", {})) + + return obj + + return replace_none_recursive(copy.deepcopy(json_object), resolve_references(copy.deepcopy(json_schema))) + def locate_data(info: dict) -> dict: """Locate data from the specifies directories using fstrings.""" @@ -154,7 +231,7 @@ def get_metadata_schema(source_data: Dict[str, dict], interfaces: dict) -> Dict[ if "Ecephys" in schema["properties"]: schema["properties"].pop("Ecephys", dict()) - return json.loads(json.dumps(dict(results=metadata, schema=schema), cls=NWBMetaDataEncoder)) + return json.loads(json.dumps(replace_nan_with_none(dict(results=metadata, schema=schema)), cls=NWBMetaDataEncoder)) def get_check_function(check_function_name: str) -> callable: @@ -287,6 +364,8 @@ def update_conversion_progress(**kwargs): if "Ecephys" not in info["metadata"]: info["metadata"].update(Ecephys=dict()) + resolved_metadata = replace_none_with_nan(info["metadata"], converter.get_metadata_schema()) # Ensure Ophys NaN values are resolved + # if is_supported_recording_interface(recording_interface, info["metadata"]): # electrode_column_results = ecephys_metadata["ElectrodeColumns"] # electrode_results = ecephys_metadata["Electrodes"] @@ -302,7 +381,7 @@ def update_conversion_progress(**kwargs): # Actually run the conversion converter.run_conversion( - metadata=info["metadata"], + metadata=resolved_metadata, nwbfile_path=resolved_output_path, overwrite=info.get("overwrite", False), conversion_options=options, diff --git a/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js b/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js index ab9ddeb7b..c72c7a6ee 100644 --- a/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js +++ b/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js @@ -105,7 +105,12 @@ export class GuidedMetadataPage extends ManagedPage { results, globals: aggregateGlobalMetadata, - ignore: ["subject_id", "session_id"], + ignore: + [ + "Ophys", + "subject_id", + "session_id" + ], conditionalRequirements: [ { diff --git a/src/renderer/src/validation/validation.json b/src/renderer/src/validation/validation.json index d3cfb6d9b..0d88e4510 100644 --- a/src/renderer/src/validation/validation.json +++ b/src/renderer/src/validation/validation.json @@ -23,6 +23,10 @@ "session_start_time": ["check_session_start_time_future_date", "check_session_start_time_old_date"] }, + "Ophys": { + "*": false + }, + "Ecephys": { "*": false, "ElectrodeGroup": { From 6dcfcd1939180565d22083694b4b3342c348767b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 26 Sep 2023 00:34:19 +0000 Subject: [PATCH 2/4] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pyflask/manageNeuroconv/manage_neuroconv.py | 15 +++++++++------ .../pages/guided-mode/data/GuidedMetadata.js | 7 +------ 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index 089e7edcf..aaee348bc 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -12,6 +12,7 @@ announcer = MessageAnnouncer() + def replace_nan_with_none(data): if isinstance(data, dict): # If it's a dictionary, iterate over its items and replace NaN values with None @@ -23,10 +24,11 @@ def replace_nan_with_none(data): return None # Replace NaN with None else: return data - -def resolve_references(schema, root_schema = None): + +def resolve_references(schema, root_schema=None): from jsonschema import RefResolver + """ Recursively resolve references in a JSON schema based on the root schema. @@ -45,11 +47,10 @@ def resolve_references(schema, root_schema = None): resolver = RefResolver.from_schema(root_schema) return resolver.resolve(schema["$ref"])[1] - if "properties" in schema: for key, prop_schema in schema["properties"].items(): schema["properties"][key] = resolve_references(prop_schema, root_schema) - + if "items" in schema: schema["items"] = resolve_references(schema["items"], root_schema) @@ -57,7 +58,6 @@ def resolve_references(schema, root_schema = None): def replace_none_with_nan(json_object, json_schema): - import math import copy @@ -71,6 +71,7 @@ def replace_none_with_nan(json_object, json_schema): Returns: dict: The modified JSON object with None values replaced by NaN. """ + def replace_none_recursive(obj, schema): if isinstance(obj, dict): for key, value in obj.items(): @@ -364,7 +365,9 @@ def update_conversion_progress(**kwargs): if "Ecephys" not in info["metadata"]: info["metadata"].update(Ecephys=dict()) - resolved_metadata = replace_none_with_nan(info["metadata"], converter.get_metadata_schema()) # Ensure Ophys NaN values are resolved + resolved_metadata = replace_none_with_nan( + info["metadata"], converter.get_metadata_schema() + ) # Ensure Ophys NaN values are resolved # if is_supported_recording_interface(recording_interface, info["metadata"]): # electrode_column_results = ecephys_metadata["ElectrodeColumns"] diff --git a/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js b/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js index c72c7a6ee..56af58b73 100644 --- a/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js +++ b/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js @@ -105,12 +105,7 @@ export class GuidedMetadataPage extends ManagedPage { results, globals: aggregateGlobalMetadata, - ignore: - [ - "Ophys", - "subject_id", - "session_id" - ], + ignore: ["Ophys", "subject_id", "session_id"], conditionalRequirements: [ { From 90229f9c882d6dac495b742edd199065c7757136 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Tue, 26 Sep 2023 16:40:39 -0400 Subject: [PATCH 3/4] Update pyflask/manageNeuroconv/manage_neuroconv.py --- pyflask/manageNeuroconv/manage_neuroconv.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index aaee348bc..a4edef26d 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -27,8 +27,6 @@ def replace_nan_with_none(data): def resolve_references(schema, root_schema=None): - from jsonschema import RefResolver - """ Recursively resolve references in a JSON schema based on the root schema. @@ -39,6 +37,7 @@ def resolve_references(schema, root_schema=None): Returns: dict: The resolved JSON schema. """ + from jsonschema import RefResolver if root_schema is None: root_schema = schema From 0d94e7d96b60f5b85bfc100eb94cb13c24738bf5 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Tue, 26 Sep 2023 16:42:11 -0400 Subject: [PATCH 4/4] Apply suggestions from code review --- pyflask/manageNeuroconv/manage_neuroconv.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index a4edef26d..dfab86f58 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -2,6 +2,7 @@ import os import json import math +import copy from datetime import datetime from typing import Dict, Optional # , List, Union # TODO: figure way to add these back in without importing other class from shutil import rmtree, copytree @@ -57,9 +58,6 @@ def resolve_references(schema, root_schema=None): def replace_none_with_nan(json_object, json_schema): - import math - import copy - """ Recursively search a JSON object and replace None values with NaN where appropriate.