Merge branch 'main' into updated-path-expansion-page

.github/workflows/Build-and-deploy-linux.yml

@@ -27,7 +27,7 @@ jobs:
 
       - uses: actions/setup-node@v3
         with:
-          node-version: "16"
+          node-version: "18"
 
       - name: Install package.json modules and their dependencies
         run: npm install --ignore-scripts

environments/environment-MAC-arm64.yml

@@ -10,6 +10,7 @@ dependencies:
   - lxml = 4.9.3  # pypi build fails due to x64/arm64 mismatch so install from conda-forge
   - pyedflib = 0.1.32  # pypi build fails due to x64/arm64 mismatch so install from conda-forge
   - numpy  # may have x64/arm64 mismatch issues so install from conda-forge
+  - pytables = 3.8 # pypi build fails on arm64 so install from conda-forge (used by neuroconv deps)
   # install these from conda-forge so that dependent packages get included in the distributable
   - jsonschema = 4.18.0  # installs jsonschema-specifications
   - pydantic[email] = 1.10.12  # installs email-validator

guideGlobalMetadata.json

@@ -13,6 +13,7 @@
         "BlackrockSortingInterface",
         "CellExplorerSortingInterface",
         "KiloSortSortingInterface",
-        "Spike2RecordingInterface"
+        "Spike2RecordingInterface",
+        "BrukerTiffSinglePlaneImagingInterface"
     ]
 }

package.json

@@ -4,6 +4,9 @@
   "version": "0.0.5",
   "description": "",
   "main": "./build/main/main.js",
+  "engine": {
+    "node": ">=18"
+  },
   "scripts": {
     "start": "electron-vite dev --outDir build",
     "postinstall": "electron-builder install-app-deps",
@@ -133,38 +136,37 @@
   },
   "license": "MIT",
   "dependencies": {
-    "@electron-toolkit/utils": "^1.0.2",
-    "@electron/remote": "2.0.8",
     "@fortawesome/fontawesome-free": "^5.15.3",
     "@sweetalert2/theme-bulma": "^4.0.3",
-    "axios": "^0.19.2",
     "chokidar": "^3.5.3",
     "concurrently": "^7.6.0",
     "dandi": "^0.0.2",
-    "electron-notarize": "^1.2.1",
-    "electron-updater": "4.3.9",
     "find-free-port": "^2.0.0",
     "fomantic-ui": "^2.8.8",
     "fs-extra": "^10.0.0",
     "handsontable": "^12.3.3",
     "lit": "^2.6.1",
     "lottie-web": "^5.9.5",
     "notyf": "^3.9.0",
-    "sweetalert2": "^10.16.9",
+    "sweetalert2": "^11.6.13",
     "v8-compile-cache": "^2.3.0"
   },
   "devDependencies": {
+    "@electron-toolkit/utils": "^2.0.1",
+    "@electron/remote": "^2.0.11",
     "@storybook/addon-essentials": "^7.0.0-beta.62",
     "@storybook/addon-links": "^7.0.0-beta.62",
     "@storybook/blocks": "^7.0.0-beta.62",
     "@storybook/web-components": "^7.0.0-beta.62",
     "@storybook/web-components-vite": "^7.0.0-beta.62",
     "@vitest/coverage-v8": "^0.34.1",
     "chromatic": "^6.17.1",
-    "electron": "19.0.0",
-    "electron-builder": "^24.4.0",
-    "electron-packager": "^14.0.6",
-    "electron-vite": "^1.0.22",
+    "electron": "^26.2.2",
+    "electron-builder": "^24.6.4",
+    "electron-notarize": "^1.2.2",
+    "electron-packager": "^17.1.2",
+    "electron-updater": "^6.1.4",
+    "electron-vite": "^1.0.28",
     "jsdom": "^22.0.0",
     "jsonschema": "^1.4.1",
     "prettier": "^2.3.2",

paths.config.json

@@ -3,7 +3,7 @@
     "subfolders": {
         "progress": ["pipelines"],
         "conversions": ["conversions"],
-        "stubs": ["stubs"],
+        "preview": ["preview"],
         "tutorial": ["tutorial"]
     }
 }

pyflask/app.py

@@ -84,8 +84,8 @@ def send_conversions(path):
     return send_from_directory(CONVERSION_SAVE_FOLDER_PATH, path)
 
 
-@app.route("/stubs/<path:path>")
-def send_stubs(path):
+@app.route("/preview/<path:path>")
+def send_preview(path):
     return send_from_directory(STUB_SAVE_FOLDER_PATH, path)
 
 

pyflask/manageNeuroconv/info/urls.py

@@ -20,7 +20,7 @@ def resource_path(relative_path):
 )  # NOTE: Must have pyflask for running the GUIDE as a whole, but errors for just the server
 f = path_config.open()
 data = json.load(f)
-STUB_SAVE_FOLDER_PATH = Path(Path.home(), data["root"], *data["subfolders"]["stubs"])
+STUB_SAVE_FOLDER_PATH = Path(Path.home(), data["root"], *data["subfolders"]["preview"])
 CONVERSION_SAVE_FOLDER_PATH = Path(Path.home(), data["root"], *data["subfolders"]["conversions"])
 TUTORIAL_SAVE_FOLDER_PATH = Path(Path.home(), data["root"], *data["subfolders"]["tutorial"])
 

pyflask/manageNeuroconv/manage_neuroconv.py

@@ -1,6 +1,8 @@
 """Collection of utility functions used by the NeuroConv Flask API."""
 import os
 import json
+import math
+import copy
 from datetime import datetime
 from typing import Dict, Optional  # , List, Union # TODO: figure way to add these back in without importing other class
 from shutil import rmtree, copytree
@@ -12,6 +14,79 @@
 announcer = MessageAnnouncer()
 
 
+def replace_nan_with_none(data):
+    if isinstance(data, dict):
+        # If it's a dictionary, iterate over its items and replace NaN values with None
+        return {key: replace_nan_with_none(value) for key, value in data.items()}
+    elif isinstance(data, list):
+        # If it's a list, iterate over its elements and replace NaN values with None
+        return [replace_nan_with_none(item) for item in data]
+    elif isinstance(data, (float, int)) and (data != data):
+        return None  # Replace NaN with None
+    else:
+        return data
+
+
+def resolve_references(schema, root_schema=None):
+    """
+    Recursively resolve references in a JSON schema based on the root schema.
+
+    Args:
+        schema (dict): The JSON schema to resolve.
+        root_schema (dict): The root JSON schema.
+
+    Returns:
+        dict: The resolved JSON schema.
+    """
+    from jsonschema import RefResolver
+
+    if root_schema is None:
+        root_schema = schema
+
+    if "$ref" in schema:
+        resolver = RefResolver.from_schema(root_schema)
+        return resolver.resolve(schema["$ref"])[1]
+
+    if "properties" in schema:
+        for key, prop_schema in schema["properties"].items():
+            schema["properties"][key] = resolve_references(prop_schema, root_schema)
+
+    if "items" in schema:
+        schema["items"] = resolve_references(schema["items"], root_schema)
+
+    return schema
+
+
+def replace_none_with_nan(json_object, json_schema):
+    """
+    Recursively search a JSON object and replace None values with NaN where appropriate.
+
+    Args:
+        json_object (dict): The JSON object to search and modify.
+        json_schema (dict): The JSON schema to validate against.
+
+    Returns:
+        dict: The modified JSON object with None values replaced by NaN.
+    """
+
+    def replace_none_recursive(obj, schema):
+        if isinstance(obj, dict):
+            for key, value in obj.items():
+                if key in schema.get("properties", {}):
+                    prop_schema = schema["properties"][key]
+                    if prop_schema.get("type") == "number" and value is None:
+                        obj[key] = math.nan
+                    else:
+                        replace_none_recursive(value, prop_schema)
+        elif isinstance(obj, list):
+            for item in obj:
+                replace_none_recursive(item, schema.get("items", {}))
+
+        return obj
+
+    return replace_none_recursive(copy.deepcopy(json_object), resolve_references(copy.deepcopy(json_schema)))
+
+
 def locate_data(info: dict) -> dict:
     """Locate data from the specifies directories using fstrings."""
     from neuroconv.tools import LocalPathExpander
@@ -154,7 +229,7 @@ def get_metadata_schema(source_data: Dict[str, dict], interfaces: dict) -> Dict[
     if "Ecephys" in schema["properties"]:
         schema["properties"].pop("Ecephys", dict())
 
-    return json.loads(json.dumps(dict(results=metadata, schema=schema), cls=NWBMetaDataEncoder))
+    return json.loads(json.dumps(replace_nan_with_none(dict(results=metadata, schema=schema)), cls=NWBMetaDataEncoder))
 
 
 def get_check_function(check_function_name: str) -> callable:
@@ -287,6 +362,10 @@ def update_conversion_progress(**kwargs):
     if "Ecephys" not in info["metadata"]:
         info["metadata"].update(Ecephys=dict())
 
+    resolved_metadata = replace_none_with_nan(
+        info["metadata"], converter.get_metadata_schema()
+    )  # Ensure Ophys NaN values are resolved
+
     # if is_supported_recording_interface(recording_interface, info["metadata"]):
     #     electrode_column_results = ecephys_metadata["ElectrodeColumns"]
     #     electrode_results = ecephys_metadata["Electrodes"]
@@ -302,7 +381,7 @@ def update_conversion_progress(**kwargs):
 
     # Actually run the conversion
     converter.run_conversion(
-        metadata=info["metadata"],
+        metadata=resolved_metadata,
         nwbfile_path=resolved_output_path,
         overwrite=info.get("overwrite", False),
         conversion_options=options,

schemas/json/generated/BrukerTiffSinglePlaneImagingInterface.json

@@ -0,0 +1,33 @@
+{
+    "required": [],
+    "properties": {
+        "BrukerTiffSinglePlaneImagingInterface": {
+            "required": [
+                "folder_path"
+            ],
+            "properties": {
+                "folder_path": {
+                    "format": "directory",
+                    "type": "string",
+                    "description": "The path that points to the folder containing the Bruker TIF image files and configuration files."
+                },
+                "stream_name": {
+                    "type": "string"
+                },
+                "verbose": {
+                    "type": "boolean",
+                    "default": true
+                }
+            },
+            "type": "object",
+            "additionalProperties": false
+        }
+    },
+    "type": "object",
+    "additionalProperties": false,
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "$id": "source.schema.json",
+    "title": "Source data schema",
+    "description": "Schema for the source data, files and directories",
+    "version": "0.1.0"
+}

schemas/json/generated/OpenEphysRecordingInterface.json

@@ -16,6 +16,10 @@
                 "verbose": {
                     "type": "boolean",
                     "default": true
+                },
+                "es_key": {
+                    "type": "string",
+                    "default": "ElectricalSeries"
                 }
             },
             "type": "object",

schemas/json/generated/Spike2RecordingInterface.json

@@ -0,0 +1,34 @@
+{
+    "required": [],
+    "properties": {
+        "Spike2RecordingInterface": {
+            "required": [
+                "file_path"
+            ],
+            "properties": {
+                "file_path": {
+                    "format": "file",
+                    "type": "string",
+                    "description": "Path to CED data file."
+                },
+                "verbose": {
+                    "type": "boolean",
+                    "default": true
+                },
+                "es_key": {
+                    "type": "string",
+                    "default": "ElectricalSeries"
+                }
+            },
+            "type": "object",
+            "additionalProperties": true
+        }
+    },
+    "type": "object",
+    "additionalProperties": false,
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "$id": "source.schema.json",
+    "title": "Source data schema",
+    "description": "Schema for the source data, files and directories",
+    "version": "0.1.0"
+}

schemas/json/project/globals.json

@@ -6,10 +6,10 @@
             "description": "Provide a custom output location for your NWB files. Will default to ~/NWB_GUIDE/conversions"
         },
 
-        "stub_output_folder": {
+        "preview_output_folder": {
             "type": "string",
             "format": "directory",
-            "description": "Provide a custom output location for your NWB stub files. Will default to ~/NWB_GUIDE/stubs"
+            "description": "Provide a custom output location for your NWB stub files. Will default to ~/NWB_GUIDE/preview. These files are expected to much smaller than the conversion files (only ~MB in scale)."
         }
     }
 }