Skip to content

Commit

Permalink
Merge branch 'main' into improve-list-rendering-for-structure
Browse files Browse the repository at this point in the history
  • Loading branch information
garrettmflynn committed Aug 23, 2023
2 parents 5024d33 + 50e317d commit 8dc9bf5
Show file tree
Hide file tree
Showing 73 changed files with 1,825 additions and 619 deletions.
10 changes: 7 additions & 3 deletions .github/workflows/pyflask-build-and-dist-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -81,17 +81,21 @@ jobs:
with:
node-version: "18"

- run: npm ci

- name: Build PyFlask distribution
run: npm run build:flask:${{ matrix.shorthand }}

# encountering parsing issue for second arg
#- if: matrix.os == 'windows-latest'
# name: Run test on build executable
# run: node tests/testPyinstallerExecutable.js ./build/nwb-guide/nwb-guide.exe
# run: node tests/testPyinstallerExecutable.js --script ./build/nwb-guide/nwb-guide.exe

#- if: matrix.os != 'windows-latest'
# name: Run test on build executable
# run: node tests/testPyinstallerExecutable.js ./build/nwb-guide/nwb-guide
# run: node tests/testPyinstallerExecutable.js --script ./build/nwb-guide/nwb-guide

- name: Run test on distributed executable
run: node tests/testPyinstallerExecutable.js

- name: Run Python tests on distributed executable
run: npm run test:executable
2 changes: 1 addition & 1 deletion .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ repos:
- id: black
exclude: ^docs/
- repo: https://github.com/pre-commit/mirrors-prettier
rev: "v3.0.1"
rev: "v3.0.2"
hooks:
- id: prettier
types_or: [css, javascript]
2 changes: 1 addition & 1 deletion docs/conf.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
templates_path = ["_templates"]
master_doc = "index"
exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
# html_theme = "sphinx_rtd_theme"
html_theme = "sphinx_rtd_theme"

# The name of an image file (within the static path) to use as favicon of the
# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32
Expand Down
85 changes: 85 additions & 0 deletions generateInterfaceSchema.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
from pathlib import Path
import json
from neuroconv import datainterfaces, NWBConverter

filepath = Path("guideGlobalMetadata.json")
generatedJSONSchemaPath = Path("schemas", "json", "generated")
generatedJSONSchemaPath.mkdir(exist_ok=True, parents=True)

f = filepath.open()
data = json.load(f)

# Create JSON for the Schema
paths = {}
for interface in data["supported_interfaces"]:
interface_class_dict = {interface: interface}

class CustomNWBConverter(NWBConverter):
data_interface_classes = {
custom_name: getattr(datainterfaces, interface_name)
for custom_name, interface_name in interface_class_dict.items()
}

schema = CustomNWBConverter.get_source_schema()

json_object = json.dumps(schema, indent=4)
paths[interface] = filepath = generatedJSONSchemaPath / f"{interface}.json"
with open(filepath, "w") as outfile:
outfile.write(json.dumps(schema, indent=4))


sourceDataStoryPath = Path("src/renderer/src/stories/pages/guided-mode/SourceData.stories.js")

importCode = "\n".join(map(lambda arr: f"import {arr[0]}Schema from '../../../../../../{arr[1]}'", paths.items()))
storyCode = "\n".join(
map(
lambda arr: f"""export const {arr[0]} = PageTemplate.bind({{}});
const {arr[0]}GlobalCopy = JSON.parse(JSON.stringify(globalState))
{arr[0]}GlobalCopy.interfaces.interface = {arr[0]}
{arr[0]}GlobalCopy.schema.source_data = {arr[0]}Schema
{arr[0]}.args = {{ activePage, globalState: {arr[0]}GlobalCopy }};
""",
paths.items(),
)
)


allInterfaceCode = "\n".join(
map(
lambda arr: f"globalStateCopy.schema.source_data.properties.{arr[0]} = {arr[0]}Schema.properties.{arr[0]}",
paths.items(),
)
)

setDummyPathCode = f"""
const results = globalStateCopy.results
for (let sub in results){{
for (let ses in results[sub]) results[sub][ses].source_data = {{{list(paths.keys())[1]}: {{file_path: '/dummy/file/path'}}}}
}}
"""

with open(sourceDataStoryPath, "w") as outfile:
outfile.write(
f"""import {{ globalState, PageTemplate }} from "./storyStates";
{importCode}
export default {{
title: "Pages/Guided Mode/Source Data",
parameters: {{
chromatic: {{ disableSnapshot: false }},
}}
}};
const activePage = "conversion/sourcedata"
const globalStateCopy = JSON.parse(JSON.stringify(globalState))
{allInterfaceCode}
{setDummyPathCode}
export const All = PageTemplate.bind({{}});
All.args = {{ activePage, globalState: globalStateCopy }};
{storyCode}
"""
)
12 changes: 10 additions & 2 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -15,15 +15,16 @@
"build:win": "npm run build && npm run build:flask:win && npm run build:electron:win",
"build:mac": "npm run build && npm run build:flask:unix && npm run build:electron:mac",
"build:linux": "npm run build && npm run build:flask:unix && npm run build:electron:linux",
"build:flask:base": "python -m PyInstaller --name nwb-guide --onedir --clean --noconfirm ./pyflask/app.py --distpath ./build/flask --collect-all nwbinspector --collect-all neuroconv --collect-all pynwb --collect-all hdmf --collect-all ci_info --hidden-import scipy._distributor_init --hidden-import scipy._lib.messagestream --hidden-import scipy._lib._ccallback --hidden-import scipy._lib._testutils",
"build:flask:base": "python -m PyInstaller --name nwb-guide --onedir --clean --noconfirm ./pyflask/app.py --distpath ./build/flask --collect-data jsonschema_specifications --collect-all nwbinspector --collect-all neuroconv --collect-all pynwb --collect-all hdmf --collect-all ci_info --hidden-import scipy._distributor_init --hidden-import scipy._lib.messagestream --hidden-import scipy._lib._ccallback --hidden-import scipy._lib._testutils",
"build:flask:win": "npm run build:flask:base -- --add-data ./paths.config.json;. --add-data ./package.json;.",
"build:flask:unix": "npm run build:flask:base -- --add-data ./paths.config.json:. --add-data ./package.json:. --collect-data jsonschema_specifications --collect-all ndx_dandi_icephys",
"build:flask:unix": "npm run build:flask:base -- --add-data ./paths.config.json:. --add-data ./package.json:. --collect-all ndx_dandi_icephys",
"build:electron:win": "electron-builder build --win --publish never",
"build:electron:mac": "electron-builder build --mac --publish never",
"build:electron:linux": "electron-builder build --linux --publish never",
"test": "npm run test:app && npm run test:server",
"test:app": "vitest run",
"test:server": "pytest pyflask/tests/ -s",
"test:executable": "concurrently -n EXE,TEST --kill-others \"node tests/testPyinstallerExecutable.js --port 3434 --forever\" \"pytest pyflask/tests/ -s --target http://localhost:3434\"",
"test:coverage": "npm run coverage:app && npm run coverage:server",
"coverage:app": "vitest run --coverage",
"coverage:server": "pytest pyflask/tests/ -s --cov=pyflask --cov-report=xml",
Expand Down Expand Up @@ -51,6 +52,13 @@
"appId": "com.catalystneuro.nwbguide",
"generateUpdatesFilesForAllChannels": true,
"afterSign": "./notarize.js",
"fileAssociations": [
{
"ext": "nwb",
"name": "NWB File",
"role": "Editor"
}
],
"files": [
"src",
"schemas",
Expand Down
2 changes: 0 additions & 2 deletions pyflask/manageNeuroconv/manage_neuroconv.py
Original file line number Diff line number Diff line change
Expand Up @@ -287,8 +287,6 @@ def update_conversion_progress(**kwargs):
if "Ecephys" not in info["metadata"]:
info["metadata"].update(Ecephys=dict())

# ecephys_metadata = info["metadata"].get("Ecephys", dict())

# if is_supported_recording_interface(recording_interface, info["metadata"]):
# electrode_column_results = ecephys_metadata["ElectrodeColumns"]
# electrode_results = ecephys_metadata["Electrodes"]
Expand Down
33 changes: 18 additions & 15 deletions pyflask/tests/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,21 +2,24 @@
import app as flask


@pytest.fixture()
def app():
app = flask.app
app.config.update(
{
"TESTING": True,
}
)

yield app


@pytest.fixture()
def client(app):
return app.test_client()
def pytest_addoption(parser):
parser.addoption("--target", action="store", help="Run the executable instead of the standard Flask app")


@pytest.fixture(scope="session")
def client(request):
target = request.config.getoption("--target")
if target:
return target
else:
app = flask.app
app.config.update(
{
"TESTING": True,
}
)

return app.test_client()


@pytest.fixture()
Expand Down
34 changes: 4 additions & 30 deletions pyflask/tests/test_neuroconv.py
Original file line number Diff line number Diff line change
@@ -1,37 +1,12 @@
from jsonschema import validate


def get_converter_output_schema(interfaces: dict):
return {
"type": "object",
"properties": {
"title": {"type": "string"},
"description": {"type": "string"},
"version": {"type": "string"},
"properties": {
"type": "object",
"properties": {
interface: {
"type": "object",
"properties": {
"properties": {"type": "object"},
"required": {"type": "array"},
},
}
for interface in interfaces.keys()
},
"additionalProperties": False,
},
},
}
from utils import get, post, get_converter_output_schema


# --------------------- Tests ---------------------
# Accesses the dictionary of all interfaces and their metadata
def test_get_all_interfaces(client):
all_interfaces = client.get("/neuroconv", follow_redirects=True).json
validate(
all_interfaces,
get("neuroconv", client),
schema={
"type": "object",
"patternProperties": {
Expand All @@ -51,12 +26,11 @@ def test_get_all_interfaces(client):
# Test single interface schema request
def test_single_schema_request(client):
interfaces = {"myname": "SpikeGLXRecordingInterface"}
data = client.post("/neuroconv/schema", json=interfaces, follow_redirects=True).json
validate(data, schema=get_converter_output_schema(interfaces))
validate(post("neuroconv/schema", interfaces, client), schema=get_converter_output_schema(interfaces))


# Uses the NWBConverter Class to combine multiple interfaces
def test_multiple_schema_request(client):
interfaces = {"myname": "SpikeGLXRecordingInterface", "myphyinterface": "PhySortingInterface"}
data = client.post("/neuroconv/schema", json=interfaces, follow_redirects=True).json
data = post("/neuroconv/schema", interfaces, client)
validate(data, schema=get_converter_output_schema(interfaces))
44 changes: 44 additions & 0 deletions pyflask/tests/utils.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
import requests


def get(path, client):
if isinstance(client, str):
r = requests.get(f"{client}/{path}", allow_redirects=True)
r.raise_for_status()
return r.json()
else:
return client.get(f"/{path}", follow_redirects=True).json


def post(path, json, client):
if isinstance(client, str):
r = requests.post(f"{client}/{path}", json=json, allow_redirects=True)
r.raise_for_status()
return r.json()
else:
return client.post(path, json=json, follow_redirects=True).json


def get_converter_output_schema(interfaces: dict):
return {
"type": "object",
"properties": {
"title": {"type": "string"},
"description": {"type": "string"},
"version": {"type": "string"},
"properties": {
"type": "object",
"properties": {
interface: {
"type": "object",
"properties": {
"properties": {"type": "object"},
"required": {"type": "array"},
},
}
for interface in interfaces.keys()
},
"additionalProperties": False,
},
},
}
33 changes: 33 additions & 0 deletions schemas/json/generated/BiocamRecordingInterface.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
{
"required": [],
"properties": {
"BiocamRecordingInterface": {
"required": [
"file_path"
],
"properties": {
"file_path": {
"format": "file",
"type": "string"
},
"verbose": {
"type": "boolean",
"default": true
},
"es_key": {
"type": "string",
"default": "ElectricalSeries"
}
},
"type": "object",
"additionalProperties": false
}
},
"type": "object",
"additionalProperties": false,
"$schema": "http://json-schema.org/draft-07/schema#",
"$id": "source.schema.json",
"title": "Source data schema",
"description": "Schema for the source data, files and directories",
"version": "0.1.0"
}
38 changes: 38 additions & 0 deletions schemas/json/generated/BlackrockRecordingInterface.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
{
"required": [],
"properties": {
"BlackrockRecordingInterface": {
"required": [
"file_path"
],
"properties": {
"file_path": {
"format": "file",
"type": "string",
"description": "Path to Blackrock file."
},
"nsx_override": {
"format": "file",
"type": "string"
},
"verbose": {
"type": "boolean",
"default": true
},
"es_key": {
"type": "string",
"default": "ElectricalSeries"
}
},
"type": "object",
"additionalProperties": false
}
},
"type": "object",
"additionalProperties": false,
"$schema": "http://json-schema.org/draft-07/schema#",
"$id": "source.schema.json",
"title": "Source data schema",
"description": "Schema for the source data, files and directories",
"version": "0.1.0"
}
Loading

0 comments on commit 8dc9bf5

Please sign in to comment.