diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9273adef7..248023898 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -10,6 +10,10 @@ repos: hooks: - id: black exclude: ^docs/ +- repo: https://github.com/PyCQA/isort + rev: 5.13.2 + hooks: + - id: isort - repo: https://github.com/pre-commit/mirrors-prettier rev: "v4.0.0-alpha.8" hooks: diff --git a/demos/sse/test_sse_display_of_tqdm.py b/demos/sse/test_sse_display_of_tqdm.py index 0b705a971..b32452bd6 100644 --- a/demos/sse/test_sse_display_of_tqdm.py +++ b/demos/sse/test_sse_display_of_tqdm.py @@ -1,12 +1,12 @@ -from flask import Flask, render_template, Response -from typing import List -import random import asyncio +import os +import random +import sys import time -from tqdm import tqdm as base_tqdm +from typing import List -import sys -import os +from flask import Flask, Response, render_template +from tqdm import tqdm as base_tqdm SCRIPT_DIR = os.path.dirname(os.path.abspath(os.path.join(__file__, "..", "..", "pyflask"))) sys.path.append(os.path.dirname(SCRIPT_DIR)) diff --git a/docs/conf.py b/docs/conf.py index 5067e13b6..0c466f25e 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,15 +1,14 @@ -import sys import inspect -from pathlib import Path import json import os +import sys +from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parents[0])) sys.path.insert(0, str(Path(__file__).resolve().parents[1])) from conf_extlinks import extlinks, intersphinx_mapping # noqa: E402, F401 - project = "NWB GUIDE" copyright = "2022, CatalystNeuro" # TODO: how to include NWB? author = "Garrett Flynn, Cody Baker, Ryan Ly, Oliver Ruebel, and Ben Dichter" diff --git a/generateInterfaceSchema.py b/generateInterfaceSchema.py index 9a04a8b85..f63eb41f4 100644 --- a/generateInterfaceSchema.py +++ b/generateInterfaceSchema.py @@ -1,6 +1,7 @@ -from pathlib import Path import json -from neuroconv import converters, datainterfaces, NWBConverter +from pathlib import Path + +from neuroconv import NWBConverter, converters, datainterfaces filepath = Path("guideGlobalMetadata.json") generatedJSONSchemaPath = Path("schemas", "json", "generated") diff --git a/pyflask/apis/__init__.py b/pyflask/apis/__init__.py index 9368a0aa8..f492dc7df 100644 --- a/pyflask/apis/__init__.py +++ b/pyflask/apis/__init__.py @@ -1,3 +1,3 @@ -from .startup import startup_api -from .neuroconv import neuroconv_api from .data import data_api +from .neuroconv import neuroconv_api +from .startup import startup_api diff --git a/pyflask/apis/data.py b/pyflask/apis/data.py index 122559150..de6dfdc6d 100644 --- a/pyflask/apis/data.py +++ b/pyflask/apis/data.py @@ -2,10 +2,9 @@ import traceback -from flask_restx import Namespace, Resource, reqparse - -from manageNeuroconv import generate_test_data, generate_dataset from errorHandlers import notBadRequestException +from flask_restx import Namespace, Resource, reqparse +from manageNeuroconv import generate_dataset, generate_test_data data_api = Namespace("data", description="API route for dataset generation in the NWB GUIDE.") diff --git a/pyflask/apis/neuroconv.py b/pyflask/apis/neuroconv.py index 1877e763e..015cb61e8 100644 --- a/pyflask/apis/neuroconv.py +++ b/pyflask/apis/neuroconv.py @@ -2,32 +2,28 @@ import traceback -from flask_restx import Namespace, Resource, reqparse +from errorHandlers import notBadRequestException from flask import Response, request - -from manageNeuroconv.info import announcer - +from flask_restx import Namespace, Resource, reqparse from manageNeuroconv import ( - get_all_interface_info, - get_all_converter_info, - locate_data, autocomplete_format_string, - get_source_schema, - get_metadata_schema, convert_to_nwb, - validate_metadata, - listen_to_neuroconv_events, + get_all_converter_info, + get_all_interface_info, + get_interface_alignment, + get_metadata_schema, + get_source_schema, + inspect_multiple_filesystem_objects, inspect_nwb_file, inspect_nwb_folder, - inspect_multiple_filesystem_objects, - upload_project_to_dandi, + listen_to_neuroconv_events, + locate_data, upload_folder_to_dandi, upload_multiple_filesystem_objects_to_dandi, - get_interface_alignment, + upload_project_to_dandi, + validate_metadata, ) - -from errorHandlers import notBadRequestException - +from manageNeuroconv.info import announcer neuroconv_api = Namespace("neuroconv", description="Neuroconv neuroconv_api for the NWB GUIDE.") diff --git a/pyflask/apis/startup.py b/pyflask/apis/startup.py index feb59d871..63be4fd0a 100644 --- a/pyflask/apis/startup.py +++ b/pyflask/apis/startup.py @@ -1,8 +1,7 @@ """API endpoint definitions for startup operations.""" -from flask_restx import Namespace, Resource - from errorHandlers import notBadRequestException +from flask_restx import Namespace, Resource startup_api = Namespace("startup", description="API for startup commands related to the NWB GUIDE.") diff --git a/pyflask/app.py b/pyflask/app.py index 924e9454a..d84b27bae 100644 --- a/pyflask/app.py +++ b/pyflask/app.py @@ -1,28 +1,29 @@ """The primary Flask server for the Python backend.""" -import sys import json import multiprocessing -from os import kill, getpid -from os.path import isabs - -from signal import SIGINT -from logging import Formatter, DEBUG +import sys +from logging import DEBUG, Formatter from logging.handlers import RotatingFileHandler +from os import getpid, kill +from os.path import isabs from pathlib import Path +from signal import SIGINT from urllib.parse import unquote - # https://stackoverflow.com/questions/32672596/pyinstaller-loads-script-multiple-times#comment103216434_32677108 multiprocessing.freeze_support() -from flask import Flask, request, send_from_directory, send_file +from apis import data_api, neuroconv_api, startup_api +from flask import Flask, request, send_file, send_from_directory from flask_cors import CORS from flask_restx import Api, Resource - -from apis import startup_api, neuroconv_api, data_api -from manageNeuroconv.info import resource_path, STUB_SAVE_FOLDER_PATH, CONVERSION_SAVE_FOLDER_PATH +from manageNeuroconv.info import ( + CONVERSION_SAVE_FOLDER_PATH, + STUB_SAVE_FOLDER_PATH, + resource_path, +) app = Flask(__name__) diff --git a/pyflask/manageNeuroconv/__init__.py b/pyflask/manageNeuroconv/__init__.py index ddcc2b8b6..f9f55e082 100644 --- a/pyflask/manageNeuroconv/__init__.py +++ b/pyflask/manageNeuroconv/__init__.py @@ -1,23 +1,21 @@ +from .info import CONVERSION_SAVE_FOLDER_PATH, STUB_SAVE_FOLDER_PATH from .manage_neuroconv import ( - get_all_interface_info, - get_all_converter_info, - locate_data, autocomplete_format_string, - get_source_schema, - get_metadata_schema, convert_to_nwb, - validate_metadata, - upload_project_to_dandi, - upload_folder_to_dandi, - upload_multiple_filesystem_objects_to_dandi, - listen_to_neuroconv_events, generate_dataset, + generate_test_data, + get_all_converter_info, + get_all_interface_info, + get_interface_alignment, + get_metadata_schema, + get_source_schema, + inspect_multiple_filesystem_objects, inspect_nwb_file, inspect_nwb_folder, - inspect_multiple_filesystem_objects, - get_interface_alignment, - generate_test_data, + listen_to_neuroconv_events, + locate_data, + upload_folder_to_dandi, + upload_multiple_filesystem_objects_to_dandi, + upload_project_to_dandi, + validate_metadata, ) - - -from .info import STUB_SAVE_FOLDER_PATH, CONVERSION_SAVE_FOLDER_PATH diff --git a/pyflask/manageNeuroconv/info/__init__.py b/pyflask/manageNeuroconv/info/__init__.py index 915d74aee..edde04113 100644 --- a/pyflask/manageNeuroconv/info/__init__.py +++ b/pyflask/manageNeuroconv/info/__init__.py @@ -1,8 +1,7 @@ +from .sse import announcer, format_sse from .urls import ( - resource_path, + CONVERSION_SAVE_FOLDER_PATH, GUIDE_ROOT_FOLDER, STUB_SAVE_FOLDER_PATH, - CONVERSION_SAVE_FOLDER_PATH, + resource_path, ) - -from .sse import announcer, format_sse diff --git a/pyflask/manageNeuroconv/info/sse.py b/pyflask/manageNeuroconv/info/sse.py index bb90f34f7..b9593cba5 100644 --- a/pyflask/manageNeuroconv/info/sse.py +++ b/pyflask/manageNeuroconv/info/sse.py @@ -1,5 +1,5 @@ -import queue import json +import queue def format_sse(data: str, event=None) -> str: diff --git a/pyflask/manageNeuroconv/info/urls.py b/pyflask/manageNeuroconv/info/urls.py index 261f1188c..bf8a65116 100644 --- a/pyflask/manageNeuroconv/info/urls.py +++ b/pyflask/manageNeuroconv/info/urls.py @@ -1,7 +1,7 @@ -from pathlib import Path import json import os import sys +from pathlib import Path def resource_path(relative_path): diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index b399b681d..dedabd683 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -1,19 +1,22 @@ """Collection of utility functions used by the NeuroConv Flask API.""" -import os +import copy +import hashlib import json import math -import copy +import os import re -import hashlib -from pathlib import Path from datetime import datetime -from typing import Dict, Optional, Union -from shutil import rmtree, copytree from pathlib import Path -from typing import Any, Dict, List, Optional +from shutil import copytree, rmtree +from typing import Any, Dict, List, Optional, Union -from .info import GUIDE_ROOT_FOLDER, STUB_SAVE_FOLDER_PATH, CONVERSION_SAVE_FOLDER_PATH, announcer +from .info import ( + CONVERSION_SAVE_FOLDER_PATH, + GUIDE_ROOT_FOLDER, + STUB_SAVE_FOLDER_PATH, + announcer, +) EXCLUDED_RECORDING_INTERFACE_PROPERTIES = ["contact_vector", "contact_shapes", "group", "location"] @@ -112,7 +115,7 @@ def replace_nan_with_none(data): return data -def resolve_references(schema, root_schema=None): +def resolve_references(schema: dict, root_schema: Optional[dict] = None) -> dict: """ Recursively resolve references in a JSON schema based on the root schema. @@ -142,7 +145,7 @@ def resolve_references(schema, root_schema=None): return schema -def replace_none_with_nan(json_object, json_schema): +def replace_none_with_nan(json_object: dict, json_schema: dict) -> dict: """ Recursively search a JSON object and replace None values with NaN where appropriate. @@ -254,7 +257,7 @@ def locate_data(info: dict) -> dict: return json.loads(json.dumps(obj=organized_output, cls=NWBMetaDataEncoder)) -def module_to_dict(my_module): +def module_to_dict(my_module) -> dict: # Create an empty dictionary module_dict = {} @@ -278,7 +281,7 @@ def get_class_ref_in_docstring(input_string): return match.group(1) -def derive_interface_info(interface): +def derive_interface_info(interface) -> dict: info = {"keywords": getattr(interface, "keywords", []), "description": ""} @@ -338,7 +341,7 @@ def get_all_interface_info() -> dict: # Combine Multiple Interfaces def get_custom_converter(interface_class_dict: dict): # -> NWBConverter: - from neuroconv import converters, datainterfaces, NWBConverter + from neuroconv import NWBConverter, converters, datainterfaces class CustomNWBConverter(NWBConverter): data_interface_classes = { @@ -349,7 +352,7 @@ class CustomNWBConverter(NWBConverter): return CustomNWBConverter -def instantiate_custom_converter(source_data, interface_class_dict): # -> NWBConverter: +def instantiate_custom_converter(source_data: dict, interface_class_dict: dict): # -> NWBConverter: CustomNWBConverter = get_custom_converter(interface_class_dict) return CustomNWBConverter(source_data) @@ -360,7 +363,7 @@ def get_source_schema(interface_class_dict: dict) -> dict: return CustomNWBConverter.get_source_schema() -def map_interfaces(callback, converter, to_match: Union["BaseDataInterface", None] = None, parent_name=None): +def map_interfaces(callback, converter, to_match: Union["BaseDataInterface", None] = None, parent_name=None) -> list: from neuroconv import NWBConverter output = [] @@ -478,8 +481,12 @@ def on_recording_interface(name, recording_interface): return recording_interface - from neuroconv.datainterfaces.ecephys.baserecordingextractorinterface import BaseRecordingExtractorInterface - from neuroconv.datainterfaces.ecephys.basesortingextractorinterface import BaseSortingExtractorInterface + from neuroconv.datainterfaces.ecephys.baserecordingextractorinterface import ( + BaseRecordingExtractorInterface, + ) + from neuroconv.datainterfaces.ecephys.basesortingextractorinterface import ( + BaseSortingExtractorInterface, + ) # Map recording interfaces to metadata map_interfaces(on_recording_interface, converter=converter, to_match=BaseRecordingExtractorInterface) @@ -606,7 +613,7 @@ def get_check_function(check_function_name: str) -> callable: def run_check_function(check_function: callable, arg: dict) -> dict: """.Function used to run an arbitrary NWB Inspector function.""" - from nwbinspector.register_checks import InspectorMessage, Importance + from nwbinspector.register_checks import Importance, InspectorMessage output = check_function(arg) if isinstance(output, InspectorMessage): @@ -649,8 +656,8 @@ def validate_nwbfile_metadata( def validate_metadata(metadata: dict, check_function_name: str) -> dict: """Function used to validate data using an arbitrary NWB Inspector function.""" - from pynwb.file import NWBFile, Subject from nwbinspector.nwbinspector import InspectorOutputJSONEncoder + from pynwb.file import NWBFile, Subject check_function = get_check_function(check_function_name) @@ -836,7 +843,7 @@ def update_conversion_progress(**kwargs): return dict(file=str(resolved_output_path)) -def upload_multiple_filesystem_objects_to_dandi(**kwargs): +def upload_multiple_filesystem_objects_to_dandi(**kwargs) -> list[Path]: tmp_folder_path = _aggregate_symlinks_in_new_directory(kwargs["filesystem_paths"], "upload") innerKwargs = {**kwargs} del innerKwargs["filesystem_paths"] @@ -855,7 +862,7 @@ def upload_folder_to_dandi( number_of_jobs: Optional[int] = None, number_of_threads: Optional[int] = None, ignore_cache: bool = False, -): +) -> list[Path]: from neuroconv.tools.data_transfers import automatic_dandi_upload os.environ["DANDI_API_KEY"] = api_key # Update API Key @@ -884,7 +891,7 @@ def upload_project_to_dandi( number_of_jobs: Optional[int] = None, number_of_threads: Optional[int] = None, ignore_cache: bool = False, -): +) -> list[Path]: from neuroconv.tools.data_transfers import automatic_dandi_upload # CONVERSION_SAVE_FOLDER_PATH.mkdir(exist_ok=True, parents=True) # Ensure base directory exists @@ -914,7 +921,7 @@ def listen_to_neuroconv_events(): yield msg -def generate_dataset(input_path: str, output_path: str): +def generate_dataset(input_path: str, output_path: str) -> dict: base_path = Path(input_path) output_path = Path(output_path) @@ -956,7 +963,7 @@ def generate_dataset(input_path: str, output_path: str): return {"output_path": str(output_path)} -def inspect_nwb_file(payload): +def inspect_nwb_file(payload) -> dict: from nwbinspector import inspect_nwbfile, load_config from nwbinspector.inspector_tools import format_messages, get_report_header from nwbinspector.nwbinspector import InspectorOutputJSONEncoder @@ -987,12 +994,12 @@ def _inspect_file_per_job( url, ignore: Optional[List[str]] = None, request_id: Optional[str] = None, -): +) -> list: + import requests from nwbinspector import nwbinspector from pynwb import NWBHDF5IO from tqdm_publisher import TQDMProgressSubscriber - import requests checks = nwbinspector.configure_checks( checks=nwbinspector.available_checks, @@ -1025,6 +1032,7 @@ def _inspect_file_per_job( def inspect_all(url, config): from concurrent.futures import ProcessPoolExecutor, as_completed + from nwbinspector.utils import calculate_number_of_cpu from tqdm_publisher import TQDMProgressSubscriber @@ -1085,11 +1093,12 @@ def on_progress_update(message): return messages -def inspect_nwb_folder(url, payload): +def inspect_nwb_folder(url, payload) -> dict: + from pickle import PicklingError + from nwbinspector import load_config from nwbinspector.inspector_tools import format_messages, get_report_header from nwbinspector.nwbinspector import InspectorOutputJSONEncoder - from pickle import PicklingError kwargs = dict( ignore=[ @@ -1118,7 +1127,7 @@ def inspect_nwb_folder(url, payload): return json.loads(json.dumps(obj=json_report, cls=InspectorOutputJSONEncoder)) -def _aggregate_symlinks_in_new_directory(paths, reason="", folder_path=None): +def _aggregate_symlinks_in_new_directory(paths, reason="", folder_path=None) -> Path: if folder_path is None: folder_path = GUIDE_ROOT_FOLDER / ".temp" / reason / f"temp_{datetime.now().strftime('%Y%m%d-%H%M%S')}" @@ -1137,7 +1146,7 @@ def _aggregate_symlinks_in_new_directory(paths, reason="", folder_path=None): return folder_path -def inspect_multiple_filesystem_objects(url, paths, **kwargs): +def inspect_multiple_filesystem_objects(url, paths, **kwargs) -> dict: tmp_folder_path = _aggregate_symlinks_in_new_directory(paths, "inspect") result = inspect_nwb_folder(url, {"path": tmp_folder_path, **kwargs}) rmtree(tmp_folder_path) @@ -1211,9 +1220,8 @@ def generate_test_data(output_path: str): Consists of a single-probe single-segment SpikeGLX recording (both AP and LF bands) as well as Phy spiking data. """ import spikeinterface - from spikeinterface.extractors import NumpyRecording from spikeinterface.exporters import export_to_phy - from spikeinterface.preprocessing import scale, bandpass_filter, resample + from spikeinterface.preprocessing import bandpass_filter, resample, scale base_path = Path(output_path) spikeglx_output_folder = base_path / "spikeglx" @@ -1283,7 +1291,7 @@ def map_dtype(dtype: str) -> str: return dtype -def get_property_dtype(extractor, property_name: str, ids: list, extra_props: dict): +def get_property_dtype(extractor, property_name: str, ids: list, extra_props: dict) -> str: if property_name in extra_props: dtype = extra_props[property_name]["data_type"] else: diff --git a/pyflask/tests/conftest.py b/pyflask/tests/conftest.py index 4ecf00e8b..ec72c792b 100644 --- a/pyflask/tests/conftest.py +++ b/pyflask/tests/conftest.py @@ -1,5 +1,5 @@ -import pytest import app as flask +import pytest def pytest_addoption(parser): diff --git a/pyflask/tests/test_generate_tutorial_data.py b/pyflask/tests/test_generate_tutorial_data.py index 80a5f8c53..d87e20281 100644 --- a/pyflask/tests/test_generate_tutorial_data.py +++ b/pyflask/tests/test_generate_tutorial_data.py @@ -1,6 +1,7 @@ -from utils import post from pathlib import Path +from utils import post + def test_generate_test_data(client, tmp_path: Path): # assert client is None diff --git a/pyflask/tests/test_neuroconv.py b/pyflask/tests/test_neuroconv.py index 7e6bd33e8..07563763e 100644 --- a/pyflask/tests/test_neuroconv.py +++ b/pyflask/tests/test_neuroconv.py @@ -1,5 +1,5 @@ from jsonschema import validate -from utils import get, post, get_converter_output_schema +from utils import get, get_converter_output_schema, post def test_get_all_interfaces(client): diff --git a/pyflask/tests/test_startup.py b/pyflask/tests/test_startup.py index 42cead8c7..8fa114535 100644 --- a/pyflask/tests/test_startup.py +++ b/pyflask/tests/test_startup.py @@ -1,4 +1,4 @@ -from utils import get, post, get_converter_output_schema +from utils import get, get_converter_output_schema, post def test_preload_imports(client):