diff --git a/.github/workflows/dailies.yml b/.github/workflows/dailies.yml index 4518ff772..0a579039a 100644 --- a/.github/workflows/dailies.yml +++ b/.github/workflows/dailies.yml @@ -26,8 +26,8 @@ jobs: notify: runs-on: ubuntu-latest - needs: [run-daily-tests, run-daily-doc-link-checks] - if: ${{ always() && contains(needs.*.result, 'failure') }} + needs: [run-daily-tests, run-daily-doc-link-checks, test-dandi-latest, test-dandi-dev, test-dandi-dev-live] + if: failure() steps: - uses: dawidd6/action-send-mail@v3 with: @@ -36,6 +36,6 @@ jobs: username: ${{ secrets.MAIL_USERNAME }} password: ${{ secrets.MAIL_PASSWORD }} subject: NWB Inspector Daily Failure - to: ${{ secrets.DAILY_FAILURE_EMAIL_LIST }} # add more with commas, no separation + to: cody.c.baker.phd@gmail.com,smprince@lbl.gov from: NWB Inspector body: "The daily workflow for the NWB Inspector failed: please check status at https://github.com/NeurodataWithoutBorders/nwbinspector/actions/workflows/dailies.yml" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 213095446..cc3feb660 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -10,6 +10,12 @@ repos: hooks: - id: black +- repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.6.2 + hooks: + - id: ruff + args: [ --fix ] + - repo: https://github.com/codespell-project/codespell rev: v2.3.0 hooks: diff --git a/CHANGELOG.md b/CHANGELOG.md index 389ba0455..f3d243ec0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,9 +8,13 @@ ### Pending Deprecation (API) * The `stream` and `version_id` arguments have been removed from `nwbinspector.inspect_all`. Please use `nwbinspector.inspect_dandiset` instead. [#490](https://github.com/NeurodataWithoutBorders/nwbinspector/pull/490) -## New Features +### New Features * Introduced the `inspect_dandiset` and `inspect_dandi_file_path` API functions to replace the functionality in `nwbinspector --stream`. The new feature uses `remfile` instead of `ros3`. [#490](https://github.com/NeurodataWithoutBorders/nwbinspector/pull/490) +### Fixes +* Fixed import error when using the CLI with `--config dandi`. [#494](https://github.com/NeurodataWithoutBorders/nwbinspector/issues/494) +* Removed unused imports throughout package. [#496](https://github.com/NeurodataWithoutBorders/nwbinspector/issues/496) + # v0.5.0 diff --git a/pyproject.toml b/pyproject.toml index 9f18a76f8..9231896ca 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,6 +21,27 @@ extend-exclude = ''' )/ ''' + + +[tool.ruff] +exclude = ["docs/*"] + +[tool.ruff.lint] +select = ["F401", "I002"] # TODO: eventually, expand to other 'I', 'D', and other 'F' linting +fixable = ["ALL"] + +[tool.ruff.lint.per-file-ignores] +"src/nwbinspector/__init__.py" = ["I"] # Must maintain explicit order for imports for check registration +"src/nwbinspector/utils/__init__.py" = ["F401", "I"] # Several items planned for long-term deprecation + +# Temporarily keeping all exposure below for back-compatibility +# Deprecation scheduled for 9/15/2024 +"src/nwbinspector/nwbinspector/__init__.py" = ["F401", "I"] +"src/nwbinspector/inspector_tools/__init__.py" = ["F401", "I"] +"src/nwbinspector/version/__init__.py" = ["F401", "I"] +"src/nwbinspector/register_checks/__init__.py" = ["F401", "I"] + + [tool.codespell] skip = '.git*,*.pdf,*.css' check-hidden = true diff --git a/src/nwbinspector/__init__.py b/src/nwbinspector/__init__.py index f27e518ea..20d81ee1c 100644 --- a/src/nwbinspector/__init__.py +++ b/src/nwbinspector/__init__.py @@ -30,6 +30,7 @@ "Importance", "Severity", "InspectorMessage", + "validate_config", "load_config", "configure_checks", "InspectorOutputJSONEncoder", diff --git a/src/nwbinspector/_configuration.py b/src/nwbinspector/_configuration.py index 17daeca2e..24530f109 100644 --- a/src/nwbinspector/_configuration.py +++ b/src/nwbinspector/_configuration.py @@ -3,15 +3,13 @@ import json import jsonschema from pathlib import Path -from enum import Enum from typing import Optional, List from types import FunctionType -from packaging.version import Version import yaml from . import available_checks -from ._registration import InspectorMessage, Importance +from ._registration import Importance from nwbinspector.utils._utils import ( PathType, ) diff --git a/src/nwbinspector/_inspection_cli.py b/src/nwbinspector/_inspection_cli.py new file mode 100644 index 000000000..ef3150f17 --- /dev/null +++ b/src/nwbinspector/_inspection_cli.py @@ -0,0 +1,147 @@ +"""Primary functions for inspecting NWBFiles.""" + +import os +import re +import json +from pathlib import Path +from typing import Optional +from warnings import warn + +import click + +from ._formatting import _get_report_header, InspectorOutputJSONEncoder +from ._types import Importance +from ._nwb_inspection import inspect_all +from ._formatting import format_messages, print_to_console, save_report, InspectorOutputJSONEncoder +from ._version import __version__ +from ._configuration import load_config +from .utils import strtobool + + +@click.command() +@click.argument("path") +@click.option("--modules", help="Modules to import prior to reading the file(s).") +@click.option( + "--report-file-path", + default=None, + help="Save path for the report file.", + type=click.Path(writable=True), +) +@click.option("--levels", help="Comma-separated names of InspectorMessage attributes to organize by.") +@click.option( + "--reverse", help="Comma-separated booleans corresponding to reversing the order for each value of 'levels'." +) +@click.option("--overwrite", help="Overwrite an existing report file at the location.", is_flag=True) +@click.option("--ignore", help="Comma-separated names of checks to skip.") +@click.option("--select", help="Comma-separated names of checks to run.") +@click.option( + "--threshold", + default="BEST_PRACTICE_SUGGESTION", + type=click.Choice(["CRITICAL", "BEST_PRACTICE_VIOLATION", "BEST_PRACTICE_SUGGESTION"]), + help="Ignores tests with an assigned importance below this threshold.", +) +@click.option("--config", help="Name of config or path of config .yaml file that overwrites importance of checks.") +@click.option("--json-file-path", help="Write json output to this location.") +@click.option("--n-jobs", help="Number of jobs to use in parallel.", default=1) +@click.option("--skip-validate", help="Skip the PyNWB validation step.", is_flag=True) +@click.option( + "--detailed", + help=( + "If file_path is the last of 'levels' (the default), identical checks will be aggregated in the display. " + "Use '--detailed' to see the complete report." + ), + is_flag=True, +) +@click.option("--progress-bar", help="Set this flag to False to disable display of the progress bar.") +@click.option( + "--stream", + help=( + "Stream data from the DANDI archive. If the 'path' is a local copy of the target DANDISet, specifying this " + "flag will still force the data to be streamed instead of using the local copy. To use the local copy, simply " + "remove this flag. Requires the Read Only S3 (ros3) driver to be installed with h5py." + ), + is_flag=True, +) +@click.option( + "--version-id", + help=( + "When 'path' is a six-digit DANDISet ID, this further specifies which version of " "the DANDISet to inspect." + ), +) +@click.version_option(__version__) +def _inspect_all_cli( + path: str, + modules: Optional[str] = None, + report_file_path: str = None, + levels: str = None, + reverse: Optional[str] = None, + overwrite: bool = False, + ignore: Optional[str] = None, + select: Optional[str] = None, + threshold: str = "BEST_PRACTICE_SUGGESTION", + config: Optional[str] = None, + json_file_path: Optional[str] = None, + n_jobs: int = 1, + skip_validate: bool = False, + detailed: bool = False, + progress_bar: Optional[str] = None, + stream: bool = False, + version_id: Optional[str] = None, +): + """ + Run the NWBInspector via the command line. + + path : + Path to either a local NWBFile, a local folder containing NWBFiles, a link to a dataset on + DANDI archive (i.e., https://dandiarchive.org/dandiset/{dandiset_id}/{version_id}), or a six-digit Dandiset ID. + """ + levels = ["importance", "file_path"] if levels is None else levels.split(",") + modules = [] if modules is None else modules.split(",") + reverse = [False] * len(levels) if reverse is None else [strtobool(x) for x in reverse.split(",")] + progress_bar = strtobool(progress_bar) if progress_bar is not None else True + if config is not None: + config = load_config(filepath_or_keyword=config) + if stream: + url_path = path if path.startswith("https://") else None + if url_path: + dandiset_id, version_id = url_path.split("/")[-2:] + path = dandiset_id + assert url_path or re.fullmatch( + pattern="^[0-9]{6}$", string=path + ), "'--stream' flag was enabled, but 'path' is neither a full link to the DANDI archive nor a DANDISet ID." + if Path(path).is_dir(): + warn( + f"The local DANDISet '{path}' exists, but the '--stream' flag was used. " + "NWBInspector will use S3 streaming from DANDI. To use local data, remove the '--stream' flag." + ) + messages = list( + inspect_all( + path=path, + modules=modules, + ignore=ignore if ignore is None else ignore.split(","), + select=select if select is None else select.split(","), + importance_threshold=Importance[threshold], + config=config, + n_jobs=n_jobs, + skip_validate=skip_validate, + progress_bar=progress_bar, + stream=stream, + version_id=version_id, + ) + ) + if json_file_path is not None: + if Path(json_file_path).exists() and not overwrite: + raise FileExistsError(f"The file {json_file_path} already exists! Specify the '-o' flag to overwrite.") + with open(file=json_file_path, mode="w") as fp: + json_report = dict(header=_get_report_header(), messages=messages) + json.dump(obj=json_report, fp=fp, cls=InspectorOutputJSONEncoder) + print(f"{os.linesep*2}Report saved to {str(Path(json_file_path).absolute())}!{os.linesep}") + formatted_messages = format_messages(messages=messages, levels=levels, reverse=reverse, detailed=detailed) + print_to_console(formatted_messages=formatted_messages) + if report_file_path is not None: + save_report(report_file_path=report_file_path, formatted_messages=formatted_messages, overwrite=overwrite) + print(f"{os.linesep*2}Report saved to {str(Path(report_file_path).absolute())}!{os.linesep}") + + +if __name__ == "__main__": + _inspect_all_cli() diff --git a/src/nwbinspector/_registration.py b/src/nwbinspector/_registration.py index 40ad7e734..e736301fc 100644 --- a/src/nwbinspector/_registration.py +++ b/src/nwbinspector/_registration.py @@ -1,9 +1,6 @@ """Primary decorator used on a check function to add it to the registry and automatically parse its output.""" -from collections.abc import Iterable from functools import wraps -from enum import Enum -from dataclasses import dataclass from typing import Optional import h5py diff --git a/src/nwbinspector/_types.py b/src/nwbinspector/_types.py index 623400f1c..6e920a965 100644 --- a/src/nwbinspector/_types.py +++ b/src/nwbinspector/_types.py @@ -1,16 +1,9 @@ """Primary decorator used on a check function to add it to the registry and automatically parse its output.""" -from collections.abc import Iterable -from functools import wraps from enum import Enum from dataclasses import dataclass from typing import Optional -import h5py -from pynwb import NWBFile -from pynwb.file import Subject -from pynwb.ecephys import Device, ElectrodeGroup - class Importance(Enum): """A definition of the valid importance levels for a given check function."""