From 82dee312f77c88f0007ebaa1c4d9d8de3ecf641d Mon Sep 17 00:00:00 2001 From: Peter Van Dyken Date: Mon, 19 Feb 2024 15:57:13 -0500 Subject: [PATCH 1/4] Allow custom snakebids version in app template Adds flag --snakebids-version to the snakebids create cli that takes either a version specifier or @url --- snakebids/admin.py | 49 +++++++- snakebids/jinja2_ext/format_dep_spec.py | 27 ++++ snakebids/jinja2_ext/snakebids_version.py | 2 +- snakebids/jinja2_ext/toml_encode.py | 10 ++ snakebids/project_template/copier.yaml | 3 +- ...'poetry' %}pyproject.toml{% endif %}.jinja | 2 +- ...'poetry' %}pyproject.toml{% endif %}.jinja | 2 +- snakebids/tests/test_admin.py | 116 ++++++++++++++++-- snakebids/tests/test_template.py | 30 ++++- snakebids/utils/utils.py | 6 + 10 files changed, 227 insertions(+), 20 deletions(-) create mode 100644 snakebids/jinja2_ext/format_dep_spec.py diff --git a/snakebids/admin.py b/snakebids/admin.py index 4393e963..21cc76f3 100644 --- a/snakebids/admin.py +++ b/snakebids/admin.py @@ -12,6 +12,7 @@ import snakebids from snakebids.app import SnakeBidsApp from snakebids.cli import add_dynamic_args +from snakebids.utils.utils import text_fold def create_app(args: argparse.Namespace) -> None: @@ -31,6 +32,39 @@ def create_app(args: argparse.Namespace) -> None: file=sys.stderr, ) sys.exit(1) + + data = {"app_full_name": output.name} + + if args.snakebids_version is not None: + version = args.snakebids_version + if ("@" not in version and ";" in version) or ( + "@" in version and " ;" in version + ): + print( + f"{Fore.RED}Snakebids version may not specify markers{Style.RESET_ALL}", + file=sys.stderr, + ) + sys.exit(1) + if ( + "@" in version + and version[1:].lstrip().startswith("git+") + and "@" in version[1:] + ): + print( + f"{Fore.RED}Credentials and rev specifiers in git requirement " + f"specifications are not supported{Style.RESET_ALL}", + file=sys.stderr, + ) + sys.exit(1) + if version.strip().startswith("["): + print( + f"{Fore.RED}Snakebids version may not specify extras{Style.RESET_ALL}", + file=sys.stderr, + ) + sys.exit(1) + + data["snakebids_version"] = args.snakebids_version + print( f"Creating Snakebids app at {Fore.GREEN}{output}{Fore.RESET}", file=sys.stderr ) @@ -39,7 +73,7 @@ def create_app(args: argparse.Namespace) -> None: copier.run_copy( str(Path(itx.first(snakebids.__path__), "project_template")), output, - data={"app_full_name": output.name}, + data=data, unsafe=True, ) except KeyboardInterrupt: @@ -64,6 +98,19 @@ def gen_parser() -> argparse.ArgumentParser: parser_create = subparsers.add_parser("create", help="Create a new Snakebids app.") parser_create.add_argument("output_dir", nargs="?", default=".") + parser_create.add_argument( + "--snakebids-version", + default=None, + metavar="VERSION_SPECIFIER", + help=text_fold( + """ + Specify snakebids version requirement. Supports either a valid version + specifier (e.g. `>=x.x.x`, `==a.b.c`) or a url prepended with `@` (e.g. `@ + https://...`). Paths can be specified with `@ file:///absolute/path/...`. + Markers and extras may not be specified. + """ + ), + ) parser_create.set_defaults(func=create_app) parser_boutiques = subparsers.add_parser( diff --git a/snakebids/jinja2_ext/format_dep_spec.py b/snakebids/jinja2_ext/format_dep_spec.py new file mode 100644 index 00000000..41b02453 --- /dev/null +++ b/snakebids/jinja2_ext/format_dep_spec.py @@ -0,0 +1,27 @@ +import jinja2 +from jinja2.ext import Extension + +from snakebids.jinja2_ext.toml_encode import toml_string + + +def format_poetry(item: str): + """Format a pip style dependency specification for a poetry pyproject.toml. + + Only supports urls (prefixed with @) and version specifiers. No markers or extras. + The package name should already be stripped. + """ + if item.strip().startswith("@"): + url = item[1:].strip() + if url.startswith("git+"): + return f"{{ git = {toml_string(url[4:])} }}" + if url.startswith("file://"): + return f"{{ path = {toml_string(url[7:])} }}" + return f"{{ url = {toml_string(url)} }}" + return toml_string(item) + + +class FormatDepSpec(Extension): + """Enable the toml_string filter, which encodes provided value as toml.""" + + def __init__(self, env: jinja2.Environment): + env.filters["format_poetry"] = format_poetry # type: ignore diff --git a/snakebids/jinja2_ext/snakebids_version.py b/snakebids/jinja2_ext/snakebids_version.py index 358e1c22..714bbdf5 100644 --- a/snakebids/jinja2_ext/snakebids_version.py +++ b/snakebids/jinja2_ext/snakebids_version.py @@ -15,7 +15,7 @@ class SnakebidsVersionExtension(Extension): """ def __init__(self, env: jinja2.Environment): - env.globals["snakebids_version"] = self._lookup_version() # type: ignore + env.globals["curr_snakebids_version"] = self._lookup_version() # type: ignore super().__init__(env) def _lookup_version(self): diff --git a/snakebids/jinja2_ext/toml_encode.py b/snakebids/jinja2_ext/toml_encode.py index c811e09d..e7dc19f0 100644 --- a/snakebids/jinja2_ext/toml_encode.py +++ b/snakebids/jinja2_ext/toml_encode.py @@ -13,8 +13,18 @@ def toml_string(item: str): return json.dumps(item, ensure_ascii=False).replace("\x7F", "\\u007f") +def toml_encode(item: str): + """Encode string for inclusion in toml without wrapping in quotes. + + Technically encodes as json, a (mostly) strict subset of toml, with some encoding + fixes + """ + return toml_string(item)[1:-1] + + class TomlEncodeExtension(Extension): """Enable the toml_string filter, which encodes provided value as toml.""" def __init__(self, env: jinja2.Environment): env.filters["toml_string"] = toml_string # type: ignore + env.filters["toml_encode"] = toml_encode # type: ignore diff --git a/snakebids/project_template/copier.yaml b/snakebids/project_template/copier.yaml index 084f44de..566ce977 100644 --- a/snakebids/project_template/copier.yaml +++ b/snakebids/project_template/copier.yaml @@ -69,7 +69,7 @@ python_version: when: false snakebids_version: - default: "{{ snakebids_version or '0.9.3' }}" + default: ">={{ curr_snakebids_version or '0.9.3' }}" when: false snakemake_version: @@ -128,3 +128,4 @@ _jinja_extensions: - snakebids.jinja2_ext.colorama.ColoramaExtension - snakebids.jinja2_ext.toml_encode.TomlEncodeExtension - snakebids.jinja2_ext.snakebids_version.SnakebidsVersionExtension + - snakebids.jinja2_ext.format_dep_spec.FormatDepSpec diff --git a/snakebids/project_template/{% if build_system != 'poetry' %}pyproject.toml{% endif %}.jinja b/snakebids/project_template/{% if build_system != 'poetry' %}pyproject.toml{% endif %}.jinja index 049b5671..f46c567e 100644 --- a/snakebids/project_template/{% if build_system != 'poetry' %}pyproject.toml{% endif %}.jinja +++ b/snakebids/project_template/{% if build_system != 'poetry' %}pyproject.toml{% endif %}.jinja @@ -29,7 +29,7 @@ classifiers = [ requires-python = "{{ python_version }}" dependencies = [ "snakemake >= {{ snakemake_version }},<8", - "snakebids >= {{ snakebids_version }}", + "snakebids {{ snakebids_version | toml_encode }}", {#- newer pulps are incompatible with old snakemakes, and we need to support old snakemakes for python versions <3.11. So cap pulp to the last working version diff --git a/snakebids/project_template/{% if build_system == 'poetry' %}pyproject.toml{% endif %}.jinja b/snakebids/project_template/{% if build_system == 'poetry' %}pyproject.toml{% endif %}.jinja index a890f0ed..0ebb9513 100644 --- a/snakebids/project_template/{% if build_system == 'poetry' %}pyproject.toml{% endif %}.jinja +++ b/snakebids/project_template/{% if build_system == 'poetry' %}pyproject.toml{% endif %}.jinja @@ -23,7 +23,7 @@ classifiers = [ [tool.poetry.dependencies] python = "{{ python_version }}" snakemake = ">={{ snakemake_version }},<8" -snakebids = ">={{ snakebids_version }}" +snakebids = {{ snakebids_version | format_poetry }} {#- newer pulps are incompatible with old snakemakes, and we need to support old snakemakes for python versions <3.11. So cap pulp to the last working version diff --git a/snakebids/tests/test_admin.py b/snakebids/tests/test_admin.py index 441a6808..a82c3336 100644 --- a/snakebids/tests/test_admin.py +++ b/snakebids/tests/test_admin.py @@ -1,8 +1,11 @@ +from __future__ import annotations + import re import sys from argparse import ArgumentParser, Namespace from pathlib import Path +import more_itertools as itx import pytest from hypothesis import given from hypothesis import strategies as st @@ -18,21 +21,19 @@ def parser(): return gen_parser() -class TestAdminCli: - def test_fails_if_no_subcommand( - self, parser: ArgumentParser, mocker: MockerFixture - ): - mocker.patch.object(sys, "argv", ["snakebids"]) - with pytest.raises(SystemExit): - parser.parse_args() +def test_fails_if_no_subcommand(parser: ArgumentParser, mocker: MockerFixture): + mocker.patch.object(sys, "argv", ["snakebids"]) + with pytest.raises(SystemExit): + parser.parse_args() + + +def test_fails_if_invalid_subcommand(parser: ArgumentParser, mocker: MockerFixture): + mocker.patch.object(sys, "argv", ["snakebids", "dummy"]) + with pytest.raises(SystemExit): + parser.parse_args() - def test_fails_if_invalid_subcommand( - self, parser: ArgumentParser, mocker: MockerFixture - ): - mocker.patch.object(sys, "argv", ["snakebids", "dummy"]) - with pytest.raises(SystemExit): - parser.parse_args() +class TestCreateCommand: @given( name=st.text() .filter(lambda s: not re.match(r"^[a-zA-Z_][a-zA-Z_0-9]*$", s)) @@ -79,6 +80,95 @@ def test_create_fails_missing_parent_dir( assert "does not exist" in capture.err assert str(path.parent) in capture.err + def test_create_fails_when_markers_in_snakebids_version( + self, + parser: ArgumentParser, + mocker: MockerFixture, + capsys: pytest.CaptureFixture[str], + ): + mocker.patch.object( + sys, + "argv", + ["snakebids", "create", "name", "--snakebids-version", "... ; markers"], + ) + args = parser.parse_args() + with pytest.raises(SystemExit): + args.func(args) + capture = capsys.readouterr() + assert "may not specify markers" in capture.err + + def test_create_fails_when_get_spec_has_at_sign( + self, + parser: ArgumentParser, + mocker: MockerFixture, + capsys: pytest.CaptureFixture[str], + ): + mocker.patch.object( + sys, + "argv", + ["snakebids", "create", "name", "--snakebids-version", "@ git+...@...@..."], + ) + args = parser.parse_args() + with pytest.raises(SystemExit): + args.func(args) + capture = capsys.readouterr() + assert "in git requirement" in capture.err + + def test_create_fails_when_snakebids_version_specifies_extras( + self, + parser: ArgumentParser, + mocker: MockerFixture, + capsys: pytest.CaptureFixture[str], + ): + mocker.patch.object( + sys, + "argv", + ["snakebids", "create", "name", "--snakebids-version", "[...] ..."], + ) + args = parser.parse_args() + with pytest.raises(SystemExit): + args.func(args) + capture = capsys.readouterr() + assert "may not specify extras" in capture.err + + @given( + name=st.from_regex(r"^[a-zA-Z_][a-zA-Z_0-9]*$"), + version=st.text(st.characters(blacklist_characters=["@", ";", "["])) + | st.none(), + ) + @allow_function_scoped + def test_create_calls_copier_correctly( + self, + parser: ArgumentParser, + mocker: MockerFixture, + name: str, + version: str | None, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + ): + import snakebids + from snakebids.admin import copier + + run_copy = mocker.patch.object(copier, "run_copy") + path = tmp_path / name + args = ["snakebids", "create", str(path)] + if version is not None: + args.extend(["--snakebids-version", version]) + mocker.patch.object(sys, "argv", args) + args = parser.parse_args() + args.func(args) + capture = capsys.readouterr() + assert "Creating Snakebids app at" in capture.err + run_copy.assert_called_once_with( + str(Path(itx.first(snakebids.__path__), "project_template")), + path, + data={ + "app_full_name": name, + **({"snakebids_version": version} if version is not None else {}), + }, + unsafe=True, + ) + def test_create_succeeds(self, parser: ArgumentParser, mocker: MockerFixture): mocker.patch.object(sys, "argv", ["snakebids", "create"]) assert isinstance(parser.parse_args(), Namespace) diff --git a/snakebids/tests/test_template.py b/snakebids/tests/test_template.py index 33f7bc13..9d2520ea 100644 --- a/snakebids/tests/test_template.py +++ b/snakebids/tests/test_template.py @@ -14,10 +14,12 @@ import prompt_toolkit.validation import pytest import requests_mock -from hypothesis import HealthCheck, given, settings +from hypothesis import HealthCheck, assume, given, settings from hypothesis import strategies as st from pytest_mock.plugin import MockerFixture -from typing_extensions import Unpack +from typing_extensions import NotRequired, Unpack + +from snakebids.jinja2_ext.format_dep_spec import format_poetry if sys.version_info < (3, 11): import tomli as tomllib @@ -41,6 +43,7 @@ class DataFields(TypedDict): app_version: str create_doc_template: bool license: str + snakebids_version: NotRequired[str] def get_empty_data(app_name: str, build: BuildBackend) -> DataFields: @@ -104,6 +107,21 @@ def test_invalid_email_raises_error(email: str, tmp_path: Path): ) +@pytest.mark.parametrize("spec_type", ["version", "url", "path", "git"]) +@given(spec=st.text().map(lambda s: s.strip()).filter(lambda s: not s.startswith("@"))) +def test_format_poetry(spec_type: Literal["version", "url", "path", "git"], spec: str): + prefix = {"url": "@ ", "path": "@ file://", "git": "@ git+", "version": ""}[ + spec_type + ] + if spec_type == "url": + assume("file://" not in spec and "git+" not in spec) + parsed = tomllib.loads("spec = " + format_poetry(prefix + spec))["spec"] + if spec_type == "version": + assert parsed == spec + else: + assert parsed[spec_type] == spec + + @pytest.mark.parametrize("build", BUILD_BACKENDS) @pytest.mark.parametrize( ("server", "server_status", "metadata", "expected"), @@ -209,6 +227,7 @@ def test_correct_build_system_used( app_version=st.text(min_size=1), create_doc_template=st.just(False), license=st.text(), + snakebids_version=st.text().filter(lambda s: not s.strip().startswith("@")), ) @settings(suppress_health_check=[HealthCheck.function_scoped_fixture], deadline=5000) @pytest.mark.parametrize("build", BUILD_BACKENDS) @@ -242,6 +261,10 @@ def test_pyproject_correctly_formatted( ) else: assert "authors" not in pyproject["tool"]["poetry"] + assert ( + pyproject["tool"]["poetry"]["dependencies"]["snakebids"] + == kwargs["snakebids_version"] # type: ignore + ) return assert kwargs["app_full_name"] == pyproject["project"]["name"] @@ -261,6 +284,9 @@ def test_pyproject_correctly_formatted( assert author_obj == pyproject["project"]["authors"][0] else: assert "authors" not in pyproject["project"] + assert pyproject["project"]["dependencies"].index( + f"snakebids {kwargs['snakebids_version']}" # type: ignore + ) @needs_docker(f"snakebids/test-template:{platform.python_version()}") diff --git a/snakebids/utils/utils.py b/snakebids/utils/utils.py index aea360b3..268a98e2 100644 --- a/snakebids/utils/utils.py +++ b/snakebids/utils/utils.py @@ -5,6 +5,7 @@ import operator as op import os import re +import textwrap from os import PathLike from pathlib import Path from typing import Any, Callable, Iterable, Mapping, Protocol, Sequence, TypeVar, cast @@ -322,3 +323,8 @@ def get_wildcard_dict(entities: str | Iterable[str], /) -> dict[str, str]: def entity_to_wildcard(entities: str | Iterable[str], /): """Turn entity strings into wildcard dicts as {"entity": "{entity}"}.""" return {entity: f"{{{entity}}}" for entity in itx.always_iterable(entities)} + + +def text_fold(text: str): + """Fold a block of text into a single line as in yaml folded multiline string.""" + return " ".join(textwrap.dedent(text).strip().splitlines()) From aef72cf6af17d383143f62fb3437639d952bbe04 Mon Sep 17 00:00:00 2001 From: Peter Van Dyken Date: Mon, 19 Feb 2024 16:07:12 -0500 Subject: [PATCH 2/4] Update template tests to use current repository The dry run tests now specify the development repository as the snakebids version to be installed, rather than pulling from pip --- containers/test-template/test-template.sh | 6 ++-- snakebids/tests/test_template.py | 44 ++++------------------- 2 files changed, 9 insertions(+), 41 deletions(-) diff --git a/containers/test-template/test-template.sh b/containers/test-template/test-template.sh index f2b2a84c..27dea773 100755 --- a/containers/test-template/test-template.sh +++ b/containers/test-template/test-template.sh @@ -11,9 +11,9 @@ case "$method" in "setuptools" ) python -m venv .venv .venv/bin/python -m pip install . - if [ -d /src ]; then - .venv/bin/python -m pip install /src - fi + # if [ -d /src ]; then + # .venv/bin/python -m pip install /src + # fi PATH=".venv/bin:$PATH" eval "$script" ;; "poetry" ) diff --git a/snakebids/tests/test_template.py b/snakebids/tests/test_template.py index 9d2520ea..b7e6d7ce 100644 --- a/snakebids/tests/test_template.py +++ b/snakebids/tests/test_template.py @@ -299,9 +299,12 @@ def test_pyproject_correctly_formatted( ("flit", "setuptools"), ], ) -def test_template_dry_runs_successfully(tmp_path: Path, build: BuildBackend, venv: str): +def test_template_dry_runs_successfully( + tmp_path: Path, request: pytest.FixtureRequest, build: BuildBackend, venv: str +): app_name = "snakebids_app" data = get_empty_data(app_name, build) + data["snakebids_version"] = "@ file:///src" copier.run_copy( str(Path(itx.first(snakebids.__path__), "project_template")), @@ -315,6 +318,8 @@ def test_template_dry_runs_successfully(tmp_path: Path, build: BuildBackend, ven "run", "-v", f"{tmp_path / app_name}:/app", + "-v", + f"{request.config.rootpath}:/src", "--rm", f"snakebids/test-template:{platform.python_version()}", venv, @@ -330,40 +335,3 @@ def test_template_dry_runs_successfully(tmp_path: Path, build: BuildBackend, ven print(cmd.stderr, file=sys.stderr) raise assert "All set" in cmd.stdout.decode() - - -def test_template_dry_runs_with_current_repository( - tmp_path: Path, request: pytest.FixtureRequest -): - app_name = "snakebids_app" - data = get_empty_data(app_name, "setuptools") - - copier.run_copy( - str(Path(itx.first(snakebids.__path__), "project_template")), - tmp_path / app_name, - data=data, - unsafe=True, - ) - cmd = sp.run( - [ - "docker", - "run", - "-v", - f"{tmp_path / app_name}:/app", - "-v", - f"{request.config.rootpath}:/src", - "--rm", - f"snakebids/test-template:{platform.python_version()}", - "setuptools", - app_name, - ], - capture_output=True, - check=False, - ) - try: - cmd.check_returncode() - except Exception: - print(cmd.stdout.decode()) - print(cmd.stderr.decode(), file=sys.stderr) - raise - assert "All set" in cmd.stdout.decode() From 1a90ab65ca18919e32338ee2411a8c29e29e04c5 Mon Sep 17 00:00:00 2001 From: Peter Van Dyken Date: Mon, 19 Feb 2024 16:17:29 -0500 Subject: [PATCH 3/4] Remove exports from jinja2_ext.__init__ --- snakebids/jinja2_ext/__init__.py | 10 ++-------- snakebids/jinja2_ext/__init__.pyi | 24 +----------------------- 2 files changed, 3 insertions(+), 31 deletions(-) diff --git a/snakebids/jinja2_ext/__init__.py b/snakebids/jinja2_ext/__init__.py index eb375a46..07cad78f 100644 --- a/snakebids/jinja2_ext/__init__.py +++ b/snakebids/jinja2_ext/__init__.py @@ -1,16 +1,10 @@ # type: ignore +__submodules__ = [] # import lazy_loader __getattr__, __dir__, __all__ = lazy_loader.attach_stub(__name__, __file__) -__all__ = [ - "ColoramaExtension", - "GitConfigExtension", - "SnakebidsVersionExtension", - "TomlEncodeExtension", - "executable", - "toml_string", -] +__all__ = [] # diff --git a/snakebids/jinja2_ext/__init__.pyi b/snakebids/jinja2_ext/__init__.pyi index c87881c4..a9a2c5b3 100644 --- a/snakebids/jinja2_ext/__init__.pyi +++ b/snakebids/jinja2_ext/__init__.pyi @@ -1,23 +1 @@ -from .colorama import ( - ColoramaExtension, -) -from .snakebids_version import ( - SnakebidsVersionExtension, -) -from .toml_encode import ( - TomlEncodeExtension, - toml_string, -) -from .vcs import ( - GitConfigExtension, - executable, -) - -__all__ = [ - "ColoramaExtension", - "GitConfigExtension", - "SnakebidsVersionExtension", - "TomlEncodeExtension", - "executable", - "toml_string", -] +__all__ = [] From f25478b126623b3ff300809ae12ee341dae07b17 Mon Sep 17 00:00:00 2001 From: Peter Van Dyken Date: Mon, 19 Feb 2024 17:01:17 -0500 Subject: [PATCH 4/4] Add direct-reference specifier for hatch When snakebids version is directly specified as a path, we need this configured for hatch for the install to work --- snakebids/project_template/copier.yaml | 4 ++++ ...ild_system != 'poetry' %}pyproject.toml{% endif %}.jinja | 6 ++++++ snakebids/tests/test_template.py | 2 +- 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/snakebids/project_template/copier.yaml b/snakebids/project_template/copier.yaml index 566ce977..977fddd0 100644 --- a/snakebids/project_template/copier.yaml +++ b/snakebids/project_template/copier.yaml @@ -72,6 +72,10 @@ snakebids_version: default: ">={{ curr_snakebids_version or '0.9.3' }}" when: false +snakebids_is_direct_reference: + default: "{{ snakebids_version.strip().startswith('@') }}" + when: false + snakemake_version: default: "7.20" when: false diff --git a/snakebids/project_template/{% if build_system != 'poetry' %}pyproject.toml{% endif %}.jinja b/snakebids/project_template/{% if build_system != 'poetry' %}pyproject.toml{% endif %}.jinja index f46c567e..b72a073a 100644 --- a/snakebids/project_template/{% if build_system != 'poetry' %}pyproject.toml{% endif %}.jinja +++ b/snakebids/project_template/{% if build_system != 'poetry' %}pyproject.toml{% endif %}.jinja @@ -51,3 +51,9 @@ build-backend = "hatchling.build" requires = ["setuptools"] build-backend = "setuptools.build_meta" {%- endif %} + +{%- if snakebids_is_direct_reference == "True" and build_system == "hatch" %} + +[tool.hatch.metadata] +allow-direct-references = true +{%- endif %} diff --git a/snakebids/tests/test_template.py b/snakebids/tests/test_template.py index b7e6d7ce..91b10419 100644 --- a/snakebids/tests/test_template.py +++ b/snakebids/tests/test_template.py @@ -173,7 +173,7 @@ def test_gets_correct_snakebids_version( pyproject["tool"]["poetry"]["dependencies"]["snakebids"] == f">={expected}" ) else: - assert f"snakebids >= {expected}" in pyproject["project"]["dependencies"] + assert f"snakebids >={expected}" in pyproject["project"]["dependencies"] @given(