From d3c82a7dcfc2dc252c107f01073d97f4242d771f Mon Sep 17 00:00:00 2001 From: Marius van Niekerk Date: Wed, 15 Jul 2020 13:07:25 -0400 Subject: [PATCH] Add support for creating a lock file from a meta.yaml --- conda_lock/conda_lock.py | 61 +++++-------- conda_lock/src_parser/__init__.py | 22 +++++ conda_lock/src_parser/environment_yaml.py | 41 +++++++++ conda_lock/src_parser/meta_yaml.py | 102 ++++++++++++++++++++++ conda_lock/src_parser/selectors.py | 28 ++++++ requirements.txt | 1 + tests/test-recipe/meta.yaml | 33 +++++++ tests/test_conda_lock.py | 23 ++++- 8 files changed, 266 insertions(+), 45 deletions(-) create mode 100644 conda_lock/src_parser/__init__.py create mode 100644 conda_lock/src_parser/environment_yaml.py create mode 100644 conda_lock/src_parser/meta_yaml.py create mode 100644 conda_lock/src_parser/selectors.py create mode 100644 tests/test-recipe/meta.yaml diff --git a/conda_lock/conda_lock.py b/conda_lock/conda_lock.py index 60a8412c3..f68841f1e 100644 --- a/conda_lock/conda_lock.py +++ b/conda_lock/conda_lock.py @@ -19,7 +19,10 @@ from typing import Dict, Iterable, List, MutableSequence, Optional, Set, Tuple, Union import requests -import yaml + +from conda_lock.src_parser import LockSpecification +from conda_lock.src_parser.environment_yaml import parse_environment_file +from conda_lock.src_parser.meta_yaml import parse_meta_yaml_file PathLike = Union[str, pathlib.Path] @@ -184,34 +187,6 @@ def search_for_md5s(conda: PathLike, package_specs: List[dict], platform: str): found.add(name) -def parse_environment_file(environment_file: pathlib.Path) -> Dict: - if not environment_file.exists(): - raise FileNotFoundError(f"{environment_file} not found") - with environment_file.open("r") as fo: - env_yaml_data = yaml.safe_load(fo) - # TODO: we basically ignore most of the fields for now. - # notable pip deps are just skipped below - specs = env_yaml_data["dependencies"] - channels = env_yaml_data.get("channels", []) - - # Split out any sub spec sections from the dependencies mapping - mapping_specs = [x for x in specs if not isinstance(x, str)] - specs = [x for x in specs if isinstance(x, str)] - - # Print a warning if there are pip specs in the dependencies - for mapping_spec in mapping_specs: - if "pip" in mapping_spec: - print( - ( - "Warning, found pip deps not included in the lock file! You'll need to install " - "them separately" - ), - file=sys.stderr, - ) - - return {"specs": specs, "channels": channels} - - def fn_to_dist_name(fn: str) -> str: if fn.endswith(".conda"): fn, _, _ = fn.partition(".conda") @@ -223,23 +198,21 @@ def fn_to_dist_name(fn: str) -> str: def make_lock_files( - conda: PathLike, platforms: List[str], channels: List[str], specs: List[str] + conda: PathLike, platforms: List[str], src_file: pathlib.Path, ): for plat in platforms: print(f"generating lockfile for {plat}", file=sys.stderr) + lock_spec = parse_source_file(src_file=src_file, platform=plat) dry_run_install = solve_specs_for_arch( - conda=conda, platform=plat, channels=channels, specs=specs + conda=conda, + platform=lock_spec.platform, + channels=lock_spec.channels, + specs=lock_spec.specs, ) - - env_spec = json.dumps( - {"channels": channels, "platform": plat, "specs": sorted(specs)}, - sort_keys=True, - ) - env_hash: "hashlib._Hash" = hashlib.sha256(env_spec.encode("utf-8")) with open(f"conda-{plat}.lock", "w") as fo: fo.write(f"# platform: {plat}\n") - fo.write(f"# env_hash: {env_hash.hexdigest()}\n") + fo.write(f"# env_hash: {lock_spec.env_hash()}\n") fo.write("@EXPLICIT\n") link_actions = dry_run_install["actions"]["LINK"] for link in link_actions: @@ -339,17 +312,23 @@ def parser(): return parser +def parse_source_file(src_file: pathlib.Path, platform: str) -> LockSpecification: + if src_file.name == "meta.yaml": + desired_env = parse_meta_yaml_file(src_file, platform) + else: + desired_env = parse_environment_file(src_file, platform) + return desired_env + + def run_lock( environment_file: pathlib.Path, conda_exe: Optional[str], platforms: Optional[List[str]] = None, ) -> None: - desired_env = parse_environment_file(environment_file) _conda_exe = ensure_conda(conda_exe) make_lock_files( conda=_conda_exe, - channels=desired_env["channels"] or [], - specs=desired_env["specs"], + src_file=environment_file, platforms=platforms or DEFAULT_PLATFORMS, ) diff --git a/conda_lock/src_parser/__init__.py b/conda_lock/src_parser/__init__.py new file mode 100644 index 000000000..0bf0ad259 --- /dev/null +++ b/conda_lock/src_parser/__init__.py @@ -0,0 +1,22 @@ +import hashlib +import json + +from typing import List + + +class LockSpecification: + def __init__(self, specs: List[str], channels: List[str], platform: str): + self.specs = specs + self.channels = channels + self.platform = platform + + def env_hash(self) -> str: + env_spec = json.dumps( + { + "channels": self.channels, + "platform": self.platform, + "specs": sorted(self.specs), + }, + sort_keys=True, + ) + return hashlib.sha256(env_spec.encode("utf-8")).hexdigest() diff --git a/conda_lock/src_parser/environment_yaml.py b/conda_lock/src_parser/environment_yaml.py new file mode 100644 index 000000000..1fd802f79 --- /dev/null +++ b/conda_lock/src_parser/environment_yaml.py @@ -0,0 +1,41 @@ +import pathlib +import sys + +import yaml + +from conda_lock.src_parser import LockSpecification +from conda_lock.src_parser.selectors import filter_platform_selectors + + +def parse_environment_file( + environment_file: pathlib.Path, platform: str +) -> LockSpecification: + if not environment_file.exists(): + raise FileNotFoundError(f"{environment_file} not found") + + with environment_file.open("r") as fo: + filtered_content = "\n".join( + filter_platform_selectors(fo.read(), platform=platform) + ) + env_yaml_data = yaml.safe_load(filtered_content) + # TODO: we basically ignore most of the fields for now. + # notable pip deps are just skipped below + specs = env_yaml_data["dependencies"] + channels = env_yaml_data.get("channels", []) + + # Split out any sub spec sections from the dependencies mapping + mapping_specs = [x for x in specs if not isinstance(x, str)] + specs = [x for x in specs if isinstance(x, str)] + + # Print a warning if there are pip specs in the dependencies + for mapping_spec in mapping_specs: + if "pip" in mapping_spec: + print( + ( + "Warning, found pip deps not included in the lock file! You'll need to install " + "them separately" + ), + file=sys.stderr, + ) + + return LockSpecification(specs=specs, channels=channels, platform=platform) diff --git a/conda_lock/src_parser/meta_yaml.py b/conda_lock/src_parser/meta_yaml.py new file mode 100644 index 000000000..9d762e25b --- /dev/null +++ b/conda_lock/src_parser/meta_yaml.py @@ -0,0 +1,102 @@ +import pathlib + +import jinja2 +import yaml + +from conda_lock.src_parser import LockSpecification +from conda_lock.src_parser.selectors import filter_platform_selectors + + +class NullUndefined(jinja2.Undefined): + def __getattr__(self, key): + return "" + + # Using any of these methods on an Undefined variable + # results in another Undefined variable. + __add__ = ( + __radd__ + ) = ( + __mul__ + ) = ( + __rmul__ + ) = ( + __div__ + ) = ( + __rdiv__ + ) = ( + __truediv__ + ) = ( + __rtruediv__ + ) = ( + __floordiv__ + ) = ( + __rfloordiv__ + ) = ( + __mod__ + ) = ( + __rmod__ + ) = ( + __pos__ + ) = ( + __neg__ + ) = ( + __call__ + ) = ( + __getitem__ + ) = ( + __lt__ + ) = ( + __le__ + ) = ( + __gt__ + ) = ( + __ge__ + ) = ( + __complex__ + ) = __pow__ = __rpow__ = lambda self, *args, **kwargs: self._return_undefined( + self._undefined_name + ) + + def _return_undefined(self, result_name): + # Record that this undefined variable was actually used. + return NullUndefined( + hint=self._undefined_hint, + obj=self._undefined_obj, + name=result_name, + exc=self._undefined_exception, + ) + + +def parse_meta_yaml_file( + meta_yaml_file: pathlib.Path, platform: str +) -> LockSpecification: + """Parse a simple meta-yaml file for dependencies. + + * This does not support multi-output files and will ignore all lines with selectors + """ + if not meta_yaml_file.exists(): + raise FileNotFoundError(f"{meta_yaml_file} not found") + + with meta_yaml_file.open("r") as fo: + filtered_recipe = "\n".join( + filter_platform_selectors(fo.read(), platform=platform) + ) + t = jinja2.Template(filtered_recipe, undefined=NullUndefined) + rendered = t.render() + meta_yaml_data = yaml.safe_load(rendered) + channels = meta_yaml_data.get("extra", {}).get("channels", []) + specs = [] + + def add_spec(spec): + if spec is None: + return + specs.append(spec) + + for s in meta_yaml_data.get("requirements", {}).get("host", []): + add_spec(s) + for s in meta_yaml_data.get("requirements", {}).get("run", []): + add_spec(s) + for s in meta_yaml_data.get("test", {}).get("requires", []): + add_spec(s) + + return LockSpecification(specs=specs, channels=channels, platform=platform) diff --git a/conda_lock/src_parser/selectors.py b/conda_lock/src_parser/selectors.py new file mode 100644 index 000000000..6bdcc4ec6 --- /dev/null +++ b/conda_lock/src_parser/selectors.py @@ -0,0 +1,28 @@ +import re + +from typing import Iterator + + +def filter_platform_selectors(content: str, platform) -> Iterator[str]: + """""" + # we support a very limited set of selectors that adhere to platform only + platform_sel = { + "linux-64": {"linux64", "unix", "linux"}, + "linux-aarch64": {"aarch64", "unix", "linux"}, + "linux-ppc64le": {"ppc64le", "unix", "linux"}, + "osx-64": {"osx", "osx64", "unix"}, + "win-64": {"win", "win64"}, + } + + # This code is adapted from conda-build + sel_pat = re.compile(r"(.+?)\s*(#.*)?\[([^\[\]]+)\](?(2)[^\(\)]*)$") + for line in content.splitlines(keepends=False): + if line.lstrip().startswith("#"): + continue + m = sel_pat.match(line) + if m: + cond = m.group(3) + if cond in platform_sel[platform]: + yield line + else: + yield line diff --git a/requirements.txt b/requirements.txt index 1c6d8b40e..29839cdef 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ pyyaml requests +jinja2 \ No newline at end of file diff --git a/tests/test-recipe/meta.yaml b/tests/test-recipe/meta.yaml new file mode 100644 index 000000000..010bfd9e1 --- /dev/null +++ b/tests/test-recipe/meta.yaml @@ -0,0 +1,33 @@ +{% set version = "1.0.5" %} + +package: + name: foo + version: {{ version }} + +build: + number: 0 + script: + - export PYTHONUNBUFFERED=1 # [ppc64le] + - {{ PYTHON }} -m pip install --no-deps --ignore-installed . + skip: True # [py2k] + +requirements: + build: + - {{ compiler('c') }} + - {{ compiler('cxx') }} + host: + - python + - pip + - cython >=0.28.2 + - numpy + run: + - python + - {{ pin_compatible('numpy') }} + - python-dateutil >=2.6.1 + - pytz >=2017.2 + - enum34 # [py27] + - zlib # [unix] + +test: + requires: + - pytest diff --git a/tests/test_conda_lock.py b/tests/test_conda_lock.py index 775e197cb..3f77e93df 100644 --- a/tests/test_conda_lock.py +++ b/tests/test_conda_lock.py @@ -6,9 +6,10 @@ from conda_lock.conda_lock import ( ensure_conda, install_conda_exe, - parse_environment_file, + parse_meta_yaml_file, run_lock, ) +from conda_lock.src_parser.environment_yaml import parse_environment_file @pytest.fixture @@ -21,6 +22,11 @@ def zlib_environment(): return pathlib.Path(__file__).parent.joinpath("zlib").joinpath("environment.yml") +@pytest.fixture +def meta_yaml_environment(): + return pathlib.Path(__file__).parent.joinpath("test-recipe").joinpath("meta.yaml") + + def test_ensure_conda_nopath(): assert pathlib.Path(ensure_conda()).is_file() @@ -36,9 +42,18 @@ def test_install_conda_exe(): def test_parse_environment_file(gdal_environment): - res = parse_environment_file(gdal_environment) - assert all(x in res["specs"] for x in ["python >=3.7,<3.8", "gdal"]) - assert all(x in res["channels"] for x in ["conda-forge", "defaults"]) + res = parse_environment_file(gdal_environment, "linux-64") + assert all(x in res.specs for x in ["python >=3.7,<3.8", "gdal"]) + assert all(x in res.channels for x in ["conda-forge", "defaults"]) + + +def test_parse_meta_yaml_file(meta_yaml_environment): + res = parse_meta_yaml_file(meta_yaml_environment, platform="linux-64") + assert all(x in res.specs for x in ["python", "numpy"]) + # Ensure that this dep specified by a python selector is ignored + assert "enum34" not in res.specs + # Ensure that this platform specific dep is included + assert "zlib" in res.specs def test_run_lock_conda(monkeypatch, zlib_environment):