Merge pull request #33 from mariusvniekerk/meta-yaml

conda_lock/conda_lock.py

@@ -31,7 +31,10 @@
 )
 
 import requests
-import yaml
+
+from conda_lock.src_parser import LockSpecification
+from conda_lock.src_parser.environment_yaml import parse_environment_file
+from conda_lock.src_parser.meta_yaml import parse_meta_yaml_file
 
 
 PathLike = Union[str, pathlib.Path]
@@ -240,34 +243,6 @@ def search_for_md5s(conda: PathLike, package_specs: List[dict], platform: str):
  found.add(name)
 
 
-def parse_environment_file(environment_file: pathlib.Path) -> Dict:
- if not environment_file.exists():
- raise FileNotFoundError(f"{environment_file} not found")
- with environment_file.open("r") as fo:
- env_yaml_data = yaml.safe_load(fo)
- # TODO: we basically ignore most of the fields for now.
- # notable pip deps are just skipped below
- specs = env_yaml_data["dependencies"]
- channels = env_yaml_data.get("channels", [])
-
- # Split out any sub spec sections from the dependencies mapping
- mapping_specs = [x for x in specs if not isinstance(x, str)]
- specs = [x for x in specs if isinstance(x, str)]
-
- # Print a warning if there are pip specs in the dependencies
- for mapping_spec in mapping_specs:
- if "pip" in mapping_spec:
- print(
- (
- "Warning, found pip deps not included in the lock file! You'll need to install "
- "them separately"
- ),
- file=sys.stderr,
- )
-
- return {"specs": specs, "channels": channels}
-
-
 def fn_to_dist_name(fn: str) -> str:
  if fn.endswith(".conda"):
  fn, _, _ = fn.partition(".conda")
@@ -279,23 +254,21 @@ def fn_to_dist_name(fn: str) -> str:
 
 
 def make_lock_files(
- conda: PathLike, platforms: List[str], channels: List[str], specs: List[str]
+ conda: PathLike, platforms: List[str], src_file: pathlib.Path,
 ):
  for plat in platforms:
  print(f"generating lockfile for {plat}", file=sys.stderr)
+ lock_spec = parse_source_file(src_file=src_file, platform=plat)
 
  dry_run_install = solve_specs_for_arch(
- conda=conda, platform=plat, channels=channels, specs=specs
+ conda=conda,
+ platform=lock_spec.platform,
+ channels=lock_spec.channels,
+ specs=lock_spec.specs,
  )
-
- env_spec = json.dumps(
- {"channels": channels, "platform": plat, "specs": sorted(specs)},
- sort_keys=True,
- )
- env_hash: "hashlib._Hash" = hashlib.sha256(env_spec.encode("utf-8"))
  with open(f"conda-{plat}.lock", "w") as fo:
  fo.write(f"# platform: {plat}\n")
- fo.write(f"# env_hash: {env_hash.hexdigest()}\n")
+ fo.write(f"# env_hash: {lock_spec.env_hash()}\n")
  fo.write("@EXPLICIT\n")
  link_actions = dry_run_install["actions"]["LINK"]
  for link in link_actions:
@@ -400,18 +373,24 @@ def parser():
  return parser
 
 
+def parse_source_file(src_file: pathlib.Path, platform: str) -> LockSpecification:
+ if src_file.name == "meta.yaml":
+ desired_env = parse_meta_yaml_file(src_file, platform)
+ else:
+ desired_env = parse_environment_file(src_file, platform)
+ return desired_env
+
+
 def run_lock(
  environment_file: pathlib.Path,
  conda_exe: Optional[str],
  platforms: Optional[List[str]] = None,
  no_mamba: bool = False,
 ) -> None:
- desired_env = parse_environment_file(environment_file)
  _conda_exe = ensure_conda(conda_exe, no_mamba=no_mamba)
  make_lock_files(
  conda=_conda_exe,
- channels=desired_env["channels"] or [],
- specs=desired_env["specs"],
+ src_file=environment_file,
  platforms=platforms or DEFAULT_PLATFORMS,
  )
 

conda_lock/src_parser/__init__.py

@@ -0,0 +1,22 @@
+import hashlib
+import json
+
+from typing import List
+
+
+class LockSpecification:
+ def __init__(self, specs: List[str], channels: List[str], platform: str):
+ self.specs = specs
+ self.channels = channels
+ self.platform = platform
+
+ def env_hash(self) -> str:
+ env_spec = json.dumps(
+ {
+ "channels": self.channels,
+ "platform": self.platform,
+ "specs": sorted(self.specs),
+ },
+ sort_keys=True,
+ )
+ return hashlib.sha256(env_spec.encode("utf-8")).hexdigest()

conda_lock/src_parser/environment_yaml.py

@@ -0,0 +1,41 @@
+import pathlib
+import sys
+
+import yaml
+
+from conda_lock.src_parser import LockSpecification
+from conda_lock.src_parser.selectors import filter_platform_selectors
+
+
+def parse_environment_file(
+ environment_file: pathlib.Path, platform: str
+) -> LockSpecification:
+ if not environment_file.exists():
+ raise FileNotFoundError(f"{environment_file} not found")
+
+ with environment_file.open("r") as fo:
+ filtered_content = "\n".join(
+ filter_platform_selectors(fo.read(), platform=platform)
+ )
+ env_yaml_data = yaml.safe_load(filtered_content)
+ # TODO: we basically ignore most of the fields for now.
+ # notable pip deps are just skipped below
+ specs = env_yaml_data["dependencies"]
+ channels = env_yaml_data.get("channels", [])
+
+ # Split out any sub spec sections from the dependencies mapping
+ mapping_specs = [x for x in specs if not isinstance(x, str)]
+ specs = [x for x in specs if isinstance(x, str)]
+
+ # Print a warning if there are pip specs in the dependencies
+ for mapping_spec in mapping_specs:
+ if "pip" in mapping_spec:
+ print(
+ (
+ "Warning, found pip deps not included in the lock file! You'll need to install "
+ "them separately"
+ ),
+ file=sys.stderr,
+ )
+
+ return LockSpecification(specs=specs, channels=channels, platform=platform)

conda_lock/src_parser/meta_yaml.py

@@ -0,0 +1,102 @@
+import pathlib
+
+import jinja2
+import yaml
+
+from conda_lock.src_parser import LockSpecification
+from conda_lock.src_parser.selectors import filter_platform_selectors
+
+
+class NullUndefined(jinja2.Undefined):
+ def __getattr__(self, key):
+ return ""
+
+ # Using any of these methods on an Undefined variable
+ # results in another Undefined variable.
+ __add__ = (
+ __radd__
+ ) = (
+ __mul__
+ ) = (
+ __rmul__
+ ) = (
+ __div__
+ ) = (
+ __rdiv__
+ ) = (
+ __truediv__
+ ) = (
+ __rtruediv__
+ ) = (
+ __floordiv__
+ ) = (
+ __rfloordiv__
+ ) = (
+ __mod__
+ ) = (
+ __rmod__
+ ) = (
+ __pos__
+ ) = (
+ __neg__
+ ) = (
+ __call__
+ ) = (
+ __getitem__
+ ) = (
+ __lt__
+ ) = (
+ __le__
+ ) = (
+ __gt__
+ ) = (
+ __ge__
+ ) = (
+ __complex__
+ ) = __pow__ = __rpow__ = lambda self, *args, **kwargs: self._return_undefined(
+ self._undefined_name
+ )
+
+ def _return_undefined(self, result_name):
+ # Record that this undefined variable was actually used.
+ return NullUndefined(
+ hint=self._undefined_hint,
+ obj=self._undefined_obj,
+ name=result_name,
+ exc=self._undefined_exception,
+ )
+
+
+def parse_meta_yaml_file(
+ meta_yaml_file: pathlib.Path, platform: str
+) -> LockSpecification:
+ """Parse a simple meta-yaml file for dependencies.
+
+ * This does not support multi-output files and will ignore all lines with selectors
+ """
+ if not meta_yaml_file.exists():
+ raise FileNotFoundError(f"{meta_yaml_file} not found")
+
+ with meta_yaml_file.open("r") as fo:
+ filtered_recipe = "\n".join(
+ filter_platform_selectors(fo.read(), platform=platform)
+ )
+ t = jinja2.Template(filtered_recipe, undefined=NullUndefined)
+ rendered = t.render()
+ meta_yaml_data = yaml.safe_load(rendered)
+ channels = meta_yaml_data.get("extra", {}).get("channels", [])
+ specs = []
+
+ def add_spec(spec):
+ if spec is None:
+ return
+ specs.append(spec)
+
+ for s in meta_yaml_data.get("requirements", {}).get("host", []):
+ add_spec(s)
+ for s in meta_yaml_data.get("requirements", {}).get("run", []):
+ add_spec(s)
+ for s in meta_yaml_data.get("test", {}).get("requires", []):
+ add_spec(s)
+
+ return LockSpecification(specs=specs, channels=channels, platform=platform)

conda_lock/src_parser/selectors.py

@@ -0,0 +1,28 @@
+import re
+
+from typing import Iterator
+
+
+def filter_platform_selectors(content: str, platform) -> Iterator[str]:
+ """"""
+ # we support a very limited set of selectors that adhere to platform only
+ platform_sel = {
+ "linux-64": {"linux64", "unix", "linux"},
+ "linux-aarch64": {"aarch64", "unix", "linux"},
+ "linux-ppc64le": {"ppc64le", "unix", "linux"},
+ "osx-64": {"osx", "osx64", "unix"},
+ "win-64": {"win", "win64"},
+ }
+
+ # This code is adapted from conda-build
+ sel_pat = re.compile(r"(.+?)\s*(#.*)?\[([^\[\]]+)\](?(2)[^\(\)]*)$")
+ for line in content.splitlines(keepends=False):
+ if line.lstrip().startswith("#"):
+ continue
+ m = sel_pat.match(line)
+ if m:
+ cond = m.group(3)
+ if cond in platform_sel[platform]:
+ yield line
+ else:
+ yield line

requirements.txt

@@ -1,2 +1,3 @@
 pyyaml
 requests
+jinja2

tests/test-recipe/meta.yaml

@@ -0,0 +1,33 @@
+{% set version = "1.0.5" %}
+
+package:
+ name: foo
+ version: {{ version }}
+
+build:
+ number: 0
+ script:
+ - export PYTHONUNBUFFERED=1 # [ppc64le]
+ - {{ PYTHON }} -m pip install --no-deps --ignore-installed .
+ skip: True # [py2k]
+
+requirements:
+ build:
+ - {{ compiler('c') }}
+ - {{ compiler('cxx') }}
+ host:
+ - python
+ - pip
+ - cython >=0.28.2
+ - numpy
+ run:
+ - python
+ - {{ pin_compatible('numpy') }}
+ - python-dateutil >=2.6.1
+ - pytz >=2017.2
+ - enum34 # [py27]
+ - zlib # [unix]
+
+test:
+ requires:
+ - pytest

tests/test_conda_lock.py

@@ -6,9 +6,10 @@
 from conda_lock.conda_lock import (
  ensure_conda,
  install_conda_exe,
- parse_environment_file,
+ parse_meta_yaml_file,
  run_lock,
 )
+from conda_lock.src_parser.environment_yaml import parse_environment_file
 
 
 @pytest.fixture
@@ -21,6 +22,11 @@ def zlib_environment():
  return pathlib.Path(__file__).parent.joinpath("zlib").joinpath("environment.yml")
 
 
+@pytest.fixture
+def meta_yaml_environment():
+ return pathlib.Path(__file__).parent.joinpath("test-recipe").joinpath("meta.yaml")
+
+
 def test_ensure_conda_nopath():
  assert pathlib.Path(ensure_conda()).is_file()
 
@@ -36,9 +42,18 @@ def test_install_conda_exe():
 
 
 def test_parse_environment_file(gdal_environment):
- res = parse_environment_file(gdal_environment)
- assert all(x in res["specs"] for x in ["python >=3.7,<3.8", "gdal"])
- assert all(x in res["channels"] for x in ["conda-forge", "defaults"])
+ res = parse_environment_file(gdal_environment, "linux-64")
+ assert all(x in res.specs for x in ["python >=3.7,<3.8", "gdal"])
+ assert all(x in res.channels for x in ["conda-forge", "defaults"])
+
+
+def test_parse_meta_yaml_file(meta_yaml_environment):
+ res = parse_meta_yaml_file(meta_yaml_environment, platform="linux-64")
+ assert all(x in res.specs for x in ["python", "numpy"])
+ # Ensure that this dep specified by a python selector is ignored
+ assert "enum34" not in res.specs
+ # Ensure that this platform specific dep is included
+ assert "zlib" in res.specs
 
 
 def test_run_lock_conda(monkeypatch, zlib_environment):