Merge pull request #684 from maresb/pep-508-environment-markers

Add support for PEP 508 environment markers

conda_lock/interfaces/vendored_poetry_markers.py

@@ -0,0 +1,20 @@
+from conda_lock._vendor.poetry.core.version.markers import (
+ AnyMarker,
+ BaseMarker,
+ EmptyMarker,
+ MarkerUnion,
+ MultiMarker,
+ SingleMarker,
+ parse_marker,
+)
+
+
+__all__ = [
+ "AnyMarker",
+ "BaseMarker",
+ "EmptyMarker",
+ "MarkerUnion",
+ "MultiMarker",
+ "SingleMarker",
+ "parse_marker",
+]

conda_lock/models/lock_spec.py

@@ -19,6 +19,7 @@ class _BaseDependency(StrictModel):
  manager: Literal["conda", "pip"] = "conda"
  category: str = "main"
  extras: List[str] = []
+ markers: Optional[str] = None
 
  @validator("extras")
  def sorted_extras(cls, v: List[str]) -> List[str]:
@@ -55,6 +56,7 @@ class PoetryMappedDependencySpec(StrictModel):
  url: Optional[str]
  manager: Literal["conda", "pip"]
  extras: List
+ markers: Optional[str]
  poetry_version_spec: Optional[str]
 
 

conda_lock/pypi_solver.py

@@ -74,13 +74,14 @@ class PlatformEnv(Env):
  _platform_system: Literal["Darwin", "Linux", "Windows"]
  _os_name: Literal["posix", "nt"]
  _platforms: List[str]
- _python_version: Tuple[int, ...]
+ _python_version: Optional[Tuple[int, ...]]
 
  def __init__(
  self,
- python_version: str,
+ *,
  platform: str,
  platform_virtual_packages: Optional[Dict[str, dict]] = None,
+ python_version: Optional[str] = None,
  ):
  super().__init__(path=Path(sys.prefix))
  system, arch = platform.split("-")
@@ -102,7 +103,10 @@ def __init__(
  self._platforms = ["win_amd64"]
  else:
  raise ValueError(f"Unsupported platform '{platform}'")
- self._python_version = tuple(map(int, python_version.split(".")))
+ if python_version is None:
+ self._python_version = None
+ else:
+ self._python_version = tuple(map(int, python_version.split(".")))
 
  if system == "osx":
  self._sys_platform = "darwin"
@@ -133,13 +137,19 @@ def get_supported_tags(self) -> List["Tag"]:
 
  def get_marker_env(self) -> Dict[str, str]:
  """Return the subset of info needed to match common markers"""
- return {
- "python_full_version": ".".join([str(c) for c in self._python_version]),
- "python_version": ".".join([str(c) for c in self._python_version[:2]]),
+ result: Dict[str, str] = {
  "sys_platform": self._sys_platform,
  "platform_system": self._platform_system,
  "os_name": self._os_name,
  }
+ if self._python_version is not None:
+ result["python_full_version"] = ".".join(
+ [str(c) for c in self._python_version]
+ )
+ result["python_version"] = ".".join(
+ [str(c) for c in self._python_version[:2]]
+ )
+ return result
 
 
 def _extract_glibc_version_from_virtual_packages(
@@ -529,7 +539,11 @@ def solve_pypi(
  use_latest
  )
  )
- env = PlatformEnv(python_version, platform, platform_virtual_packages)
+ env = PlatformEnv(
+ python_version=python_version,
+ platform=platform,
+ platform_virtual_packages=platform_virtual_packages,
+ )
  # find platform-specific solution (e.g. dependencies conditioned on markers)
  with s.use_environment(env):
  result = s.solve(use_latest=to_update)

conda_lock/src_parser/environment_yaml.py

@@ -8,6 +8,7 @@
 
 from conda_lock.models.lock_spec import Dependency, LockSpecification
 from conda_lock.src_parser.conda_common import conda_spec_to_versioned_dep
+from conda_lock.src_parser.markers import evaluate_marker
 from conda_lock.src_parser.selectors import filter_platform_selectors
 
 from .pyproject_toml import parse_python_requirement
@@ -69,14 +70,14 @@ def _parse_environment_file_for_platform(
  )
  continue
 
- dependencies.append(
- parse_python_requirement(
- spec,
- manager="pip",
- category=category,
- normalize_name=False,
- )
+ dependency = parse_python_requirement(
+ spec, manager="pip", category=category, normalize_name=False
  )
+ if evaluate_marker(dependency.markers, platform):
+ # The above condition will skip adding the dependency if a
+ # marker specifies a platform that doesn't match the target,
+ # e.g. sys_platform == 'win32' for a linux target.
+ dependencies.append(dependency)
 
  # ensure pip is in target env
  dependencies.append(parse_python_requirement("pip", manager="conda"))

conda_lock/src_parser/markers.py

@@ -0,0 +1,77 @@
+"""Evaluate PEP 508 environment markers.
+
+Environment markers are expressions such as `sys_platform == 'darwin'` that can
+be attached to dependency specifications.
+
+<https://www.python.org/dev/peps/pep-0508/#environment-markers>
+"""
+
+import warnings
+
+from typing import Set, Union
+
+from conda_lock.interfaces.vendored_poetry_markers import (
+ AnyMarker,
+ BaseMarker,
+ EmptyMarker,
+ MarkerUnion,
+ MultiMarker,
+ SingleMarker,
+ parse_marker,
+)
+from conda_lock.pypi_solver import PlatformEnv
+
+
+def get_names(marker: Union[BaseMarker, str]) -> Set[str]:
+ """Extract all environment marker names from a marker expression.
+
+ >>> names = get_names(
+ ... "python_version < '3.9' and os_name == 'nt' or os_name == 'posix'"
+ ... )
+ >>> sorted(names)
+ ['os_name', 'python_version']
+ """
+ if isinstance(marker, str):
+ marker = parse_marker(marker)
+ if isinstance(marker, SingleMarker):
+ return {marker.name}
+ if isinstance(marker, (MarkerUnion, MultiMarker)):
+ return set.union(*[get_names(m) for m in marker.markers])
+ if isinstance(marker, (AnyMarker, EmptyMarker)):
+ return set()
+ raise NotImplementedError(f"Unknown marker type: {marker!r}")
+
+
+def evaluate_marker(marker: Union[BaseMarker, str, None], platform: str) -> bool:
+ """Evaluate a marker expression for a given platform.
+
+ This is intended to be used for parsing lock specifications, before the Python
+ version is known, so markers like `python_version` are not supported.
+ If the marker contains any unsupported names, a warning is issued, and the
+ corresponding clause will evaluate to `True`.
+
+ >>> evaluate_marker("sys_platform == 'darwin'", "osx-arm64")
+ True
+ >>> evaluate_marker("sys_platform == 'darwin'", "linux-64")
+ False
+ >>> evaluate_marker(None, "win-64")
+ True
+
+ # Unsupported names evaluate to True
+ >>> evaluate_marker("python_version < '0' and implementation_name == 'q'", "win-64")
+ True
+ """
+ if marker is None:
+ return True
+ if isinstance(marker, str):
+ marker = parse_marker(marker)
+ env = PlatformEnv(platform=platform)
+ marker_env = env.get_marker_env()
+ names = get_names(marker)
+ supported_names = set(marker_env.keys())
+ if not names <= supported_names:
+ warnings.warn(
+ f"Marker '{marker}' contains environment markers: "
+ f"{names - supported_names}. Only {supported_names} are supported."
+ )
+ return marker.validate(marker_env)

conda_lock/src_parser/pyproject_toml.py

@@ -40,6 +40,7 @@
  VersionedDependency,
 )
 from conda_lock.src_parser.conda_common import conda_spec_to_versioned_dep
+from conda_lock.src_parser.markers import evaluate_marker
 
 
 POETRY_INVALID_EXTRA_LOC = (
@@ -177,9 +178,12 @@ def handle_mapping(
  # or is a URL dependency, delegate to the pip section
  if depattrs.get("source", None) == "pypi" or poetry_version_spec is None:
  manager = "pip"
- # TODO: support additional features such as markers for things like sys_platform, platform_system
  return PoetryMappedDependencySpec(
- url=url, manager=manager, extras=extras, poetry_version_spec=poetry_version_spec
+ url=url,
+ manager=manager,
+ extras=extras,
+ poetry_version_spec=poetry_version_spec,
+ markers=depattrs.get("markers", None),
  )
 
 
@@ -237,6 +241,7 @@ def parse_poetry_pyproject_toml(
  url = None
  extras: List[Any] = []
  in_extra: bool = False
+ markers: Optional[str] = None
 
  # Poetry spec includes Python version in "tool.poetry.dependencies"
  # Cannot be managed by pip
@@ -264,11 +269,12 @@ def parse_poetry_pyproject_toml(
  manager,
  poetry_version_spec,
  )
- url, manager, extras, poetry_version_spec = (
+ url, manager, extras, poetry_version_spec, markers = (
  pvs.url,
  pvs.manager,
  pvs.extras,
  pvs.poetry_version_spec,
+ pvs.markers,
  )
 
  elif isinstance(depattrs, str):
@@ -297,6 +303,7 @@ def parse_poetry_pyproject_toml(
  dependencies.append(
  VCSDependency(
  name=name,
+ markers=markers,
  source=url,
  manager=manager,
  vcs="git",
@@ -312,6 +319,7 @@ def parse_poetry_pyproject_toml(
  dependencies.append(
  URLDependency(
  name=name,
+ markers=markers,
  url=url,
  hashes=[hashes],
  manager=manager,
@@ -323,6 +331,7 @@ def parse_poetry_pyproject_toml(
  dependencies.append(
  VersionedDependency(
  name=name,
+ markers=markers,
  version=version,
  manager=manager,
  category=category,
@@ -369,8 +378,13 @@ def specification_with_dependencies(
  ["tool", "conda-lock", "pip-repositories"], toml_contents, []
  )
 
+ platform_specific_dependencies: Dict[str, List[Dependency]] = {}
+ for platform in platforms:
+ platform_specific_dependencies[platform] = [
+ d for d in dependencies if evaluate_marker(d.markers, platform)
+ ]
  return LockSpecification(
- dependencies={platform: dependencies for platform in platforms},
+ dependencies=platform_specific_dependencies,
  channels=channels,
  pip_repositories=pip_repositories,
  sources=[path],
@@ -446,7 +460,44 @@ def parse_python_requirement(
  category: str = "main",
  normalize_name: bool = True,
 ) -> Dependency:
- """Parse a requirements.txt like requirement to a conda spec"""
+ """Parse a requirements.txt like requirement to a conda spec.
+
+ >>> parse_python_requirement("my_package") # doctest: +NORMALIZE_WHITESPACE
+ VersionedDependency(name='my-package', manager='conda', category='main', extras=[],
+ markers=None, version='*', build=None, conda_channel=None, hash=None)
+
+ >>> parse_python_requirement(
+ ... "My_Package[extra]==1.23"
+ ... ) # doctest: +NORMALIZE_WHITESPACE
+ VersionedDependency(name='my-package', manager='conda', category='main',
+ extras=['extra'], markers=None, version='==1.23', build=None,
+ conda_channel=None, hash=None)
+
+ >>> parse_python_requirement(
+ ... "conda-lock @ git+https://github.com/conda/[email protected]"
+ ... ) # doctest: +NORMALIZE_WHITESPACE
+ VCSDependency(name='conda-lock', manager='conda', category='main', extras=[],
+ markers=None, source='https://github.com/conda/conda-lock.git', vcs='git',
+ rev='v2.4.1')
+
+ >>> parse_python_requirement(
+ ... "some-package @ https://some-repository.org/some-package-1.2.3.tar.gz"
+ ... ) # doctest: +NORMALIZE_WHITESPACE
+ URLDependency(name='some-package', manager='conda', category='main', extras=[],
+ markers=None, url='https://some-repository.org/some-package-1.2.3.tar.gz',
+ hashes=[''])
+
+ >>> parse_python_requirement(
+ ... "some-package ; sys_platform == 'darwin'"
+ ... ) # doctest: +NORMALIZE_WHITESPACE
+ VersionedDependency(name='some-package', manager='conda', category='main',
+ extras=[], markers="sys_platform == 'darwin'", version='*', build=None,
+ conda_channel=None, hash=None)
+ """
+ if ";" in requirement:
+ requirement, markers = (s.strip() for s in requirement.rsplit(";", 1))
+ else:
+ markers = None
  parsed_req = parse_requirement_specifier(requirement)
  name = canonicalize_pypi_name(parsed_req.name)
  collapsed_version = str(parsed_req.specifier)
@@ -466,8 +517,10 @@ def parse_python_requirement(
  name=conda_dep_name,
  source=url,
  manager=manager,
+ category=category,
  vcs="git",
  rev=rev,
+ markers=markers,
  )
  elif parsed_req.url:
  assert conda_version in {"", "*", None}
@@ -479,6 +532,7 @@ def parse_python_requirement(
  extras=extras,
  url=url,
  hashes=[frag.replace("=", ":")],
+ markers=markers,
  )
  else:
  return VersionedDependency(
@@ -488,6 +542,7 @@ def parse_python_requirement(
  category=category,
  extras=extras,
  hash=parsed_req.hash,
+ markers=markers,
  )