From 7aba4b63e49c790336bc64399704dbc36aebd1d9 Mon Sep 17 00:00:00 2001 From: Ben Mares Date: Sun, 8 Sep 2024 12:48:25 +0200 Subject: [PATCH 1/7] Add missing category to VCSDependency --- conda_lock/src_parser/pyproject_toml.py | 1 + 1 file changed, 1 insertion(+) diff --git a/conda_lock/src_parser/pyproject_toml.py b/conda_lock/src_parser/pyproject_toml.py index a513eaed..541877b5 100644 --- a/conda_lock/src_parser/pyproject_toml.py +++ b/conda_lock/src_parser/pyproject_toml.py @@ -468,6 +468,7 @@ def parse_python_requirement( name=conda_dep_name, source=url, manager=manager, + category=category, vcs="git", rev=rev, ) From d5be43fcb4167e3c37c524f3ca20cbe562d8f48a Mon Sep 17 00:00:00 2001 From: Ben Mares Date: Sat, 7 Sep 2024 19:46:33 +0200 Subject: [PATCH 2/7] Make python_version optional in PlatformEnv This is in preparation for evaluating PEP 508 environment markers when parsing lock specification source files. In this case we know the platform we're evaluating, but we haven't yet fixed the Python version. --- conda_lock/pypi_solver.py | 28 +++++++++++++++++++++------- tests/test_conda_lock.py | 26 +++++++++++++++++++------- 2 files changed, 40 insertions(+), 14 deletions(-) diff --git a/conda_lock/pypi_solver.py b/conda_lock/pypi_solver.py index 1a67caa8..fedddf95 100644 --- a/conda_lock/pypi_solver.py +++ b/conda_lock/pypi_solver.py @@ -74,13 +74,14 @@ class PlatformEnv(Env): _platform_system: Literal["Darwin", "Linux", "Windows"] _os_name: Literal["posix", "nt"] _platforms: List[str] - _python_version: Tuple[int, ...] + _python_version: Optional[Tuple[int, ...]] def __init__( self, - python_version: str, + *, platform: str, platform_virtual_packages: Optional[Dict[str, dict]] = None, + python_version: Optional[str] = None, ): super().__init__(path=Path(sys.prefix)) system, arch = platform.split("-") @@ -102,7 +103,10 @@ def __init__( self._platforms = ["win_amd64"] else: raise ValueError(f"Unsupported platform '{platform}'") - self._python_version = tuple(map(int, python_version.split("."))) + if python_version is None: + self._python_version = None + else: + self._python_version = tuple(map(int, python_version.split("."))) if system == "osx": self._sys_platform = "darwin" @@ -133,13 +137,19 @@ def get_supported_tags(self) -> List["Tag"]: def get_marker_env(self) -> Dict[str, str]: """Return the subset of info needed to match common markers""" - return { - "python_full_version": ".".join([str(c) for c in self._python_version]), - "python_version": ".".join([str(c) for c in self._python_version[:2]]), + result: Dict[str, str] = { "sys_platform": self._sys_platform, "platform_system": self._platform_system, "os_name": self._os_name, } + if self._python_version is not None: + result["python_full_version"] = ".".join( + [str(c) for c in self._python_version] + ) + result["python_version"] = ".".join( + [str(c) for c in self._python_version[:2]] + ) + return result def _extract_glibc_version_from_virtual_packages( @@ -529,7 +539,11 @@ def solve_pypi( use_latest ) ) - env = PlatformEnv(python_version, platform, platform_virtual_packages) + env = PlatformEnv( + python_version=python_version, + platform=platform, + platform_virtual_packages=platform_virtual_packages, + ) # find platform-specific solution (e.g. dependencies conditioned on markers) with s.use_environment(env): result = s.solve(use_latest=to_update) diff --git a/tests/test_conda_lock.py b/tests/test_conda_lock.py index 4899f2b4..451e54ed 100644 --- a/tests/test_conda_lock.py +++ b/tests/test_conda_lock.py @@ -2652,18 +2652,22 @@ def test_platformenv_linux_platforms(): ] + ["linux_x86_64"] # Check that we get the default platforms when no virtual packages are specified - e = PlatformEnv("3.12", "linux-64") + e = PlatformEnv(python_version="3.12", platform="linux-64") assert e._platforms == all_expected_platforms # Check that we get the default platforms when the virtual packages are empty - e = PlatformEnv("3.12", "linux-64", platform_virtual_packages={}) + e = PlatformEnv( + python_version="3.12", platform="linux-64", platform_virtual_packages={} + ) assert e._platforms == all_expected_platforms # Check that we get the default platforms when the virtual packages are nonempty # but don't include __glibc platform_virtual_packages = {"x.bz2": {"name": "not_glibc"}} e = PlatformEnv( - "3.12", "linux-64", platform_virtual_packages=platform_virtual_packages + python_version="3.12", + platform="linux-64", + platform_virtual_packages=platform_virtual_packages, ) assert e._platforms == all_expected_platforms @@ -2672,7 +2676,9 @@ def test_platformenv_linux_platforms(): default_repodata = default_virtual_package_repodata() platform_virtual_packages = default_repodata.all_repodata["linux-64"]["packages"] e = PlatformEnv( - "3.12", "linux-64", platform_virtual_packages=platform_virtual_packages + python_version="3.12", + platform="linux-64", + platform_virtual_packages=platform_virtual_packages, ) assert e._platforms == all_expected_platforms @@ -2684,7 +2690,9 @@ def test_platformenv_linux_platforms(): if record["name"] != "__glibc" } e = PlatformEnv( - "3.12", "linux-64", platform_virtual_packages=platform_virtual_packages + python_version="3.12", + platform="linux-64", + platform_virtual_packages=platform_virtual_packages, ) assert e._platforms == all_expected_platforms @@ -2701,7 +2709,9 @@ def test_platformenv_linux_platforms(): name="__glibc", version="2.17" ) e = PlatformEnv( - "3.12", "linux-64", platform_virtual_packages=platform_virtual_packages + python_version="3.12", + platform="linux-64", + platform_virtual_packages=platform_virtual_packages, ) assert e._platforms == restricted_platforms @@ -2711,7 +2721,9 @@ def test_platformenv_linux_platforms(): ) with pytest.warns(UserWarning): e = PlatformEnv( - "3.12", "linux-64", platform_virtual_packages=platform_virtual_packages + python_version="3.12", + platform="linux-64", + platform_virtual_packages=platform_virtual_packages, ) assert e._platforms == restricted_platforms From 5e17181f57bbda834df005fcc66f27f2216be98d Mon Sep 17 00:00:00 2001 From: Ben Mares Date: Sat, 7 Sep 2024 23:45:59 +0200 Subject: [PATCH 3/7] Evaluate markers for poetry dependencies --- .../interfaces/vendored_poetry_markers.py | 20 +++++ conda_lock/models/lock_spec.py | 2 + conda_lock/src_parser/markers.py | 77 +++++++++++++++++++ conda_lock/src_parser/pyproject_toml.py | 22 +++++- 4 files changed, 117 insertions(+), 4 deletions(-) create mode 100644 conda_lock/interfaces/vendored_poetry_markers.py create mode 100644 conda_lock/src_parser/markers.py diff --git a/conda_lock/interfaces/vendored_poetry_markers.py b/conda_lock/interfaces/vendored_poetry_markers.py new file mode 100644 index 00000000..b141a09d --- /dev/null +++ b/conda_lock/interfaces/vendored_poetry_markers.py @@ -0,0 +1,20 @@ +from conda_lock._vendor.poetry.core.version.markers import ( + AnyMarker, + BaseMarker, + EmptyMarker, + MarkerUnion, + MultiMarker, + SingleMarker, + parse_marker, +) + + +__all__ = [ + "AnyMarker", + "BaseMarker", + "EmptyMarker", + "MarkerUnion", + "MultiMarker", + "SingleMarker", + "parse_marker", +] diff --git a/conda_lock/models/lock_spec.py b/conda_lock/models/lock_spec.py index db69960d..bcba92c5 100644 --- a/conda_lock/models/lock_spec.py +++ b/conda_lock/models/lock_spec.py @@ -19,6 +19,7 @@ class _BaseDependency(StrictModel): manager: Literal["conda", "pip"] = "conda" category: str = "main" extras: List[str] = [] + markers: Optional[str] = None @validator("extras") def sorted_extras(cls, v: List[str]) -> List[str]: @@ -55,6 +56,7 @@ class PoetryMappedDependencySpec(StrictModel): url: Optional[str] manager: Literal["conda", "pip"] extras: List + markers: Optional[str] poetry_version_spec: Optional[str] diff --git a/conda_lock/src_parser/markers.py b/conda_lock/src_parser/markers.py new file mode 100644 index 00000000..1f61f821 --- /dev/null +++ b/conda_lock/src_parser/markers.py @@ -0,0 +1,77 @@ +"""Evaluate PEP 508 environment markers. + +Environment markers are expressions such as `sys_platform == 'darwin'` that can +be attached to dependency specifications. + + +""" + +import warnings + +from typing import Set, Union + +from conda_lock.interfaces.vendored_poetry_markers import ( + AnyMarker, + BaseMarker, + EmptyMarker, + MarkerUnion, + MultiMarker, + SingleMarker, + parse_marker, +) +from conda_lock.pypi_solver import PlatformEnv + + +def get_names(marker: Union[BaseMarker, str]) -> Set[str]: + """Extract all environment marker names from a marker expression. + + >>> names = get_names( + ... "python_version < '3.9' and os_name == 'nt' or os_name == 'posix'" + ... ) + >>> sorted(names) + ['os_name', 'python_version'] + """ + if isinstance(marker, str): + marker = parse_marker(marker) + if isinstance(marker, SingleMarker): + return {marker.name} + if isinstance(marker, (MarkerUnion, MultiMarker)): + return set.union(*[get_names(m) for m in marker.markers]) + if isinstance(marker, (AnyMarker, EmptyMarker)): + return set() + raise NotImplementedError(f"Unknown marker type: {marker!r}") + + +def evaluate_marker(marker: Union[BaseMarker, str, None], platform: str) -> bool: + """Evaluate a marker expression for a given platform. + + This is intended to be used for parsing lock specifications, before the Python + version is known, so markers like `python_version` are not supported. + If the marker contains any unsupported names, a warning is issued, and the + corresponding clause will evaluate to `True`. + + >>> evaluate_marker("sys_platform == 'darwin'", "osx-arm64") + True + >>> evaluate_marker("sys_platform == 'darwin'", "linux-64") + False + >>> evaluate_marker(None, "win-64") + True + + # Unsupported names evaluate to True + >>> evaluate_marker("python_version < '0' and implementation_name == 'q'", "win-64") + True + """ + if marker is None: + return True + if isinstance(marker, str): + marker = parse_marker(marker) + env = PlatformEnv(platform=platform) + marker_env = env.get_marker_env() + names = get_names(marker) + supported_names = set(marker_env.keys()) + if not names <= supported_names: + warnings.warn( + f"Marker '{marker}' contains environment markers: " + f"{names - supported_names}. Only {supported_names} are supported." + ) + return marker.validate(marker_env) diff --git a/conda_lock/src_parser/pyproject_toml.py b/conda_lock/src_parser/pyproject_toml.py index 541877b5..88944319 100644 --- a/conda_lock/src_parser/pyproject_toml.py +++ b/conda_lock/src_parser/pyproject_toml.py @@ -42,6 +42,7 @@ VersionedDependency, ) from conda_lock.src_parser.conda_common import conda_spec_to_versioned_dep +from conda_lock.src_parser.markers import evaluate_marker POETRY_INVALID_EXTRA_LOC = ( @@ -179,9 +180,12 @@ def handle_mapping( # or is a URL dependency, delegate to the pip section if depattrs.get("source", None) == "pypi" or poetry_version_spec is None: manager = "pip" - # TODO: support additional features such as markers for things like sys_platform, platform_system return PoetryMappedDependencySpec( - url=url, manager=manager, extras=extras, poetry_version_spec=poetry_version_spec + url=url, + manager=manager, + extras=extras, + poetry_version_spec=poetry_version_spec, + markers=depattrs.get("markers", None), ) @@ -239,6 +243,7 @@ def parse_poetry_pyproject_toml( url = None extras: List[Any] = [] in_extra: bool = False + markers: Optional[str] = None # Poetry spec includes Python version in "tool.poetry.dependencies" # Cannot be managed by pip @@ -266,11 +271,12 @@ def parse_poetry_pyproject_toml( manager, poetry_version_spec, ) - url, manager, extras, poetry_version_spec = ( + url, manager, extras, poetry_version_spec, markers = ( pvs.url, pvs.manager, pvs.extras, pvs.poetry_version_spec, + pvs.markers, ) elif isinstance(depattrs, str): @@ -299,6 +305,7 @@ def parse_poetry_pyproject_toml( dependencies.append( VCSDependency( name=name, + markers=markers, source=url, manager=manager, vcs="git", @@ -314,6 +321,7 @@ def parse_poetry_pyproject_toml( dependencies.append( URLDependency( name=name, + markers=markers, url=url, hashes=[hashes], manager=manager, @@ -325,6 +333,7 @@ def parse_poetry_pyproject_toml( dependencies.append( VersionedDependency( name=name, + markers=markers, version=version, manager=manager, category=category, @@ -371,8 +380,13 @@ def specification_with_dependencies( ["tool", "conda-lock", "pip-repositories"], toml_contents, [] ) + platform_specific_dependencies: Dict[str, List[Dependency]] = {} + for platform in platforms: + platform_specific_dependencies[platform] = [ + d for d in dependencies if evaluate_marker(d.markers, platform) + ] return LockSpecification( - dependencies={platform: dependencies for platform in platforms}, + dependencies=platform_specific_dependencies, channels=channels, pip_repositories=pip_repositories, sources=[path], From 0997a49de71090af66468aaf9bc06575d640c3a2 Mon Sep 17 00:00:00 2001 From: Ben Mares Date: Sun, 8 Sep 2024 12:24:31 +0200 Subject: [PATCH 4/7] Add doctests to parse_python_requirement --- conda_lock/src_parser/pyproject_toml.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/conda_lock/src_parser/pyproject_toml.py b/conda_lock/src_parser/pyproject_toml.py index 88944319..3ee2b66c 100644 --- a/conda_lock/src_parser/pyproject_toml.py +++ b/conda_lock/src_parser/pyproject_toml.py @@ -462,7 +462,17 @@ def parse_python_requirement( category: str = "main", normalize_name: bool = True, ) -> Dependency: - """Parse a requirements.txt like requirement to a conda spec""" + """Parse a requirements.txt like requirement to a conda spec. + + >>> parse_python_requirement("my_package") # doctest: +NORMALIZE_WHITESPACE + VersionedDependency(name='my-package', manager='conda', category='main', extras=[], + markers=None, version='*', build=None, conda_channel=None, hash=None) + + >>> parse_python_requirement("My_Package[extra]==1.23") # doctest: +NORMALIZE_WHITESPACE + VersionedDependency(name='my-package', manager='conda', category='main', + extras=['extra'], markers=None, version='==1.23', build=None, + conda_channel=None, hash=None) + """ parsed_req = parse_requirement_specifier(requirement) name = canonicalize_pypi_name(parsed_req.name) collapsed_version = str(parsed_req.specifier) From 22d465d41cf55984abf2470a33dc8ea87134163a Mon Sep 17 00:00:00 2001 From: Ben Mares Date: Sun, 8 Sep 2024 14:25:55 +0200 Subject: [PATCH 5/7] Add support for non-Poetry environment markers --- conda_lock/src_parser/pyproject_toml.py | 32 ++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/conda_lock/src_parser/pyproject_toml.py b/conda_lock/src_parser/pyproject_toml.py index 3ee2b66c..288f00e4 100644 --- a/conda_lock/src_parser/pyproject_toml.py +++ b/conda_lock/src_parser/pyproject_toml.py @@ -468,11 +468,38 @@ def parse_python_requirement( VersionedDependency(name='my-package', manager='conda', category='main', extras=[], markers=None, version='*', build=None, conda_channel=None, hash=None) - >>> parse_python_requirement("My_Package[extra]==1.23") # doctest: +NORMALIZE_WHITESPACE + >>> parse_python_requirement( + ... "My_Package[extra]==1.23" + ... ) # doctest: +NORMALIZE_WHITESPACE VersionedDependency(name='my-package', manager='conda', category='main', extras=['extra'], markers=None, version='==1.23', build=None, conda_channel=None, hash=None) + + >>> parse_python_requirement( + ... "conda-lock @ git+https://github.com/conda/conda-lock.git@v2.4.1" + ... ) # doctest: +NORMALIZE_WHITESPACE + VCSDependency(name='conda-lock', manager='conda', category='main', extras=[], + markers=None, source='https://github.com/conda/conda-lock.git', vcs='git', + rev='v2.4.1') + + >>> parse_python_requirement( + ... "some-package @ https://some-repository.org/some-package-1.2.3.tar.gz" + ... ) # doctest: +NORMALIZE_WHITESPACE + URLDependency(name='some-package', manager='conda', category='main', extras=[], + markers=None, url='https://some-repository.org/some-package-1.2.3.tar.gz', + hashes=['']) + + >>> parse_python_requirement( + ... "some-package ; sys_platform == 'darwin'" + ... ) # doctest: +NORMALIZE_WHITESPACE + VersionedDependency(name='some-package', manager='conda', category='main', + extras=[], markers="sys_platform == 'darwin'", version='*', build=None, + conda_channel=None, hash=None) """ + if ";" in requirement: + requirement, markers = (s.strip() for s in requirement.rsplit(";", 1)) + else: + markers = None parsed_req = parse_requirement_specifier(requirement) name = canonicalize_pypi_name(parsed_req.name) collapsed_version = str(parsed_req.specifier) @@ -495,6 +522,7 @@ def parse_python_requirement( category=category, vcs="git", rev=rev, + markers=markers, ) elif parsed_req.url: assert conda_version in {"", "*", None} @@ -506,6 +534,7 @@ def parse_python_requirement( extras=extras, url=url, hashes=[frag.replace("=", ":")], + markers=markers, ) else: return VersionedDependency( @@ -515,6 +544,7 @@ def parse_python_requirement( category=category, extras=extras, hash=parsed_req.hash, + markers=markers, ) From 3f3449a4ac20ff8eeaa332dd2711e2856cee3acb Mon Sep 17 00:00:00 2001 From: Ben Mares Date: Sun, 8 Sep 2024 14:26:47 +0200 Subject: [PATCH 6/7] Add support for environment markers for pip deps in environment.yml --- conda_lock/src_parser/environment_yaml.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/conda_lock/src_parser/environment_yaml.py b/conda_lock/src_parser/environment_yaml.py index df04b131..b397032f 100644 --- a/conda_lock/src_parser/environment_yaml.py +++ b/conda_lock/src_parser/environment_yaml.py @@ -8,6 +8,7 @@ from conda_lock.models.lock_spec import Dependency, LockSpecification from conda_lock.src_parser.conda_common import conda_spec_to_versioned_dep +from conda_lock.src_parser.markers import evaluate_marker from conda_lock.src_parser.selectors import filter_platform_selectors from .pyproject_toml import parse_python_requirement @@ -69,14 +70,14 @@ def _parse_environment_file_for_platform( ) continue - dependencies.append( - parse_python_requirement( - spec, - manager="pip", - category=category, - normalize_name=False, - ) + dependency = parse_python_requirement( + spec, manager="pip", category=category, normalize_name=False ) + if evaluate_marker(dependency.markers, platform): + # The above condition will skip adding the dependency if a + # marker specifies a platform that doesn't match the target, + # e.g. sys_platform == 'win32' for a linux target. + dependencies.append(dependency) # ensure pip is in target env dependencies.append(parse_python_requirement("pip", manager="conda")) From f23de39870ff553ccf9d37512b05baa65183fce9 Mon Sep 17 00:00:00 2001 From: Ben Mares Date: Sun, 8 Sep 2024 14:28:09 +0200 Subject: [PATCH 7/7] Add test for parsing environment markers --- tests/test_markers.py | 90 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 tests/test_markers.py diff --git a/tests/test_markers.py b/tests/test_markers.py new file mode 100644 index 00000000..45c61cf8 --- /dev/null +++ b/tests/test_markers.py @@ -0,0 +1,90 @@ +from pathlib import Path + +import pytest + +from conda_lock.src_parser import make_lock_spec + + +ENVIRONMENT_YAML = """ +channels: + - conda-forge + - nodefaults +dependencies: + - pip + - pip: + - cowsay; sys_platform == 'darwin' +platforms: + - osx-64 + - osx-arm64 + - linux-64 +""" + +POETRY_PYPROJECT = """ +[tool.poetry] +name = "conda-lock-test-poetry" +version = "0.0.1" +description = "" +authors = ["conda-lock"] + +[tool.poetry.dependencies] +python = "^3.9" +cowsay = {version = "*", markers = "sys_platform == 'darwin'"} + +[build-system] +requires = ["poetry>=0.12"] +build-backend = "poetry.masonry.api" + +[tool.conda-lock] +platforms = [ + "osx-64", + "osx-arm64", + "linux-64", +] +""" + +HATCH_PYPROJECT = """ +[build-system] +requires = ["hatchling >=1.25.0,<2"] +build-backend = "hatchling.build" + +[project] +name = "conda-lock-test-hatch" +version = "0.0.1" +dependencies = ["cowsay; sys_platform == 'darwin'"] + +[tool.conda-lock] +platforms = [ + "osx-64", + "osx-arm64", + "linux-64", +] +""" + + +@pytest.fixture( + params=[ + (ENVIRONMENT_YAML, "environment.yml"), + (POETRY_PYPROJECT, "pyproject.toml"), + (HATCH_PYPROJECT, "pyproject.toml"), + ], + ids=["environment.yml", "poetry", "hatch"], +) +def cowsay_src_file(request, tmp_path: Path): + contents, filename = request.param + src_file = tmp_path / filename + src_file.write_text(contents) + return src_file + + +def test_sys_platform_marker(cowsay_src_file): + lock_spec = make_lock_spec(src_files=[cowsay_src_file]) + dependencies = lock_spec.dependencies + platform_has_cowsay = { + platform: any(dep.name == "cowsay" for dep in platform_deps) + for platform, platform_deps in dependencies.items() + } + assert platform_has_cowsay == { + "osx-64": True, + "osx-arm64": True, + "linux-64": False, + }