diff --git a/docs/docs/writing-plugins/common-plugin-tasks/index.mdx b/docs/docs/writing-plugins/common-plugin-tasks/index.mdx index 0dd0311b688..3a45674ffd8 100644 --- a/docs/docs/writing-plugins/common-plugin-tasks/index.mdx +++ b/docs/docs/writing-plugins/common-plugin-tasks/index.mdx @@ -13,5 +13,4 @@ - [Add Tests](./run-tests.mdx) - [Add lockfile support](./plugin-lockfiles.mdx) - [Custom `setup-py` kwargs](./custom-python-artifact-kwargs.mdx) -- [Plugin upgrade guide](./plugin-upgrade-guide.mdx) -- [Plugin helpers](./plugin-helpers.mdx) \ No newline at end of file +- [Plugin upgrade guide](./plugin-upgrade-guide.mdx) \ No newline at end of file diff --git a/docs/docs/writing-plugins/common-plugin-tasks/plugin-helpers.mdx b/docs/docs/writing-plugins/common-plugin-tasks/plugin-helpers.mdx deleted file mode 100644 index b1fe7cd61b1..00000000000 --- a/docs/docs/writing-plugins/common-plugin-tasks/plugin-helpers.mdx +++ /dev/null @@ -1,30 +0,0 @@ ---- - title: Plugin helpers - sidebar_position: 999 ---- - -Helpers which make writing plugins easier. - ---- - -Pants has helpers to make writing plugins easier. - -## Python - -### Lockfiles - -The lockfiles for most Python tools fit into common categories. Pants has helpers to generate the rules for lockfile generation. - -- A single Python package that could be installed with `pip install my_tool` - -```python -from pants.backend.python.subsystems.python_tool_base import ( - LockfileRules, - PythonToolBase, -) - -class Isort(PythonToolBase): - options_scope = "isort" - ... - lockfile_rules_type = LockfileRules.SIMPLE -``` diff --git a/docs/docs/writing-plugins/common-subsystem-tasks.mdx b/docs/docs/writing-plugins/common-subsystem-tasks.mdx index 3331d08c02b..d5bafb9fd73 100644 --- a/docs/docs/writing-plugins/common-subsystem-tasks.mdx +++ b/docs/docs/writing-plugins/common-subsystem-tasks.mdx @@ -49,3 +49,30 @@ class FortranLintFieldSet(FieldSet): def opt_out(cls, tgt: Target) -> bool: return tgt.get(SkipFortranLintField).value ``` + +## Making subsystems exportable with their default lockfile + +:::note Support depends on language backend of the subsystem +Only some language backends support `pants export`. These include the Python and JVM backends. Only tools which are themselves written to use a backend with this feature can be exported. For example, a Python-based tool which operates on a different language is exportable. + + +1. Make the subsystem a subclass of `ExportableTool` + +:::note Language backends may have done this in their Tool base class. For example, the Python backend with `PythonToolRequirementsBase` and JVM with `JvmToolBase` are already subclasses. + +```python +from pants.backend.python.subsystems.python_tool_base import PythonToolBase +from pants.core.goals.resolves import ExportableTool + +class FortranLint(PythonToolBase, ExportableTool): + ... +``` + +2. Register your class with a `UnionRule` with `ExportableTool` + +```python +def rules(): + return [ + UnionRule(ExportableTool, FortranLint) + ] +``` diff --git a/src/python/pants/backend/python/goals/export.py b/src/python/pants/backend/python/goals/export.py index 28daf5f916a..282dcb2476d 100644 --- a/src/python/pants/backend/python/goals/export.py +++ b/src/python/pants/backend/python/goals/export.py @@ -9,8 +9,9 @@ import uuid from dataclasses import dataclass from enum import Enum -from typing import Any +from typing import Any, cast +from pants.backend.python.subsystems.python_tool_base import PythonToolBase from pants.backend.python.subsystems.setup import PythonSetup from pants.backend.python.target_types import PexLayout from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints @@ -31,12 +32,13 @@ ExportSubsystem, PostProcessingCommand, ) +from pants.core.goals.resolves import ExportableTool from pants.engine.engine_aware import EngineAwareParameter from pants.engine.internals.native_engine import AddPrefix, Digest, MergeDigests, Snapshot from pants.engine.internals.selectors import Get, MultiGet from pants.engine.process import ProcessCacheScope, ProcessResult from pants.engine.rules import collect_rules, rule -from pants.engine.unions import UnionRule +from pants.engine.unions import UnionMembership, UnionRule from pants.option.option_types import EnumOption, StrListOption from pants.util.strutil import path_safe, softwrap @@ -307,18 +309,31 @@ async def export_virtualenv_for_resolve( request: _ExportVenvForResolveRequest, python_setup: PythonSetup, export_subsys: ExportSubsystem, + union_membership: UnionMembership, ) -> MaybeExportResult: resolve = request.resolve lockfile_path = python_setup.resolves.get(resolve) - if not lockfile_path: + if lockfile_path: + lockfile = Lockfile( + url=lockfile_path, + url_description_of_origin=f"the resolve `{resolve}`", + resolve_name=resolve, + ) + else: + maybe_exportable = ExportableTool.filter_for_subclasses( + union_membership, PythonToolBase + ).get(resolve) + if maybe_exportable: + lockfile = cast( + PythonToolBase, maybe_exportable + ).pex_requirements_for_default_lockfile() + else: + lockfile = None + + if not lockfile: raise ExportError( f"No resolve named {resolve} found in [{python_setup.options_scope}].resolves." ) - lockfile = Lockfile( - url=lockfile_path, - url_description_of_origin=f"the resolve `{resolve}`", - resolve_name=resolve, - ) interpreter_constraints = InterpreterConstraints( python_setup.resolves_to_interpreter_constraints.get( diff --git a/src/python/pants/backend/python/goals/export_integration_test.py b/src/python/pants/backend/python/goals/export_integration_test.py index 14aa9d65b49..d38194a8873 100644 --- a/src/python/pants/backend/python/goals/export_integration_test.py +++ b/src/python/pants/backend/python/goals/export_integration_test.py @@ -6,7 +6,6 @@ import os import platform import shutil -from dataclasses import dataclass from textwrap import dedent from typing import Mapping, MutableMapping @@ -37,19 +36,6 @@ } -@dataclass -class _ToolConfig: - name: str - version: str - experimental: bool = False - backend_prefix: str | None = "lint" - takes_ics: bool = True - - @property - def package(self) -> str: - return self.name.replace("-", "_") - - def build_config(tmpdir: str, py_resolve_format: PythonResolveExportFormat) -> Mapping: cfg: MutableMapping = { "GLOBAL": { diff --git a/src/python/pants/backend/python/goals/export_test.py b/src/python/pants/backend/python/goals/export_test.py index f351a473bec..e46c61b166e 100644 --- a/src/python/pants/backend/python/goals/export_test.py +++ b/src/python/pants/backend/python/goals/export_test.py @@ -10,6 +10,7 @@ from pants.backend.python import target_types_rules from pants.backend.python.goals import export from pants.backend.python.goals.export import ExportVenvsRequest, PythonResolveExportFormat +from pants.backend.python.lint.isort import subsystem as isort_subsystem from pants.backend.python.macros.python_artifact import PythonArtifact from pants.backend.python.target_types import ( PythonDistribution, @@ -19,13 +20,23 @@ from pants.backend.python.util_rules import local_dists_pep660, pex_from_targets from pants.base.specs import RawSpecs from pants.core.goals.export import ExportResults +from pants.core.goals.resolves import ExportableTool from pants.core.util_rules import distdir from pants.engine.internals.parametrize import Parametrize from pants.engine.rules import QueryRule from pants.engine.target import Targets +from pants.engine.unions import UnionRule from pants.testutil.rule_runner import RuleRunner from pants.util.frozendict import FrozenDict +pants_args_for_python_lockfiles = [ + "--python-enable-resolves=True", + # Turn off lockfile validation to make the test simpler. + "--python-invalid-lockfile-behavior=ignore", + # Turn off python synthetic lockfile targets to make the test simpler. + "--no-python-enable-lockfile-targets", +] + @pytest.fixture def rule_runner() -> RuleRunner: @@ -36,6 +47,10 @@ def rule_runner() -> RuleRunner: *target_types_rules.rules(), *distdir.rules(), *local_dists_pep660.rules(), + *isort_subsystem.rules(), # add a tool that we can try exporting + UnionRule( + ExportableTool, isort_subsystem.Isort + ), # TODO: remove this manual export when we add ExportableTool to tools QueryRule(Targets, [RawSpecs]), QueryRule(ExportResults, [ExportVenvsRequest]), ], @@ -80,15 +95,11 @@ def test_export_venv_new_codepath( format_flag = f"--export-py-resolve-format={py_resolve_format.value}" rule_runner.set_options( [ + *pants_args_for_python_lockfiles, f"--python-interpreter-constraints=['=={current_interpreter}']", - "--python-enable-resolves=True", "--python-resolves={'a': 'lock.txt', 'b': 'lock.txt'}", "--export-resolve=a", "--export-resolve=b", - # Turn off lockfile validation to make the test simpler. - "--python-invalid-lockfile-behavior=ignore", - # Turn off python synthetic lockfile targets to make the test simpler. - "--no-python-enable-lockfile-targets", "--export-py-editable-in-resolve=['a', 'b']", format_flag, ], @@ -148,3 +159,13 @@ def test_export_venv_new_codepath( f"python/virtualenvs/a/{current_interpreter}", f"python/virtualenvs/b/{current_interpreter}", ] + + +def test_export_tool(rule_runner: RuleRunner) -> None: + """Test exporting an ExportableTool.""" + rule_runner.set_options([*pants_args_for_python_lockfiles, "--export-resolve=isort"]) + results = rule_runner.request(ExportResults, [ExportVenvsRequest(tuple())]) + assert len(results) == 1 + result = results[0] + assert result.resolve == isort_subsystem.Isort.options_scope + assert "isort" in result.description diff --git a/src/python/pants/backend/python/goals/lockfile.py b/src/python/pants/backend/python/goals/lockfile.py index c02c3041293..526a40fbecf 100644 --- a/src/python/pants/backend/python/goals/lockfile.py +++ b/src/python/pants/backend/python/goals/lockfile.py @@ -9,6 +9,7 @@ from dataclasses import dataclass from operator import itemgetter +from pants.backend.python.subsystems.python_tool_base import PythonToolBase from pants.backend.python.subsystems.setup import PythonSetup from pants.backend.python.target_types import ( PythonRequirementFindLinksField, @@ -26,6 +27,7 @@ ResolvePexConfigRequest, ) from pants.core.goals.generate_lockfiles import ( + DEFAULT_TOOL_LOCKFILE, GenerateLockfile, GenerateLockfileResult, GenerateLockfilesSubsystem, @@ -35,6 +37,7 @@ UserGenerateLockfiles, WrappedGenerateLockfile, ) +from pants.core.goals.resolves import ExportableTool from pants.core.util_rules.lockfile_metadata import calculate_invalidation_digest from pants.engine.fs import CreateDigest, Digest, DigestContents, FileContent, MergeDigests from pants.engine.internals.synthetic_targets import SyntheticAddressMaps, SyntheticTargetsRequest @@ -42,7 +45,8 @@ from pants.engine.process import ProcessCacheScope, ProcessResult from pants.engine.rules import Get, collect_rules, rule from pants.engine.target import AllTargets -from pants.engine.unions import UnionRule +from pants.engine.unions import UnionMembership, UnionRule +from pants.option.subsystem import _construct_subsystem from pants.util.docutil import bin_name from pants.util.logging import LogLevel from pants.util.ordered_set import FrozenOrderedSet @@ -198,12 +202,24 @@ class KnownPythonUserResolveNamesRequest(KnownUserResolveNamesRequest): pass +def python_exportable_tools(union_membership: UnionMembership) -> dict[str, type[PythonToolBase]]: + exportable_tools = union_membership.get(ExportableTool) + names_of_python_tools: dict[str, type[PythonToolBase]] = { + e.options_scope: e for e in exportable_tools if issubclass(e, PythonToolBase) # type: ignore # mypy isn't narrowing with `issubclass` + } + return names_of_python_tools + + @rule def determine_python_user_resolves( - _: KnownPythonUserResolveNamesRequest, python_setup: PythonSetup + _: KnownPythonUserResolveNamesRequest, + python_setup: PythonSetup, + union_membership: UnionMembership, ) -> KnownUserResolveNames: + python_tool_resolves = ExportableTool.filter_for_subclasses(union_membership, PythonToolBase) + return KnownUserResolveNames( - names=tuple(python_setup.resolves.keys()), + names=(*python_setup.resolves.keys(), *python_tool_resolves.keys()), option_name="[python].resolves", requested_resolve_names_cls=RequestedPythonUserResolveNames, ) @@ -211,7 +227,10 @@ def determine_python_user_resolves( @rule async def setup_user_lockfile_requests( - requested: RequestedPythonUserResolveNames, all_targets: AllTargets, python_setup: PythonSetup + requested: RequestedPythonUserResolveNames, + all_targets: AllTargets, + python_setup: PythonSetup, + union_membership: UnionMembership, ) -> UserGenerateLockfiles: if not (python_setup.enable_resolves and python_setup.resolves_generate_lockfiles): return UserGenerateLockfiles() @@ -225,23 +244,50 @@ async def setup_user_lockfile_requests( resolve_to_requirements_fields[resolve].add(tgt[PythonRequirementsField]) find_links.update(tgt[PythonRequirementFindLinksField].value or ()) - return UserGenerateLockfiles( - GeneratePythonLockfile( - requirements=PexRequirements.req_strings_from_requirement_fields( - resolve_to_requirements_fields[resolve] - ), - find_links=FrozenOrderedSet(find_links), - interpreter_constraints=InterpreterConstraints( - python_setup.resolves_to_interpreter_constraints.get( - resolve, python_setup.interpreter_constraints + tools = ExportableTool.filter_for_subclasses(union_membership, PythonToolBase) + + out = [] + for resolve in requested: + if resolve in python_setup.resolves: + out.append( + GeneratePythonLockfile( + requirements=PexRequirements.req_strings_from_requirement_fields( + resolve_to_requirements_fields[resolve] + ), + find_links=FrozenOrderedSet(find_links), + interpreter_constraints=InterpreterConstraints( + python_setup.resolves_to_interpreter_constraints.get( + resolve, python_setup.interpreter_constraints + ) + ), + resolve_name=resolve, + lockfile_dest=python_setup.resolves[resolve], + diff=False, ) - ), - resolve_name=resolve, - lockfile_dest=python_setup.resolves[resolve], - diff=False, - ) - for resolve in requested - ) + ) + else: + tool_cls: type[PythonToolBase] = tools[resolve] + tool = await _construct_subsystem(tool_cls) + + # TODO: we shouldn't be managing default ICs in lockfile identification. + # We should find a better place to do this or a better way to default + if tool.register_interpreter_constraints: + ic = tool.interpreter_constraints + else: + ic = InterpreterConstraints(tool.default_interpreter_constraints) + + out.append( + GeneratePythonLockfile( + requirements=FrozenOrderedSet(sorted(tool.requirements)), + find_links=FrozenOrderedSet(find_links), + interpreter_constraints=ic, + resolve_name=resolve, + lockfile_dest=DEFAULT_TOOL_LOCKFILE, + diff=False, + ) + ) + + return UserGenerateLockfiles(out) @dataclass(frozen=True) diff --git a/src/python/pants/backend/python/lint/isort/subsystem.py b/src/python/pants/backend/python/lint/isort/subsystem.py index 6e023127eaf..b7ead3cc3d1 100644 --- a/src/python/pants/backend/python/lint/isort/subsystem.py +++ b/src/python/pants/backend/python/lint/isort/subsystem.py @@ -91,4 +91,6 @@ def config_request(self, dirs: Iterable[str]) -> ConfigFilesRequest: def rules(): - return collect_rules() + return [ + *collect_rules(), + ] diff --git a/src/python/pants/backend/python/subsystems/python_tool_base.py b/src/python/pants/backend/python/subsystems/python_tool_base.py index 51877a8efe8..7376159ba1b 100644 --- a/src/python/pants/backend/python/subsystems/python_tool_base.py +++ b/src/python/pants/backend/python/subsystems/python_tool_base.py @@ -19,6 +19,7 @@ PexRequirements, Resolve, ) +from pants.core.goals.resolves import ExportableTool from pants.engine.fs import Digest from pants.engine.internals.selectors import Get from pants.option.errors import OptionsError @@ -31,7 +32,7 @@ logger = logging.getLogger(__name__) -class PythonToolRequirementsBase(Subsystem): +class PythonToolRequirementsBase(Subsystem, ExportableTool): """Base class for subsystems that configure a set of requirements for a python tool.""" # Subclasses must set. @@ -141,6 +142,33 @@ def default_lockfile_url(cls) -> str: ) ) + @classmethod + def help_for_generate_lockfile_with_default_location(cls, resolve_name): + return softwrap( + f""" + You requested to generate a lockfile for {resolve_name} because + you included it in `--generate-lockfiles-resolve`, but + {resolve_name} is a tool using its default lockfile. + + If you would like to generate a lockfile for {resolve_name}, + follow the instructions for setting up lockfiles for tools + {doc_url('docs/python/overview/lockfiles#lockfiles-for-tools')} + """ + ) + + @classmethod + def pex_requirements_for_default_lockfile(cls): + """Generate the pex requirements using this subsystem's default lockfile resource.""" + assert cls.default_lockfile_resource is not None + pkg, path = cls.default_lockfile_resource + url = f"resource://{pkg}/{path}" + origin = f"The built-in default lockfile for {cls.options_scope}" + return Lockfile( + url=url, + url_description_of_origin=origin, + resolve_name=cls.options_scope, + ) + def pex_requirements( self, *, @@ -155,17 +183,8 @@ def pex_requirements( from_superset=Resolve(self.install_from_resolve, use_entire_lockfile), description_of_origin=description_of_origin, ) - - assert self.default_lockfile_resource is not None - pkg, path = self.default_lockfile_resource - url = f"resource://{pkg}/{path}" - origin = f"The built-in default lockfile for {self.options_scope}" - lockfile = Lockfile( - url=url, - url_description_of_origin=origin, - resolve_name=self.options_scope, - ) - return EntireLockfile(lockfile) + else: + return EntireLockfile(self.pex_requirements_for_default_lockfile()) @property def interpreter_constraints(self) -> InterpreterConstraints: diff --git a/src/python/pants/core/goals/generate_lockfiles.py b/src/python/pants/core/goals/generate_lockfiles.py index 2c0ca5a90ad..2abd4e50fe2 100644 --- a/src/python/pants/core/goals/generate_lockfiles.py +++ b/src/python/pants/core/goals/generate_lockfiles.py @@ -8,8 +8,20 @@ from collections import defaultdict from dataclasses import dataclass, replace from enum import Enum -from typing import Callable, ClassVar, Iterable, Iterator, Mapping, Protocol, Sequence, Tuple, cast - +from typing import ( + Callable, + ClassVar, + Iterable, + Iterator, + Mapping, + Protocol, + Sequence, + Tuple, + Type, + cast, +) + +from pants.core.goals.resolves import ExportableTool from pants.engine.collection import Collection from pants.engine.console import Console from pants.engine.environment import ChosenLocalEnvironmentName, EnvironmentName @@ -369,6 +381,7 @@ def determine_resolves_to_generate( # Resolve names must be globally unique, so check for ambiguity across backends. _check_ambiguous_resolve_names(all_known_user_resolve_names) + # If no resolves have been requested, we generate lockfiles for all user resolves if not requested_resolve_names: return [ known_resolve_names.requested_resolve_names_cls(known_resolve_names.names) @@ -412,6 +425,7 @@ def filter_tool_lockfile_requests( result = [] for wrapped_req in specified_requests: req = wrapped_req.request + if req.lockfile_dest != DEFAULT_TOOL_LOCKFILE: result.append(req) continue @@ -435,6 +449,56 @@ def filter_tool_lockfile_requests( return result +def filter_lockfiles_for_unconfigured_exportable_tools( + generate_lockfile_requests: Sequence[GenerateLockfile], + exportabletools_by_name: dict[str, Type[ExportableTool]], + *, + resolve_specified: bool, +) -> Tuple[Sequence[str], Sequence[GenerateLockfile]]: + """Filter lockfile requests for tools still using their default lockfiles.""" + + valid_lockfiles = [] + errs = [] + + for req in generate_lockfile_requests: + if req.lockfile_dest != DEFAULT_TOOL_LOCKFILE: + valid_lockfiles.append(req) + continue + + if req.resolve_name in exportabletools_by_name: + if resolve_specified: + # A user has asked us to generate a tool which is using a default lockfile + errs.append( + exportabletools_by_name[ + req.resolve_name + ].help_for_generate_lockfile_with_default_location(req.resolve_name) + ) + else: + # When a user selects no resolves, we try generating lockfiles for all resolves. + # The intention is clearly to not generate lockfiles for internal tools, so we skip them here. + continue + else: + # Arriving at this case is either a user error or an implementation error, but we can be helpful + errs.append( + softwrap( + f""" + The resolve {req.resolve_name} is using the lockfile destination {DEFAULT_TOOL_LOCKFILE}. + This destination is used as a sentinel to signal that internal tools should use their bundled lockfile. + However, the resolve {req.resolve_name} does not appear to be an exportable tool. + + If you intended to generate a lockfile for a resolve you specified, + you should specify a file as the lockfile destination. + If this is indeed a tool that should be exportable, this is a bug: + This tool does not appear to be exportable the way we expect. + It may need a `UnionRule` to `ExportableTool` + """ + ) + ) + continue + + return errs, valid_lockfiles + + class GenerateLockfilesSubsystem(GoalSubsystem): name = "generate-lockfiles" help = "Generate lockfiles for third-party dependencies." @@ -540,17 +604,30 @@ async def generate_lockfiles_goal( ) for sentinel in requested_tool_sentinels ) + resolve_specified = bool(generate_lockfiles_subsystem.resolve) applicable_tool_requests = filter_tool_lockfile_requests( specified_tool_requests, - resolve_specified=bool(generate_lockfiles_subsystem.resolve), + resolve_specified=resolve_specified, ) + # We filter "user" requests because we're moving to combine user and tool lockfiles + ( + tool_request_errors, + applicable_user_requests, + ) = filter_lockfiles_for_unconfigured_exportable_tools( + list(itertools.chain(*all_specified_user_requests)), + {e.options_scope: e for e in union_membership.get(ExportableTool)}, + resolve_specified=resolve_specified, + ) + + if tool_request_errors: + raise ValueError("\n\n".join(tool_request_errors)) # Execute the actual lockfile generation in each request's environment. # Currently, since resolves specify a single filename for output, we pick a reasonable # environment to execute the request in. Currently we warn if multiple environments are # specified. all_requests: Iterator[GenerateLockfile] = itertools.chain( - *all_specified_user_requests, applicable_tool_requests + applicable_user_requests, applicable_tool_requests ) if generate_lockfiles_subsystem.request_diffs: all_requests = (replace(req, diff=True) for req in all_requests) diff --git a/src/python/pants/core/goals/generate_lockfiles_test.py b/src/python/pants/core/goals/generate_lockfiles_test.py index a80c31a8250..d706added98 100644 --- a/src/python/pants/core/goals/generate_lockfiles_test.py +++ b/src/python/pants/core/goals/generate_lockfiles_test.py @@ -36,8 +36,10 @@ WrappedGenerateLockfile, _preferred_environment, determine_resolves_to_generate, + filter_lockfiles_for_unconfigured_exportable_tools, filter_tool_lockfile_requests, ) +from pants.core.goals.resolves import ExportableTool from pants.engine.console import Console from pants.engine.environment import EnvironmentName from pants.engine.target import Dependencies, Target @@ -406,3 +408,52 @@ def test_diff_printer( re.sub(" +\n", "\n", args[0]) ) assert actual_output == expect_output + + +class TestExportableTool(ExportableTool): + options_scope = "test_exportable_tool" + + +def test_filter_unconfigured_tools_configured(): + """Test that a configured tool succeeds.""" + resolve_name = "myresolve" + resolve_dest = "resolve.lock" + + errs, results = filter_lockfiles_for_unconfigured_exportable_tools( + [GenerateLockfile(resolve_name=resolve_name, lockfile_dest=resolve_dest, diff=False)], + {resolve_name: TestExportableTool}, + resolve_specified=True, + ) + + assert len(errs) == 0 + assert len(results) == 1 + + +def test_filter_unconfigured_tools_not_configured(): + """Test that a request for a tool using a default lockfiles results in an error.""" + resolve_name = "myresolve" + resolve_dest = DEFAULT_TOOL_LOCKFILE + + errs, results = filter_lockfiles_for_unconfigured_exportable_tools( + [GenerateLockfile(resolve_name=resolve_name, lockfile_dest=resolve_dest, diff=False)], + {resolve_name: TestExportableTool}, + resolve_specified=True, + ) + + assert len(errs) == 1 + assert len(results) == 0 + + +def test_filter_unconfigured_tools_all_excludes_internal(): + """Test that when a user requests all lockfiles, we exclude unconfigured internal tools.""" + resolve_name = "myresolve" + resolve_dest = DEFAULT_TOOL_LOCKFILE + + errs, results = filter_lockfiles_for_unconfigured_exportable_tools( + [GenerateLockfile(resolve_name=resolve_name, lockfile_dest=resolve_dest, diff=False)], + {resolve_name: TestExportableTool}, + resolve_specified=False, + ) + + assert len(errs) == 0 + assert len(results) == 0 diff --git a/src/python/pants/core/goals/resolves.py b/src/python/pants/core/goals/resolves.py new file mode 100644 index 00000000000..ebcf11b9088 --- /dev/null +++ b/src/python/pants/core/goals/resolves.py @@ -0,0 +1,53 @@ +# Copyright 2024 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). +from __future__ import annotations + +from typing import TypeVar + +from pants.engine.unions import UnionMembership, union +from pants.util.strutil import softwrap + +T = TypeVar("T", bound="ExportableTool") + + +@union +class ExportableTool: + """Mark a subsystem as exportable. + + Using this class has 2 parts: + - The tool class should subclass this. + This can be done at the language-backend level, for example, `PythonToolRequirementsBase`. + The help message can be extended with instructions specific to that tool or language backend + - Each exportable tool should have a `UnionRule` to `ExportableTool`. + This `UnionRule` is what ties the class into the export machinery. + """ + + options_scope: str + + @classmethod + def help_for_generate_lockfile_with_default_location(cls, resolve_name: str): + """If this tool is configured to use the default lockfile, but a user requests to regenerate + it, this help text will be shown to the user.""" + + resolve = resolve_name + return softwrap( + f""" + You requested to generate a lockfile for {resolve} because + you included it in `--generate-lockfiles-resolve`, but + {resolve} is a tool using its default lockfile. + """ + ) + + @staticmethod + def filter_for_subclasses( + union_membership: UnionMembership, parent_class: type[T] + ) -> dict[str, type[T]]: + """Find all ExportableTools that are members of `parent_class`. + + Language backends can use this to obtain all tools they can export. + """ + exportable_tools = union_membership.get(ExportableTool) + relevant_tools: dict[str, type[T]] = { + e.options_scope: e for e in exportable_tools if issubclass(e, parent_class) # type: ignore # mypy isn't narrowing with `issubclass` + } + return relevant_tools diff --git a/src/python/pants/jvm/goals/lockfile.py b/src/python/pants/jvm/goals/lockfile.py index 06ae4cca869..2b94956f63b 100644 --- a/src/python/pants/jvm/goals/lockfile.py +++ b/src/python/pants/jvm/goals/lockfile.py @@ -8,6 +8,7 @@ from typing import Mapping from pants.core.goals.generate_lockfiles import ( + DEFAULT_TOOL_LOCKFILE, GenerateLockfile, GenerateLockfileResult, GenerateLockfilesSubsystem, @@ -17,6 +18,7 @@ UserGenerateLockfiles, WrappedGenerateLockfile, ) +from pants.core.goals.resolves import ExportableTool from pants.engine.environment import EnvironmentName from pants.engine.fs import CreateDigest, Digest, FileContent from pants.engine.internals.selectors import MultiGet @@ -24,11 +26,17 @@ from pants.engine.target import AllTargets from pants.engine.unions import UnionMembership, UnionRule, union from pants.jvm.resolve import coursier_fetch -from pants.jvm.resolve.common import ArtifactRequirement, ArtifactRequirements +from pants.jvm.resolve.common import ( + ArtifactRequirement, + ArtifactRequirements, + GatherJvmCoordinatesRequest, +) from pants.jvm.resolve.coursier_fetch import CoursierResolvedLockfile +from pants.jvm.resolve.jvm_tool import GenerateJvmLockfileFromTool, JvmToolBase from pants.jvm.resolve.lockfile_metadata import JVMLockfileMetadata from pants.jvm.subsystems import JvmSubsystem from pants.jvm.target_types import JvmArtifactResolveField, JvmResolveField +from pants.option.subsystem import _construct_subsystem from pants.util.docutil import bin_name from pants.util.logging import LogLevel from pants.util.ordered_set import OrderedSet @@ -97,10 +105,14 @@ class KnownJVMUserResolveNamesRequest(KnownUserResolveNamesRequest): @rule def determine_jvm_user_resolves( - _: KnownJVMUserResolveNamesRequest, jvm_subsystem: JvmSubsystem + _: KnownJVMUserResolveNamesRequest, + jvm_subsystem: JvmSubsystem, + union_membership: UnionMembership, ) -> KnownUserResolveNames: + jvm_tool_resolves = ExportableTool.filter_for_subclasses(union_membership, JvmToolBase) + names = (*jvm_subsystem.resolves.keys(), *jvm_tool_resolves.keys()) return KnownUserResolveNames( - names=tuple(jvm_subsystem.resolves.keys()), + names=names, option_name=f"[{jvm_subsystem.options_scope}].resolves", requested_resolve_names_cls=RequestedJVMUserResolveNames, ) @@ -135,11 +147,46 @@ async def validate_jvm_artifacts_for_resolve( ) +async def _plan_generate_lockfile(resolve, resolve_to_artifacts, tools) -> Get: + """Generate a JVM lockfile request for each requested resolve. + + This step also allows other backends to validate the proposed set of artifact requirements for + each resolve. + """ + if resolve in resolve_to_artifacts: + return Get( + GenerateJvmLockfile, + _ValidateJvmArtifactsRequest( + artifacts=ArtifactRequirements(resolve_to_artifacts[resolve]), + resolve_name=resolve, + ), + ) + elif resolve in tools: + tool_cls: type[JvmToolBase] = tools[resolve] + tool = await _construct_subsystem(tool_cls) + + return Get( + GenerateJvmLockfile, + GenerateJvmLockfileFromTool, + GenerateJvmLockfileFromTool.create(tool), + ) + + else: + return Get( + GenerateJvmLockfile, + _ValidateJvmArtifactsRequest( + artifacts=ArtifactRequirements(()), + resolve_name=resolve, + ), + ) + + @rule async def setup_user_lockfile_requests( requested: RequestedJVMUserResolveNames, all_targets: AllTargets, jvm_subsystem: JvmSubsystem, + union_membership: UnionMembership, ) -> UserGenerateLockfiles: resolve_to_artifacts: Mapping[str, OrderedSet[ArtifactRequirement]] = defaultdict(OrderedSet) for tgt in sorted(all_targets, key=lambda t: t.address): @@ -149,22 +196,37 @@ async def setup_user_lockfile_requests( resolve = tgt[JvmResolveField].normalized_value(jvm_subsystem) resolve_to_artifacts[resolve].add(artifact) - # Generate a JVM lockfile request for each requested resolve. This step also allows other backends to - # validate the proposed set of artifact requirements for each resolve. - jvm_lockfile_requests = await MultiGet( - Get( - GenerateJvmLockfile, - _ValidateJvmArtifactsRequest( - artifacts=ArtifactRequirements(resolve_to_artifacts.get(resolve, ())), - resolve_name=resolve, - ), - ) - for resolve in requested - ) + tools = ExportableTool.filter_for_subclasses(union_membership, JvmToolBase) + + gets = [] + for resolve in requested: + gets.append(await _plan_generate_lockfile(resolve, resolve_to_artifacts, tools)) + + jvm_lockfile_requests = await MultiGet(*gets) return UserGenerateLockfiles(jvm_lockfile_requests) +@rule +async def setup_lockfile_request_from_tool( + request: GenerateJvmLockfileFromTool, +) -> GenerateJvmLockfile: + artifacts = await Get( + ArtifactRequirements, + GatherJvmCoordinatesRequest(request.artifact_inputs, request.artifact_option_name), + ) + return GenerateJvmLockfile( + artifacts=artifacts, + resolve_name=request.resolve_name, + lockfile_dest=( + request.write_lockfile_dest + if request.read_lockfile_dest != DEFAULT_TOOL_LOCKFILE + else DEFAULT_TOOL_LOCKFILE + ), + diff=False, + ) + + def rules(): return ( *collect_rules(), diff --git a/src/python/pants/jvm/goals/lockfile_test.py b/src/python/pants/jvm/goals/lockfile_test.py index 5b21c12fa62..5e61ee2461a 100644 --- a/src/python/pants/jvm/goals/lockfile_test.py +++ b/src/python/pants/jvm/goals/lockfile_test.py @@ -8,18 +8,26 @@ import pytest -from pants.core.goals.generate_lockfiles import GenerateLockfileResult, UserGenerateLockfiles +from pants.core.goals.generate_lockfiles import ( + DEFAULT_TOOL_LOCKFILE, + GenerateLockfileResult, + UserGenerateLockfiles, +) +from pants.core.goals.resolves import ExportableTool from pants.core.util_rules import source_files from pants.core.util_rules.external_tool import rules as external_tool_rules from pants.engine.fs import DigestContents, FileDigest from pants.engine.internals.parametrize import Parametrize +from pants.engine.unions import UnionRule from pants.jvm.goals import lockfile from pants.jvm.goals.lockfile import GenerateJvmLockfile, RequestedJVMUserResolveNames +from pants.jvm.resolve import jvm_tool from pants.jvm.resolve.common import ArtifactRequirement, ArtifactRequirements from pants.jvm.resolve.coordinate import Coordinate, Coordinates from pants.jvm.resolve.coursier_fetch import CoursierLockfileEntry, CoursierResolvedLockfile from pants.jvm.resolve.coursier_fetch import rules as coursier_fetch_rules from pants.jvm.resolve.coursier_setup import rules as coursier_setup_rules +from pants.jvm.resolve.jvm_tool import JvmToolBase from pants.jvm.resolve.lockfile_metadata import JVMLockfileMetadata from pants.jvm.target_types import JvmArtifactTarget from pants.jvm.testutil import maybe_skip_jdk_test @@ -27,16 +35,30 @@ from pants.testutil.rule_runner import QueryRule, RuleRunner +class MockJvmTool(JvmToolBase, ExportableTool): + """This one uses the ExportableTool resolve mechanism.""" + + options_scope = "mock-tool" + help = "Hamcrest is a mocking tool for the JVM." + + default_version = "1.3" + default_artifacts = ("org.hamcrest:hamcrest-core:{version}",) + default_lockfile_resource = ("pants.backend.jvm.resolve", "mock-tool.default.lockfile.txt") + + @pytest.fixture def rule_runner() -> RuleRunner: rule_runner = RuleRunner( rules=[ *coursier_fetch_rules(), *lockfile.rules(), + *jvm_tool.rules(), *coursier_setup_rules(), *external_tool_rules(), *source_files.rules(), *util_rules(), + *MockJvmTool.rules(), + UnionRule(ExportableTool, MockJvmTool), QueryRule(UserGenerateLockfiles, [RequestedJVMUserResolveNames]), QueryRule(GenerateLockfileResult, [GenerateJvmLockfile]), ], @@ -176,3 +198,36 @@ def test_multiple_resolves(rule_runner: RuleRunner) -> None: diff=False, ), } + + +@pytest.mark.asyncio +async def test_plan_generate_lockfile_tool(rule_runner: RuleRunner): + rule_runner.write_files( + { + "BUILD": dedent( + """\ + def mk(name): + jvm_artifact( + name=name, + group='group', + artifact='artifact', + version='1', + jar='jar.jar', + ) + + mk('one') + mk('two') + """ + ), + } + ) + + result_group = rule_runner.request( + UserGenerateLockfiles, [RequestedJVMUserResolveNames([MockJvmTool.options_scope])] + ) + + assert len(result_group) == 1, f"we're expecting a single {GenerateJvmLockfile.__name__}" + result = result_group[0] + + assert result.resolve_name == MockJvmTool.options_scope + assert result.lockfile_dest == DEFAULT_TOOL_LOCKFILE diff --git a/src/python/pants/jvm/resolve/jvm_tool.py b/src/python/pants/jvm/resolve/jvm_tool.py index a4224e4cb44..fcb3d645698 100644 --- a/src/python/pants/jvm/resolve/jvm_tool.py +++ b/src/python/pants/jvm/resolve/jvm_tool.py @@ -8,12 +8,11 @@ from pants.build_graph.address import Address, AddressInput from pants.core.goals.generate_lockfiles import DEFAULT_TOOL_LOCKFILE, GenerateToolLockfileSentinel +from pants.core.goals.resolves import ExportableTool from pants.engine.addresses import Addresses from pants.engine.internals.selectors import Get, MultiGet from pants.engine.rules import collect_rules, rule from pants.engine.target import Targets -from pants.jvm.goals import lockfile -from pants.jvm.goals.lockfile import GenerateJvmLockfile from pants.jvm.resolve.common import ( ArtifactRequirement, ArtifactRequirements, @@ -29,7 +28,7 @@ from pants.util.strutil import softwrap -class JvmToolBase(Subsystem): +class JvmToolBase(Subsystem, ExportableTool): """Base class for subsystems that configure a set of artifact requirements for a JVM tool.""" # Default version of the tool. (Subclasses may set.) @@ -104,6 +103,20 @@ def default_lockfile_url(cls) -> str: ) ) + @classmethod + def help_for_generate_lockfile_with_default_location(cls, resolve_name): + return softwrap( + f""" + You requested to generate a lockfile for {resolve_name} because + you included it in `--generate-lockfiles-resolve`, but + {resolve_name} is a tool using its default lockfile. + + If you would like to generate a lockfile for {resolve_name}, please + set `[{resolve_name}].lockfile` to the path where it should be + generated and run again. + """ + ) + @property def artifact_inputs(self) -> tuple[str, ...]: return tuple(s.format(version=self.version) for s in self.artifacts) @@ -205,25 +218,7 @@ def create(cls, tool: JvmToolBase) -> GenerateJvmLockfileFromTool: ) -@rule -async def setup_lockfile_request_from_tool( - request: GenerateJvmLockfileFromTool, -) -> GenerateJvmLockfile: - artifacts = await Get( - ArtifactRequirements, - GatherJvmCoordinatesRequest(request.artifact_inputs, request.artifact_option_name), - ) - return GenerateJvmLockfile( - artifacts=artifacts, - resolve_name=request.resolve_name, - lockfile_dest=( - request.write_lockfile_dest - if request.read_lockfile_dest != DEFAULT_TOOL_LOCKFILE - else DEFAULT_TOOL_LOCKFILE - ), - diff=False, - ) - - def rules(): + from pants.jvm.goals import lockfile # TODO: Shim to avoid import cycle + return (*collect_rules(), *lockfile.rules())