From b6d763e17efc1e96b1729af9b1476852efd24f1b Mon Sep 17 00:00:00 2001 From: Daniel Goldman Date: Wed, 21 Feb 2024 22:56:08 -0500 Subject: [PATCH] hack together exporting tools --- .../pants/backend/python/goals/export.py | 31 ++++++++++++++----- .../python/goals/lockfile_generation.py | 14 ++++++++- .../pants/backend/python/lint/black/rules.py | 8 +++++ .../pants/core/goals/resolve_helpers.py | 19 ++++++++++-- 4 files changed, 61 insertions(+), 11 deletions(-) diff --git a/src/python/pants/backend/python/goals/export.py b/src/python/pants/backend/python/goals/export.py index 994647737f63..0530d7413817 100644 --- a/src/python/pants/backend/python/goals/export.py +++ b/src/python/pants/backend/python/goals/export.py @@ -9,9 +9,10 @@ import uuid from dataclasses import dataclass from enum import Enum -from typing import Any +from typing import Any, cast from pants.backend.python.goals.lockfile_generation import GeneratePythonLockfile +from pants.backend.python.subsystems.python_tool_base import PythonToolRequirementsBase from pants.backend.python.subsystems.setup import PythonSetup from pants.backend.python.target_types import PexLayout from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints @@ -32,12 +33,13 @@ ExportSubsystem, PostProcessingCommand, ) +from pants.core.goals.resolve_helpers import ExportableTool from pants.engine.engine_aware import EngineAwareParameter from pants.engine.internals.native_engine import AddPrefix, Digest, MergeDigests, Snapshot from pants.engine.internals.selectors import Get from pants.engine.process import ProcessCacheScope, ProcessResult from pants.engine.rules import collect_rules, rule -from pants.engine.unions import UnionRule +from pants.engine.unions import UnionMembership, UnionRule from pants.option.option_types import EnumOption, StrListOption from pants.util.strutil import path_safe, softwrap @@ -308,18 +310,31 @@ async def export_virtualenv_for_resolve( request: _ExportVenvForResolveRequest, python_setup: PythonSetup, export_subsys: ExportSubsystem, + union_membership: UnionMembership, ) -> MaybeExportResult: resolve = request.resolve lockfile_path = python_setup.resolves.get(resolve) - if not lockfile_path: + if lockfile_path: + lockfile = Lockfile( + url=lockfile_path, + url_description_of_origin=f"the resolve `{resolve}`", + resolve_name=resolve, + ) + else: + tools = union_membership.get(ExportableTool) + maybe_exportable = {e.resolve_name: e for e in tools}.get(resolve) + if maybe_exportable: + lockfile = cast( + PythonToolRequirementsBase, maybe_exportable.subsystem_cls + ).pex_requirements_for_default_lockfile() + + else: + lockfile = None + + if not lockfile: raise ExportError( f"No resolve named {resolve} found in [{python_setup.options_scope}].resolves." ) - lockfile = Lockfile( - url=lockfile_path, - url_description_of_origin=f"the resolve `{resolve}`", - resolve_name=resolve, - ) # TODO: from request? interpreter_constraints = InterpreterConstraints( diff --git a/src/python/pants/backend/python/goals/lockfile_generation.py b/src/python/pants/backend/python/goals/lockfile_generation.py index 8a05c126fa07..f115efebcd0d 100644 --- a/src/python/pants/backend/python/goals/lockfile_generation.py +++ b/src/python/pants/backend/python/goals/lockfile_generation.py @@ -5,6 +5,7 @@ from collections import defaultdict from dataclasses import dataclass +from pants.backend.python.lint.black.rules import BlackExportableTool from pants.backend.python.subsystems.setup import PythonSetup from pants.backend.python.target_types import ( PythonRequirementFindLinksField, @@ -216,6 +217,15 @@ def determine_python_user_resolves( ) +@rule +def determine_python_exportable_tools(req: BlackExportableTool) -> KnownUserResolveNames: + return KnownUserResolveNames( + names=(req.resolve_name,), + option_name="", + requested_resolve_names_cls=RequestedPythonUserResolveNames, + ) + + @rule async def setup_user_lockfile_requests( requested: RequestedPythonUserResolveNames, all_targets: AllTargets, python_setup: PythonSetup @@ -244,7 +254,9 @@ async def setup_user_lockfile_requests( ) ), resolve_name=resolve, - lockfile_dest=python_setup.resolves[resolve], + lockfile_dest=python_setup.resolves.get( + resolve, "TODO_dest" + ), # TODO: real thing for internal lockfiles diff=False, ) for resolve in requested diff --git a/src/python/pants/backend/python/lint/black/rules.py b/src/python/pants/backend/python/lint/black/rules.py index 61cb652dbc79..744c539d56ae 100644 --- a/src/python/pants/backend/python/lint/black/rules.py +++ b/src/python/pants/backend/python/lint/black/rules.py @@ -11,10 +11,12 @@ from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints from pants.backend.python.util_rules.pex import PexRequest, VenvPex, VenvPexProcess from pants.core.goals.fmt import AbstractFmtRequest, FmtResult, FmtTargetsRequest, Partitions +from pants.core.goals.resolve_helpers import ExportableTool from pants.core.util_rules.config_files import ConfigFiles, ConfigFilesRequest from pants.engine.fs import Digest, MergeDigests from pants.engine.process import ProcessResult from pants.engine.rules import Get, MultiGet, collect_rules, rule +from pants.engine.unions import UnionRule from pants.util.logging import LogLevel from pants.util.strutil import pluralize, softwrap @@ -115,9 +117,15 @@ async def black_fmt(request: BlackRequest.Batch, black: Black) -> FmtResult: ) +class BlackExportableTool(ExportableTool): + resolve_name = Black.options_scope + subsystem_cls = Black + + def rules(): return [ *collect_rules(), *BlackRequest.rules(), *pex.rules(), + UnionRule(ExportableTool, BlackExportableTool), ] diff --git a/src/python/pants/core/goals/resolve_helpers.py b/src/python/pants/core/goals/resolve_helpers.py index f2b55b8115ff..b24904bf5a63 100644 --- a/src/python/pants/core/goals/resolve_helpers.py +++ b/src/python/pants/core/goals/resolve_helpers.py @@ -90,6 +90,12 @@ class KnownUserResolveNames: requested_resolve_names_cls: type[RequestedUserResolveNames] +@union +class ExportableTool: + resolve_name: ClassVar[str] + subsystem_cls: ClassVar[type] + + @union(in_scope_types=[EnvironmentName]) class RequestedUserResolveNames(Collection[str]): """The user resolves requested for a particular language ecosystem. @@ -353,9 +359,18 @@ async def determine_requested_resolves( Get(KnownUserResolveNames, KnownUserResolveNamesRequest, request()) for request in union_membership.get(KnownUserResolveNamesRequest) ) - logger.debug(f"Found known user resolves {known_user_resolve_names}") + known_internal_resolve_names = await MultiGet( + Get(KnownUserResolveNames, ExportableTool, tool()) + for tool in union_membership.get(ExportableTool) + ) + + logger.debug( + f"Found known user resolves {known_user_resolve_names} and internal resolves {known_internal_resolve_names}" + ) + all_known_resolves = [*known_user_resolve_names, *known_internal_resolve_names] + requested_user_resolve_names, requested_tool_sentinels = determine_resolves_to_generate( - known_user_resolve_names, + tuple(all_known_resolves), union_membership.get(GenerateToolLockfileSentinel), set(requested_resolves.val), )