From 78775b9716f93609a599d9efd17e169e574f17f2 Mon Sep 17 00:00:00 2001 From: cburroughs Date: Mon, 18 Dec 2023 16:31:08 -0500 Subject: [PATCH 01/18] enable pyupgrade for pants itself as a regular backend (#20289) In #13317 an alias for pyupgrade was added before the `fix` goal existed. Now that `fix` exists pyupgrade can be used as a regular backend and run consistently. For now --keep-runtime-typing to avoid this balooning into a single gigantic diff. --- build-support/bin/terraform_tool_versions.py | 2 +- build-support/bin/terraform_tool_versions_test.py | 1 - pants.toml | 6 +++++- .../backend/codegen/protobuf/go/rules_integration_test.py | 2 +- .../pants/backend/codegen/protobuf/lint/buf/lint_rules.py | 2 +- src/python/pants/backend/go/lint/golangci_lint/rules.py | 2 +- .../backend/go/lint/golangci_lint/rules_integration_test.py | 2 +- src/python/pants/backend/go/testutil.py | 2 +- src/python/pants/backend/go/util_rules/build_pkg.py | 2 +- src/python/pants/backend/helm/subsystems/k8s_parser_main.py | 2 +- .../pants/backend/helm/subsystems/post_renderer_main.py | 2 +- src/python/pants/backend/openapi/lint/spectral/rules.py | 2 +- .../pants/backend/python/framework/django/detect_apps.py | 2 +- .../framework/stevedore/python_target_dependencies_test.py | 6 ++---- .../backend/python/lint/isort/rules_integration_test.py | 2 +- .../pants/backend/python/util_rules/pex_requirements.py | 6 ++---- .../python/util_rules/scripts/pep660_backend_wrapper.py | 4 ++-- src/python/pants/base/exception_sink.py | 2 +- src/python/pants/core/goals/export.py | 2 +- src/python/pants/engine/fs.py | 4 ++-- src/python/pants/engine/goal.py | 6 +++--- src/python/pants/engine/rules_test.py | 2 +- src/python/pants/util/collections_test.py | 2 +- src/python/pants_release/reversion.py | 2 +- tests/python/pants_test/pantsd/test_lock.py | 2 +- 25 files changed, 34 insertions(+), 35 deletions(-) diff --git a/build-support/bin/terraform_tool_versions.py b/build-support/bin/terraform_tool_versions.py index c9187f4f08a..2774ff04e2d 100644 --- a/build-support/bin/terraform_tool_versions.py +++ b/build-support/bin/terraform_tool_versions.py @@ -59,7 +59,7 @@ def check_import_results(import_results: gnupg.ImportResult): Looks the import results for one which has an "ok" status. We can't use the number of keys imported because a re-import of a key results in 0 keys imported. """ - has_ok = any(("ok" in r for r in import_results.results)) + has_ok = any("ok" in r for r in import_results.results) return has_ok diff --git a/build-support/bin/terraform_tool_versions_test.py b/build-support/bin/terraform_tool_versions_test.py index d51d64ea60a..2624bcb3198 100644 --- a/build-support/bin/terraform_tool_versions_test.py +++ b/build-support/bin/terraform_tool_versions_test.py @@ -1,7 +1,6 @@ #!/usr/bin/env python3 # Copyright 2022 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). -from __future__ import absolute_import, division, print_function, unicode_literals import pytest from terraform_tool_versions import ( diff --git a/pants.toml b/pants.toml index 90072551218..47baff727fd 100644 --- a/pants.toml +++ b/pants.toml @@ -14,6 +14,7 @@ backend_packages.add = [ "pants.backend.python.lint.flake8", "pants.backend.python.lint.isort", "pants.backend.python.typecheck.mypy", + "pants.backend.python.lint.pyupgrade", "pants.backend.python.mixed_interpreter_constraints", "pants.backend.shell", "pants.backend.shell.lint.shellcheck", @@ -92,7 +93,6 @@ enabled = true repo_id = "7775F8D5-FC58-4DBC-9302-D00AE4A1505F" [cli.alias] -run-pyupgrade = "--backend-packages=pants.backend.experimental.python.lint.pyupgrade fmt" --all-changed = "--changed-since=HEAD --changed-dependents=transitive" [source] @@ -206,6 +206,10 @@ template_by_globs = "@build-support/preambles/config.yaml" [generate-lockfiles] diff = true +[pyupgrade] +args = ["--keep-runtime-typing"] + + [jvm] default_resolve = "jvm_testprojects" diff --git a/src/python/pants/backend/codegen/protobuf/go/rules_integration_test.py b/src/python/pants/backend/codegen/protobuf/go/rules_integration_test.py index 7b8b075d7e7..d41461d875e 100644 --- a/src/python/pants/backend/codegen/protobuf/go/rules_integration_test.py +++ b/src/python/pants/backend/codegen/protobuf/go/rules_integration_test.py @@ -113,7 +113,7 @@ def assert_files_generated( def test_extracts_go_package() -> None: - import_path = parse_go_package_option("""option go_package = "example.com/dir1";""".encode()) + import_path = parse_go_package_option(b"""option go_package = "example.com/dir1";""") assert import_path == "example.com/dir1" diff --git a/src/python/pants/backend/codegen/protobuf/lint/buf/lint_rules.py b/src/python/pants/backend/codegen/protobuf/lint/buf/lint_rules.py index 67aaddfabf5..a888f1ffe6b 100644 --- a/src/python/pants/backend/codegen/protobuf/lint/buf/lint_rules.py +++ b/src/python/pants/backend/codegen/protobuf/lint/buf/lint_rules.py @@ -62,7 +62,7 @@ async def run_buf( ) -> LintResult: transitive_targets = await Get( TransitiveTargets, - TransitiveTargetsRequest((field_set.address for field_set in request.elements)), + TransitiveTargetsRequest(field_set.address for field_set in request.elements), ) all_stripped_sources_request = Get( diff --git a/src/python/pants/backend/go/lint/golangci_lint/rules.py b/src/python/pants/backend/go/lint/golangci_lint/rules.py index 154628d3ac9..70dc3500f20 100644 --- a/src/python/pants/backend/go/lint/golangci_lint/rules.py +++ b/src/python/pants/backend/go/lint/golangci_lint/rules.py @@ -67,7 +67,7 @@ async def run_golangci_lint( ) -> LintResult: transitive_targets = await Get( TransitiveTargets, - TransitiveTargetsRequest((field_set.address for field_set in request.elements)), + TransitiveTargetsRequest(field_set.address for field_set in request.elements), ) all_source_files_request = Get( diff --git a/src/python/pants/backend/go/lint/golangci_lint/rules_integration_test.py b/src/python/pants/backend/go/lint/golangci_lint/rules_integration_test.py index 6e9e4ac880a..da412d5094f 100644 --- a/src/python/pants/backend/go/lint/golangci_lint/rules_integration_test.py +++ b/src/python/pants/backend/go/lint/golangci_lint/rules_integration_test.py @@ -138,7 +138,7 @@ def test_failing(rule_runner: RuleRunner) -> None: "BUILD": "go_mod(name='mod')\ngo_package(name='pkg')\n", } ) - tgt = rule_runner.get_target((Address("", target_name="pkg"))) + tgt = rule_runner.get_target(Address("", target_name="pkg")) lint_results = run_golangci_lint(rule_runner, [tgt]) assert len(lint_results) == 1 assert lint_results[0].exit_code == 1 diff --git a/src/python/pants/backend/go/testutil.py b/src/python/pants/backend/go/testutil.py index 69ce01f6a08..73fd2782b41 100644 --- a/src/python/pants/backend/go/testutil.py +++ b/src/python/pants/backend/go/testutil.py @@ -40,7 +40,7 @@ def gen_module_gomodproxy( prefix = f"{import_path}@{version}" all_files = [(f"{prefix}/go.mod", go_mod_content)] - all_files.extend(((f"{prefix}/{path}", contents) for (path, contents) in files)) + all_files.extend((f"{prefix}/{path}", contents) for (path, contents) in files) mod_zip_bytes = io.BytesIO() with zipfile.ZipFile(mod_zip_bytes, "w") as mod_zip: diff --git a/src/python/pants/backend/go/util_rules/build_pkg.py b/src/python/pants/backend/go/util_rules/build_pkg.py index 633642f886b..1b639c7d965 100644 --- a/src/python/pants/backend/go/util_rules/build_pkg.py +++ b/src/python/pants/backend/go/util_rules/build_pkg.py @@ -964,7 +964,7 @@ async def compute_compile_action_id( # See https://github.com/golang/go/blob/master/src/cmd/go/internal/cache/hash.go#L32-L46 h.update(goroot.full_version.encode()) - h.update("compile\n".encode()) + h.update(b"compile\n") if bq.minimum_go_version: h.update(f"go {bq.minimum_go_version}\n".encode()) h.update(f"goos {goroot.goos} goarch {goroot.goarch}\n".encode()) diff --git a/src/python/pants/backend/helm/subsystems/k8s_parser_main.py b/src/python/pants/backend/helm/subsystems/k8s_parser_main.py index 4e722b84bf6..b6744c03a48 100644 --- a/src/python/pants/backend/helm/subsystems/k8s_parser_main.py +++ b/src/python/pants/backend/helm/subsystems/k8s_parser_main.py @@ -13,7 +13,7 @@ def main(args: list[str]): found_image_refs: dict[tuple[int, str], str] = {} - with open(input_filename, "r") as file: + with open(input_filename) as file: try: parsed_docs = load_full_yaml(stream=file) except RuntimeError: diff --git a/src/python/pants/backend/helm/subsystems/post_renderer_main.py b/src/python/pants/backend/helm/subsystems/post_renderer_main.py index c07281db16a..5f68aedc689 100644 --- a/src/python/pants/backend/helm/subsystems/post_renderer_main.py +++ b/src/python/pants/backend/helm/subsystems/post_renderer_main.py @@ -38,7 +38,7 @@ def build_manifest_map(input_file: str) -> dict[str, list[str]]: result = defaultdict(list) template_files = [] - with open(input_file, "r", encoding="utf-8") as f: + with open(input_file, encoding="utf-8") as f: template_files = f.read().split("---") for template in template_files: diff --git a/src/python/pants/backend/openapi/lint/spectral/rules.py b/src/python/pants/backend/openapi/lint/spectral/rules.py index 280ed4734c0..5a9893fc44e 100644 --- a/src/python/pants/backend/openapi/lint/spectral/rules.py +++ b/src/python/pants/backend/openapi/lint/spectral/rules.py @@ -49,7 +49,7 @@ async def run_spectral( ) -> LintResult: transitive_targets = await Get( TransitiveTargets, - TransitiveTargetsRequest((field_set.address for field_set in request.elements)), + TransitiveTargetsRequest(field_set.address for field_set in request.elements), ) all_sources_request = Get( diff --git a/src/python/pants/backend/python/framework/django/detect_apps.py b/src/python/pants/backend/python/framework/django/detect_apps.py index 24e02973da1..b90149a4cec 100644 --- a/src/python/pants/backend/python/framework/django/detect_apps.py +++ b/src/python/pants/backend/python/framework/django/detect_apps.py @@ -42,7 +42,7 @@ def label_to_name(self) -> FrozenDict[str, str]: def label_to_file(self) -> FrozenDict[str, str]: return FrozenDict((label, app.config_file) for label, app in self.items()) - def add_from_json(self, json_bytes: bytes, strip_prefix="") -> "DjangoApps": + def add_from_json(self, json_bytes: bytes, strip_prefix="") -> DjangoApps: json_dict: dict[str, dict[str, str]] = json.loads(json_bytes.decode()) apps = { label: DjangoApp( diff --git a/src/python/pants/backend/python/framework/stevedore/python_target_dependencies_test.py b/src/python/pants/backend/python/framework/stevedore/python_target_dependencies_test.py index 4063dd789a7..972e55f63b8 100644 --- a/src/python/pants/backend/python/framework/stevedore/python_target_dependencies_test.py +++ b/src/python/pants/backend/python/framework/stevedore/python_target_dependencies_test.py @@ -161,10 +161,8 @@ def test_find_python_distributions_with_entry_points_in_stevedore_namespaces( ) ) == set( StevedoreExtensionTargets( - ( - rule_runner.get_target(Address(f"runners/{runner}_runner")) - for runner in sorted(st2_runners) - ) + rule_runner.get_target(Address(f"runners/{runner}_runner")) + for runner in sorted(st2_runners) ) ) diff --git a/src/python/pants/backend/python/lint/isort/rules_integration_test.py b/src/python/pants/backend/python/lint/isort/rules_integration_test.py index 443c80d7559..31ad1a4d060 100644 --- a/src/python/pants/backend/python/lint/isort/rules_integration_test.py +++ b/src/python/pants/backend/python/lint/isort/rules_integration_test.py @@ -158,7 +158,7 @@ def test_invalid_config_file(rule_runner: RuleRunner) -> None: "BUILD": "python_sources()", } ) - tgt = rule_runner.get_target((Address("", relative_file_path="example.py"))) + tgt = rule_runner.get_target(Address("", relative_file_path="example.py")) with pytest.raises(ExecutionError) as isort_error: run_isort(rule_runner, [tgt]) assert any( diff --git a/src/python/pants/backend/python/util_rules/pex_requirements.py b/src/python/pants/backend/python/util_rules/pex_requirements.py index 8852f469972..f63f27f644e 100644 --- a/src/python/pants/backend/python/util_rules/pex_requirements.py +++ b/src/python/pants/backend/python/util_rules/pex_requirements.py @@ -554,10 +554,8 @@ def _invalid_lockfile_error( consumed_msg_parts = [f"`{str(r)}`" for r in user_requirements[0:2]] if len(user_requirements) > 2: consumed_msg_parts.append( - ( - f"{len(user_requirements) - 2} other " - f"{pluralize(len(user_requirements) - 2, 'requirement', include_count=False)}" - ) + f"{len(user_requirements) - 2} other " + f"{pluralize(len(user_requirements) - 2, 'requirement', include_count=False)}" ) yield f"\n\nYou are consuming {comma_separated_list(consumed_msg_parts)} from " diff --git a/src/python/pants/backend/python/util_rules/scripts/pep660_backend_wrapper.py b/src/python/pants/backend/python/util_rules/scripts/pep660_backend_wrapper.py index 75b9931adf8..26e8abeae77 100644 --- a/src/python/pants/backend/python/util_rules/scripts/pep660_backend_wrapper.py +++ b/src/python/pants/backend/python/util_rules/scripts/pep660_backend_wrapper.py @@ -73,7 +73,7 @@ def standardize_dist_info_path(build_dir, metadata_path): # The wrapped backend does not conform to the latest specs. pkg = pkg_version version = "" - with open(os.path.join(build_dir, metadata_path, "METADATA"), "r") as f: + with open(os.path.join(build_dir, metadata_path, "METADATA")) as f: lines = f.readlines() for line in lines: if line.startswith("Version: "): @@ -170,7 +170,7 @@ def main(build_backend, dist_dir, pth_file_path, wheel_config_settings, tags, di if __name__ == "__main__": - with open(sys.argv[1], "r") as f: + with open(sys.argv[1]) as f: settings = json.load(f) main(**settings) diff --git a/src/python/pants/base/exception_sink.py b/src/python/pants/base/exception_sink.py index 3042cd2b901..e2a8861d4aa 100644 --- a/src/python/pants/base/exception_sink.py +++ b/src/python/pants/base/exception_sink.py @@ -92,7 +92,7 @@ def __init__(self, signum, signame): self.signum = signum self.signame = signame self.traceback_lines = traceback.format_stack() - super(SignalHandler.SignalHandledNonLocalExit, self).__init__() + super().__init__() if "I/O operation on closed file" in self.traceback_lines: logger.debug( diff --git a/src/python/pants/core/goals/export.py b/src/python/pants/core/goals/export.py index 45749464044..6569ed5f3b6 100644 --- a/src/python/pants/core/goals/export.py +++ b/src/python/pants/core/goals/export.py @@ -170,7 +170,7 @@ async def export( resolves_exported.add(result.resolve) console.print_stdout(f"Wrote {result.description} to {result_dir}") - unexported_resolves = sorted((set(export_subsys.resolve) - resolves_exported)) + unexported_resolves = sorted(set(export_subsys.resolve) - resolves_exported) if unexported_resolves: all_known_user_resolve_names = await MultiGet( Get(KnownUserResolveNames, KnownUserResolveNamesRequest, request()) diff --git a/src/python/pants/engine/fs.py b/src/python/pants/engine/fs.py index e4bda5b2268..d0bbba20087 100644 --- a/src/python/pants/engine/fs.py +++ b/src/python/pants/engine/fs.py @@ -284,7 +284,7 @@ def __init__( class Workspace(SideEffecting): """A handle for operations that mutate the local filesystem.""" - _scheduler: "SchedulerSession" + _scheduler: SchedulerSession _enforce_effects: bool = True def write_digest( @@ -326,7 +326,7 @@ class SnapshotDiff: changed_files: tuple[str, ...] = () @classmethod - def from_snapshots(cls, ours: Snapshot, theirs: Snapshot) -> "SnapshotDiff": + def from_snapshots(cls, ours: Snapshot, theirs: Snapshot) -> SnapshotDiff: return cls(*ours._diff(theirs)) diff --git a/src/python/pants/engine/goal.py b/src/python/pants/engine/goal.py index 37fff07402b..a8bd026ff48 100644 --- a/src/python/pants/engine/goal.py +++ b/src/python/pants/engine/goal.py @@ -136,7 +136,7 @@ class Outputting: @final @contextmanager - def output(self, console: "Console") -> Iterator[Callable[[str], None]]: + def output(self, console: Console) -> Iterator[Callable[[str], None]]: """Given a Console, yields a function for writing data to stdout, or a file. The passed options instance will generally be the `Goal.Options` of an `Outputting` `Goal`. @@ -146,7 +146,7 @@ def output(self, console: "Console") -> Iterator[Callable[[str], None]]: @final @contextmanager - def output_sink(self, console: "Console") -> Iterator: + def output_sink(self, console: Console) -> Iterator: stdout_file = None if self.output_file: stdout_file = open(self.output_file, "w") @@ -170,7 +170,7 @@ class LineOriented(Outputting): @final @contextmanager - def line_oriented(self, console: "Console") -> Iterator[Callable[[str], None]]: + def line_oriented(self, console: Console) -> Iterator[Callable[[str], None]]: """Given a Console, yields a function for printing lines to stdout or a file. The passed options instance will generally be the `Goal.Options` of an `Outputting` `Goal`. diff --git a/src/python/pants/engine/rules_test.py b/src/python/pants/engine/rules_test.py index f03cc1c17cb..c777440a897 100644 --- a/src/python/pants/engine/rules_test.py +++ b/src/python/pants/engine/rules_test.py @@ -267,7 +267,7 @@ class Example(Goal): @goal_rule async def a_goal_rule_generator(console: Console) -> Example: - a = await Get(A, str("a str!")) + a = await Get(A, str, "a str!") console.print_stdout(str(a)) return Example(exit_code=0) diff --git a/src/python/pants/util/collections_test.py b/src/python/pants/util/collections_test.py index 8eb611a20f1..44bebdbabfb 100644 --- a/src/python/pants/util/collections_test.py +++ b/src/python/pants/util/collections_test.py @@ -97,7 +97,7 @@ def partitioned_buckets(items: list[str]) -> set[tuple[str, ...]]: return {tuple(p) for p in partition_sequentially(items, key=str, size_target=size_target)} # We start with base items containing every other element from a sorted sequence. - all_items = sorted((f"item{i}" for i in range(0, 1024))) + all_items = sorted(f"item{i}" for i in range(0, 1024)) base_items = [item for i, item in enumerate(all_items) if i % 2 == 0] base_partitions = partitioned_buckets(base_items) diff --git a/src/python/pants_release/reversion.py b/src/python/pants_release/reversion.py index adf6241da88..02d76ab586d 100644 --- a/src/python/pants_release/reversion.py +++ b/src/python/pants_release/reversion.py @@ -121,7 +121,7 @@ def reversion( # Get version from the input whl's metadata. input_version = None metadata_file = os.path.join(workspace, dist_info_dir, "METADATA") - with open(metadata_file, "r") as info: + with open(metadata_file) as info: for line in info: mo = _version_re.match(line) if mo: diff --git a/tests/python/pants_test/pantsd/test_lock.py b/tests/python/pants_test/pantsd/test_lock.py index d0ebfba3091..bab9ca25653 100644 --- a/tests/python/pants_test/pantsd/test_lock.py +++ b/tests/python/pants_test/pantsd/test_lock.py @@ -50,7 +50,7 @@ def test_message(self): self.lock_process.start() self.lock_held.wait() self.assertTrue(os.path.exists(self.lock.message_path)) - with open(self.lock.message_path, "r") as f: + with open(self.lock.message_path) as f: message_content = f.read() self.assertIn(str(self.lock_process.pid), message_content) From 91b4d011e3bbc4fac86d7ba3a90694f4f6af9468 Mon Sep 17 00:00:00 2001 From: Josh Cannon Date: Wed, 20 Dec 2023 12:18:21 -0600 Subject: [PATCH 02/18] Ensure `experimental_test_shell_command` runs in relevant environment (#20319) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes #20318 by adding an `EnvironmentField` attribute to `TestShellCommandFieldSet`. Not sure if we ought to add a dedicated test for this. I didn't see one in `src/python/pants/backend/python/goals/pytest_runner_test.py` or `src/python/pants/backend/python/goals/pytest_runner_test.py`. Verified that the test case in the linked issue "passes": ``` josh@cephandrius:~/work/pants$ pants test //:test-bind-mount PRETTY_NAME="Debian GNU/Linux 12 (bookworm)" NAME="Debian GNU/Linux" VERSION_ID="12" VERSION="12 (bookworm)" VERSION_CODENAME=bookworm ID=debian HOME_URL="https://www.debian.org/" SUPPORT_URL="https://www.debian.org/support" BUG_REPORT_URL="https://bugs.debian.org/" ✕ //:test-bind-mount failed in 0.07s (ran in docker environment `docker`). ``` --- src/python/pants/backend/shell/goals/test.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/python/pants/backend/shell/goals/test.py b/src/python/pants/backend/shell/goals/test.py index 547615f259b..8b2595aca02 100644 --- a/src/python/pants/backend/shell/goals/test.py +++ b/src/python/pants/backend/shell/goals/test.py @@ -15,6 +15,7 @@ from pants.backend.shell.util_rules import shell_command from pants.backend.shell.util_rules.shell_command import ShellCommandProcessFromTargetRequest from pants.core.goals.test import TestExtraEnv, TestFieldSet, TestRequest, TestResult, TestSubsystem +from pants.core.util_rules.environments import EnvironmentField from pants.engine.internals.selectors import Get from pants.engine.process import FallibleProcessResult, Process, ProcessCacheScope from pants.engine.rules import collect_rules, rule @@ -23,12 +24,15 @@ from pants.util.logging import LogLevel +@dataclasses.dataclass(frozen=True) class TestShellCommandFieldSet(TestFieldSet): required_fields = ( ShellCommandCommandField, ShellCommandTestDependenciesField, ) + environment: EnvironmentField + @classmethod def opt_out(cls, tgt: Target) -> bool: return tgt.get(SkipShellCommandTestsField).value From fe449a537423d6fa25ef6887e28094b1ec6959c5 Mon Sep 17 00:00:00 2001 From: Gregory Borodin Date: Wed, 20 Dec 2023 21:00:12 +0100 Subject: [PATCH 03/18] Support coursier `--force-version` argument (#20238) Adds `force_version` field to `jvm_artifact` target: ```python jvm_artifact( group="org.apache.logging.log4j", artifact="log4j-core", version="2.19.0", name="log4j-core", force_version=True, ) ``` If set to `True`, pants will pass `--force-version` argument to `coursier fetch` when you run `pants generate-lockfiles` --- src/python/pants/jvm/resolve/common.py | 3 ++ .../pants/jvm/resolve/coursier_fetch.py | 20 ++++++- .../coursier_fetch_integration_test.py | 52 +++++++++++++++++++ src/python/pants/jvm/target_types.py | 15 ++++++ 4 files changed, 88 insertions(+), 2 deletions(-) diff --git a/src/python/pants/jvm/resolve/common.py b/src/python/pants/jvm/resolve/common.py index e6c723d866b..923e4972356 100644 --- a/src/python/pants/jvm/resolve/common.py +++ b/src/python/pants/jvm/resolve/common.py @@ -15,6 +15,7 @@ JvmArtifactArtifactField, JvmArtifactExclusionsField, JvmArtifactFieldSet, + JvmArtifactForceVersionField, JvmArtifactGroupField, JvmArtifactJarSourceField, JvmArtifactUrlField, @@ -152,6 +153,7 @@ class ArtifactRequirement: url: str | None = None jar: JvmArtifactJarSourceField | None = None excludes: frozenset[str] | None = None + force_version: bool = False @classmethod def from_jvm_artifact_target(cls, target: Target) -> ArtifactRequirement: @@ -175,6 +177,7 @@ def from_jvm_artifact_target(cls, target: Target) -> ArtifactRequirement: else None ), excludes=frozenset([*(exclusion.to_coord_str() for exclusion in exclusions)]) or None, + force_version=target[JvmArtifactForceVersionField].value, ) def with_extra_excludes(self, *excludes: str) -> ArtifactRequirement: diff --git a/src/python/pants/jvm/resolve/coursier_fetch.py b/src/python/pants/jvm/resolve/coursier_fetch.py index c2f1517c6b1..8ce7c134884 100644 --- a/src/python/pants/jvm/resolve/coursier_fetch.py +++ b/src/python/pants/jvm/resolve/coursier_fetch.py @@ -304,6 +304,7 @@ def classpath_dest_filename(coord: str, src_filename: str) -> str: @dataclass(frozen=True) class CoursierResolveInfo: coord_arg_strings: FrozenSet[str] + force_version_coord_arg_strings: FrozenSet[str] extra_args: tuple[str, ...] digest: Digest @@ -313,7 +314,13 @@ def argv(self) -> Iterable[str]: Must be used in concert with `digest`. """ - return itertools.chain(self.coord_arg_strings, self.extra_args) + return itertools.chain( + self.coord_arg_strings, + itertools.chain.from_iterable( + zip(itertools.repeat("--force-version"), self.force_version_coord_arg_strings) + ), + self.extra_args, + ) @rule @@ -388,8 +395,17 @@ async def prepare_coursier_resolve_info( ), ) + coord_arg_strings = set() + force_version_coord_arg_strings = set() + for req in to_resolve: + coord_arg_str = req.to_coord_arg_str() + coord_arg_strings.add(coord_arg_str) + if req.force_version: + force_version_coord_arg_strings.add(coord_arg_str) + return CoursierResolveInfo( - coord_arg_strings=frozenset(req.to_coord_arg_str() for req in to_resolve), + coord_arg_strings=frozenset(coord_arg_strings), + force_version_coord_arg_strings=frozenset(force_version_coord_arg_strings), digest=digest, extra_args=tuple(extra_args), ) diff --git a/src/python/pants/jvm/resolve/coursier_fetch_integration_test.py b/src/python/pants/jvm/resolve/coursier_fetch_integration_test.py index b6729a64be7..d47f6e808b8 100644 --- a/src/python/pants/jvm/resolve/coursier_fetch_integration_test.py +++ b/src/python/pants/jvm/resolve/coursier_fetch_integration_test.py @@ -3,6 +3,7 @@ from __future__ import annotations +import dataclasses import textwrap import pytest @@ -723,3 +724,54 @@ def test_failed_to_fetch_jar_given_packaging_pom(rule_runner: RuleRunner) -> Non match=r"Exception: No jar found for org.apache.curator:apache-curator:5.5.0. .*", ): rule_runner.request(CoursierResolvedLockfile, [reqs]) + + +@maybe_skip_jdk_test +def test_force_version(rule_runner): + # first check that force_version=False leads to a different version + reqs = ArtifactRequirements( + [ + Coordinate( + group="org.apache.parquet", + artifact="parquet-common", + version="1.13.1", + ).as_requirement(), + Coordinate( + group="org.slf4j", + artifact="slf4j-api", + version="1.7.19", + ).as_requirement(), + ] + ) + entries = rule_runner.request(CoursierResolvedLockfile, [reqs]).entries + assert Coordinate( + group="org.slf4j", + artifact="slf4j-api", + version="1.7.22", + ) in [e.coord for e in entries] + + # then check force_version=True pins the version + reqs = ArtifactRequirements( + [ + Coordinate( + group="org.apache.parquet", + artifact="parquet-common", + version="1.13.1", + ).as_requirement(), + dataclasses.replace( + Coordinate( + group="org.slf4j", + artifact="slf4j-api", + version="1.7.19", + ).as_requirement(), + force_version=True, + ), + ] + ) + entries = rule_runner.request(CoursierResolvedLockfile, [reqs]).entries + assert Coordinate( + group="org.slf4j", + artifact="slf4j-api", + version="1.7.19", + strict=True, + ) in [e.coord for e in entries] diff --git a/src/python/pants/jvm/target_types.py b/src/python/pants/jvm/target_types.py index 846c8a2bdc7..f7e3be86c7a 100644 --- a/src/python/pants/jvm/target_types.py +++ b/src/python/pants/jvm/target_types.py @@ -19,6 +19,7 @@ from pants.engine.target import ( COMMON_TARGET_FIELDS, AsyncFieldMixin, + BoolField, Dependencies, FieldDefaultFactoryRequest, FieldDefaultFactoryResult, @@ -250,6 +251,18 @@ class JvmArtifactPackagesField(StringSequenceField): ) +class JvmArtifactForceVersionField(BoolField): + alias = "force_version" + default = False + help = help_text( + """ + Force artifact version during resolution. + + If set, pants will pass `--force-version` argument to `coursier fetch` for this artifact. + """ + ) + + class JvmProvidesTypesField(StringSequenceField): alias = "experimental_provides_types" help = help_text( @@ -370,12 +383,14 @@ class JvmArtifactFieldSet(JvmRunnableSourceFieldSet): version: JvmArtifactVersionField packages: JvmArtifactPackagesField url: JvmArtifactUrlField + force_version: JvmArtifactForceVersionField required_fields = ( JvmArtifactGroupField, JvmArtifactArtifactField, JvmArtifactVersionField, JvmArtifactPackagesField, + JvmArtifactForceVersionField, ) From a8aa72c47f19f3cf0a3fd4d2ad89291821b3c711 Mon Sep 17 00:00:00 2001 From: Josh Cannon Date: Thu, 21 Dec 2023 15:09:27 -0600 Subject: [PATCH 04/18] Allow docker image pulling without creds (#20307) Fixes #20306 by letting a failure to get credentials pass through --- .../engine/process_execution/docker/src/docker.rs | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/rust/engine/process_execution/docker/src/docker.rs b/src/rust/engine/process_execution/docker/src/docker.rs index ac712a60f00..1c7fd8bf15e 100644 --- a/src/rust/engine/process_execution/docker/src/docker.rs +++ b/src/rust/engine/process_execution/docker/src/docker.rs @@ -218,9 +218,17 @@ async fn credentials_for_image( // TODO: https://github.com/keirlawson/docker_credential/issues/7 means that this will only // work for credential helpers and credentials encoded directly in the docker config, // rather than for general credStore implementations. - let credential = docker_credential::get_credential(&server).map_err(|e| { - format!("Failed to retrieve credentials for server `{server}`: {e}") - })?; + let credential = match docker_credential::get_credential(&server) { + Ok(credential) => credential, + Err(e) => { + log::warn!( + "Failed to retrieve Docker credentials for server `{}`: {}", + server, + e + ); + return Ok(None); + } + }; let bollard_credentials = match credential { docker_credential::DockerCredential::IdentityToken(token) => { From 75888cba9f83a10234abce239ccb199c97d63e3c Mon Sep 17 00:00:00 2001 From: Benjy Weinberger Date: Thu, 21 Dec 2023 15:20:55 -0800 Subject: [PATCH 05/18] Change the slug of the Shell backend doc. (#20326) It used to be "shell", now it's "shell-overview" because we were getting 401s from readme.com for that page, and their support claims we discovered a bug whereby "shell" is not a valid slug. This will all be moot once we switch off readme, but this is a bandaid for now. --- docs/markdown/Introduction/welcome-to-pants.md | 2 +- docs/markdown/Shell/shell.md | 2 +- docs/markdown/Using Pants/concepts/enabling-backends.md | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/markdown/Introduction/welcome-to-pants.md b/docs/markdown/Introduction/welcome-to-pants.md index 7410d542f08..2d3be1999d9 100644 --- a/docs/markdown/Introduction/welcome-to-pants.md +++ b/docs/markdown/Introduction/welcome-to-pants.md @@ -42,7 +42,7 @@ Pants is designed for fast, consistent, ergonomic builds. Some noteworthy featur Which languages and frameworks does Pants support? ================================================== -- Pants [ships](page:language-support) with support for [Python](doc:python), [Go](doc:go), [Java](doc:jvm-overview), [Scala](doc:jvm-overview) and [Shell](doc:shell). +- Pants [ships](page:language-support) with support for [Python](doc:python), [Go](doc:go), [Java](doc:jvm-overview), [Scala](doc:jvm-overview) and [Shell](doc:shell-overview). - Pants supports a wide range of code generators (such as Thrift, Protobuf, Scrooge and Avro), linters and formatters, and it is easy to add support for new or custom ones - Pants can create standalone binaries, [Docker images](doc:docker), AWS Lambdas and GCP Cloud Functions diff --git a/docs/markdown/Shell/shell.md b/docs/markdown/Shell/shell.md index 21e8228cc00..347cdf0f923 100644 --- a/docs/markdown/Shell/shell.md +++ b/docs/markdown/Shell/shell.md @@ -1,6 +1,6 @@ --- title: "Shell overview" -slug: "shell" +slug: "shell-overview" excerpt: "Pants's support for Shellcheck, shfmt, and shUnit2." hidden: false createdAt: "2021-04-14T04:21:15.028Z" diff --git a/docs/markdown/Using Pants/concepts/enabling-backends.md b/docs/markdown/Using Pants/concepts/enabling-backends.md index 94ca1b196c2..426c150869d 100644 --- a/docs/markdown/Using Pants/concepts/enabling-backends.md +++ b/docs/markdown/Using Pants/concepts/enabling-backends.md @@ -47,12 +47,12 @@ This list is also available via `pants backends --help`, which includes any addi | `pants.backend.python.lint.pyupgrade` | Enables Pyupgrade, which upgrades to new Python syntax: | [Linters and formatters](doc:python-linters-and-formatters) | | `pants.backend.python.lint.yapf` | Enables Yapf, the Python formatter: | [Linters and formatters](doc:python-linters-and-formatters) | | `pants.backend.python.typecheck.mypy` | Enables MyPy, the Python type checker: . | [typecheck](doc:python-check-goal) | -| `pants.backend.shell` | Core Shell support, including shUnit2 test runner. | [Shell overview](doc:shell) | -| `pants.backend.shell.lint.shfmt` | Enables shfmt, a Shell autoformatter: . | [Shell overview](doc:shell) | -| `pants.backend.shell.lint.shellcheck` | Enables Shellcheck, a Shell linter: . | [Shell overview](doc:shell) | +| `pants.backend.shell` | Core Shell support, including shUnit2 test runner. | [Shell overview](doc:shell-overview) | +| `pants.backend.shell.lint.shfmt` | Enables shfmt, a Shell autoformatter: . | [Shell overview](doc:shell-overview) | +| `pants.backend.shell.lint.shellcheck` | Enables Shellcheck, a Shell linter: . | [Shell overview](doc:shell-overview) | | `pants.backend.tools.preamble` | Enables "preamble", a Pants fixer for copyright headers and shebang lines | [`preamble`](doc:reference-preamble) | | `pants.backend.tools.taplo` | Enables Taplo, a TOML autoformatter: | | -| `pants.backend.url_handlers.s3` | Enables accessing s3 via credentials in `file(source=http_source(...))` | | +| `pants.backend.url_handlers.s3` | Enables accessing s3 via credentials in `file(source=http_source(...))` | | Available experimental backends ------------------------------- From 8ce469730397bee7a9858b7730a62818a42a90d8 Mon Sep 17 00:00:00 2001 From: Benjy Weinberger Date: Fri, 22 Dec 2023 14:20:29 -0800 Subject: [PATCH 06/18] Implement string interpolation for config values in Rust. (#20316) This feature exists in the Python config implementation, but did not exist in the Rust equivalent, since the options consumed by the native client did not require it. However as we plan to use the Rust implementation throughout and get rid of the Python implementation, we must support this. Also updates the documentation of this feature to match what we do in practice. --- docs/markdown/Using Pants/concepts/options.md | 34 ++-- src/rust/engine/Cargo.lock | 13 ++ src/rust/engine/Cargo.toml | 1 + src/rust/engine/options/Cargo.toml | 3 + src/rust/engine/options/src/config.rs | 173 +++++++++++++++--- src/rust/engine/options/src/config_tests.rs | 128 ++++++++++++- src/rust/engine/options/src/lib.rs | 58 ++++-- 7 files changed, 358 insertions(+), 52 deletions(-) diff --git a/docs/markdown/Using Pants/concepts/options.md b/docs/markdown/Using Pants/concepts/options.md index 32ff768d765..42d0feeef35 100644 --- a/docs/markdown/Using Pants/concepts/options.md +++ b/docs/markdown/Using Pants/concepts/options.md @@ -81,23 +81,33 @@ Note that any dashes in the option flag name are converted to underscores: `--mu ### Config file interpolation -Environment variables can be interpolated by using the syntax `%(env.ENV_VAR)s`, e.g.: +A string value in a config file can contain placeholders of the form `%(key)s`, which will be replaced with a corresponding value. The `key` can be one of: + +- A string-valued option in the DEFAULT section of the same config file. +- A string-valued option in the same section of the config file as the value containing the placeholder. +- Any environment variable, prefixed with `env.`: `%(env.ENV_VAR)s`. +- The following special values: + - `%(buildroot)s`: absolute path to the root of your repository. + - `%(homedir)s`: equivalent to `$HOME` or `~`. + - `%(user)s`: the current user's username, obtained from the system password file. + - `%(pants_workdir)s`: the absolute path of the global option `--pants-workdir`, which defaults + to `{buildroot}/.pants.d/`. + - `%(pants_distdir)s`: the absolute path of the global option `--pants-distdir`, which defaults + to `{buildroot}/dist/`. + +An interpolated value may itself contain placeholders, that will be recursively interpolated. + +For example: ```toml pants.toml +[DEFAULT] +domain = "my.domain" + [python-repos] -# This will substitute `%(env.PY_REPO)s` with the value of the environment -# variable PY_REPO -indexes.add = ["http://%(env.PY_REPO)s@my.custom.repo/index +repo_host = "repo.%(domain)s" +indexes.add = ["http://%(env.PY_REPO)s@%(repo_host)s/index ``` -Additionally, a few special values are pre-populated with the `%(var)s` syntax: - -- `%(buildroot)s`: absolute path to the root of your repository -- `%(homedir)s`: equivalent to `$HOME` or `~` -- `%(user)s`: equivalent to `$USER` -- `%(pants_distdir)s`: absolute path of the global option `--pants-distdir`, which defaults - to `{buildroot}/dist/` - Option types ============ diff --git a/src/rust/engine/Cargo.lock b/src/rust/engine/Cargo.lock index 47119f51880..c7c9acea3b8 100644 --- a/src/rust/engine/Cargo.lock +++ b/src/rust/engine/Cargo.lock @@ -2317,11 +2317,14 @@ checksum = "77af24da69f9d9341038eba93a073b1fdaaa1b788221b00a69bce9e762cb32de" name = "options" version = "0.0.1" dependencies = [ + "lazy_static", "log", "peg", + "regex", "shellexpand", "tempfile", "toml", + "whoami", ] [[package]] @@ -4461,6 +4464,16 @@ dependencies = [ "thiserror", ] +[[package]] +name = "whoami" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22fc3756b8a9133049b26c7f61ab35416c130e8c09b660f5b3958b446f52cc50" +dependencies = [ + "wasm-bindgen", + "web-sys", +] + [[package]] name = "winapi" version = "0.3.9" diff --git a/src/rust/engine/Cargo.toml b/src/rust/engine/Cargo.toml index 3ebe8c90c1f..89a272ebb97 100644 --- a/src/rust/engine/Cargo.toml +++ b/src/rust/engine/Cargo.toml @@ -326,6 +326,7 @@ url = "2.4" uuid = "1.6.1" walkdir = "2" webpki = "0.22" +whoami = "1.4.1" # NB: If a change to these versions requires cache busting, bump the version of # `src/rust/engine/dep_inference/Cargo.toml`. diff --git a/src/rust/engine/options/Cargo.toml b/src/rust/engine/options/Cargo.toml index 1d2ffa3b633..4c02a817576 100644 --- a/src/rust/engine/options/Cargo.toml +++ b/src/rust/engine/options/Cargo.toml @@ -6,10 +6,13 @@ authors = ["Pants Build "] publish = false [dependencies] +lazy_static = { workspace = true } log = { workspace = true } peg = { workspace = true } shellexpand = { workspace = true } toml = { workspace = true } +regex = { workspace = true } +whoami = { workspace = true } [dev-dependencies] tempfile = { workspace = true } diff --git a/src/rust/engine/options/src/config.rs b/src/rust/engine/options/src/config.rs index 43c674bd26a..130112b2d78 100644 --- a/src/rust/engine/options/src/config.rs +++ b/src/rust/engine/options/src/config.rs @@ -1,11 +1,13 @@ // Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). // Licensed under the Apache License, Version 2.0 (see LICENSE). -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use std::fs; use std::mem; use std::path::Path; +use lazy_static::lazy_static; +use regex::Regex; use toml::value::Table; use toml::Value; @@ -13,6 +15,84 @@ use super::id::{NameTransform, OptionId}; use super::parse::parse_string_list; use super::{ListEdit, ListEditAction, OptionsSource}; +type InterpolationMap = HashMap; + +lazy_static! { + static ref PLACEHOLDER_RE: Regex = Regex::new(r"%\(([a-zA-Z0-9_.]+)\)s").unwrap(); +} + +pub(crate) fn interpolate_string( + value: String, + replacements: &InterpolationMap, +) -> Result { + let caps_vec: Vec<_> = PLACEHOLDER_RE.captures_iter(&value).collect(); + if caps_vec.is_empty() { + return Ok(value); + } + + let mut new_value = String::with_capacity(value.len()); + let mut last_match = 0; + for caps in caps_vec { + let m = caps.get(0).unwrap(); + new_value.push_str(&value[last_match..m.start()]); + let placeholder_name = &caps[1]; + let replacement = replacements.get(placeholder_name).ok_or(format!( + "Unknown value for placeholder `{}`", + placeholder_name + ))?; + new_value.push_str(replacement); + last_match = m.end(); + } + new_value.push_str(&value[last_match..]); + // A replacement string may itself contain a placeholder, so we recurse. + interpolate_string(new_value, replacements) +} + +struct InterpolationError { + key: String, + msg: String, +} + +fn interpolate_value( + key: &str, + value: Value, + replacements: &InterpolationMap, +) -> Result { + Ok(match value { + Value::String(s) => Value::String(interpolate_string(s, replacements).map_err(|msg| { + InterpolationError { + key: key.to_string(), + msg, + } + })?), + Value::Array(v) => { + let new_v: Result, _> = v + .into_iter() + .map(|x| interpolate_value(key, x, replacements)) + .collect(); + Value::Array(new_v?) + } + Value::Table(t) => { + let new_items: Result, _> = t + .into_iter() + .map(|(k, v)| { + match interpolate_value( + // Use the section-level key even if this is a nested table value. + if key.is_empty() { &k } else { key }, + v, + replacements, + ) { + Ok(new_v) => Ok((k, new_v)), + Err(s) => Err(s), + } + }) + .collect(); + Value::Table(new_items?.into_iter().collect()) + } + _ => value, + }) +} + #[derive(Clone)] pub(crate) struct Config { config: Value, @@ -25,7 +105,10 @@ impl Config { } } - pub(crate) fn parse>(file: P) -> Result { + pub(crate) fn parse>( + file: P, + seed_values: &InterpolationMap, + ) -> Result { let config_contents = fs::read_to_string(&file).map_err(|e| { format!( "Failed to read config file {}: {}", @@ -40,36 +123,80 @@ impl Config { e ) })?; - if !config.is_table() { - return Err(format!( + + fn add_section_to_interpolation_map( + mut imap: InterpolationMap, + section: Option<&Value>, + ) -> Result { + if let Some(section) = section { + if let Some(table) = section.as_table() { + for (key, value) in table.iter() { + if let Value::String(s) = value { + imap.insert(key.clone(), s.clone()); + } + } + } + } + Ok(imap) + } + + let default_imap = + add_section_to_interpolation_map(seed_values.clone(), config.get("DEFAULT"))?; + + let new_sections: Result, String> = match config { + Value::Table(t) => t + .into_iter() + .map(|(section_name, section)| { + if !section.is_table() { + return Err(format!( + "Expected the config file {} to contain tables per section, \ + but section {} contained a {}: {}", + file.as_ref().display(), + section_name, + section.type_str(), + section + )); + } + let section_imap = if section_name == "DEFAULT" { + default_imap.clone() + } else { + add_section_to_interpolation_map(default_imap.clone(), Some(§ion))? + }; + let new_section = interpolate_value("", section.clone(), §ion_imap) + .map_err(|e| { + format!( + "{} in config file {}, section {}, key {}", + e.msg, + file.as_ref().display(), + section_name, + e.key + ) + })?; + Ok((section_name, new_section)) + }) + .collect(), + + _ => Err(format!( "Expected the config file {} to contain a table but contained a {}: {}", file.as_ref().display(), config.type_str(), config - )); - } - if let Some((key, section)) = config - .as_table() - .unwrap() - .iter() - .find(|(_, section)| !section.is_table()) - { - return Err(format!( - "Expected the config file {} to contain tables per section, but section {} contained a {}: {}", - file.as_ref().display(), - key, - section.type_str(), - section - )); - } + )), + }; - Ok(Config { config }) + let new_table = Table::from_iter(new_sections?); + Ok(Config { + config: Value::Table(new_table), + }) } - pub(crate) fn merged>(files: &[P]) -> Result { + pub(crate) fn merged>( + files: &[P], + seed_values: &InterpolationMap, + ) -> Result { files .iter() - .map(Config::parse) + .map(|f| Config::parse(f, seed_values)) .try_fold(Config::default(), |config, parse_result| { parse_result.map(|parsed| config.merge(parsed)) }) diff --git a/src/rust/engine/options/src/config_tests.rs b/src/rust/engine/options/src/config_tests.rs index a5652bcdf1b..8fd9dbe504a 100644 --- a/src/rust/engine/options/src/config_tests.rs +++ b/src/rust/engine/options/src/config_tests.rs @@ -1,15 +1,17 @@ // Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). // Licensed under the Apache License, Version 2.0 (see LICENSE). +use regex::Regex; +use std::collections::HashMap; use std::fs::File; use std::io::Write; -use crate::config::Config; -use crate::{option_id, OptionId, OptionsSource}; +use crate::config::{interpolate_string, Config}; +use crate::{option_id, ListEdit, ListEditAction, OptionId, OptionsSource}; use tempfile::TempDir; -fn config>(file_contents: I) -> Config { +fn maybe_config>(file_contents: I) -> Result { let dir = TempDir::new().unwrap(); let files = file_contents .into_iter() @@ -23,7 +25,17 @@ fn config>(file_contents: I) -> Config { path }) .collect::>(); - Config::merged(&files).unwrap() + Config::merged( + &files, + &HashMap::from([ + ("seed1".to_string(), "seed1val".to_string()), + ("seed2".to_string(), "seed2val".to_string()), + ]), + ) +} + +fn config>(file_contents: I) -> Config { + maybe_config(file_contents).unwrap() } #[test] @@ -63,3 +75,111 @@ fn test_section_overlap() { assert_string("something", option_id!(["section"], "field1")); assert_string("something else", option_id!(["section"], "field2")); } + +#[test] +fn test_interpolate_string() { + fn interp( + template: &str, + interpolations: Vec<(&'static str, &'static str)>, + ) -> Result { + let interpolation_map: HashMap<_, _> = interpolations + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect(); + interpolate_string(template.to_string(), &interpolation_map) + } + + let template = "%(greeting)s world, what's your %(thing)s?"; + let replacements = vec![("greeting", "Hello"), ("thing", "deal")]; + assert_eq!( + "Hello world, what's your deal?", + interp(template, replacements).unwrap() + ); + + let template = "abc %(d5f_g)s hij"; + let replacements = vec![("d5f_g", "defg"), ("unused", "xxx")]; + assert_eq!("abc defg hij", interp(template, replacements).unwrap()); + + let template = "%(known)s %(unknown)s"; + let replacements = vec![("known", "aaa"), ("unused", "xxx")]; + let result = interp(template, replacements); + assert!(result.is_err()); + assert_eq!( + "Unknown value for placeholder `unknown`", + result.unwrap_err() + ); + + let template = "%(greeting)s world, what's your %(thing)s?"; + let replacements = vec![ + ("greeting", "Hello"), + ("thing", "real %(deal)s"), + ("deal", "name"), + ]; + assert_eq!( + "Hello world, what's your real name?", + interp(template, replacements).unwrap() + ); +} + +#[test] +fn test_interpolate_config() { + let conf = config(["[DEFAULT]\n\ + field1 = 'something'\n\ + color = 'black'\n\ + [foo]\n\ + field2 = '%(field1)s else'\n\ + field3 = 'entirely'\n\ + field4 = '%(field2)s %(field3)s %(seed2)s'\n\ + [groceries]\n\ + berryprefix = 'straw'\n\ + stringlist.add = ['apple', '%(berryprefix)sberry', 'banana']\n\ + stringlist.remove = ['%(color)sberry', 'pear']\n\ + inline_table = { fruit = '%(berryprefix)sberry', spice = '%(color)s pepper' }"]); + + assert_eq!( + "something else entirely seed2val", + conf.get_string(&option_id!(["foo"], "field4")) + .unwrap() + .unwrap() + ); + + assert_eq!( + vec![ + ListEdit { + action: ListEditAction::Add, + items: vec![ + "apple".to_string(), + "strawberry".to_string(), + "banana".to_string(), + ], + }, + ListEdit { + action: ListEditAction::Remove, + items: vec!["blackberry".to_string(), "pear".to_string()] + } + ], + conf.get_string_list(&option_id!(["groceries"], "stringlist")) + .unwrap() + .unwrap() + ); + + // TODO: Uncomment when we implement get_dict. + // assert_eq!( + // HashMap::from([("fruit", "strawberry"), ("spice", "black pepper")]), + // conf.get_dict(&option_id!(["groceries"], "inline_table")).unwrap().unwrap() + // ); + + let bad_conf = maybe_config(["[DEFAULT]\n\ + field1 = 'something'\n\ + [foo]\n\ + bad_field = '%(unknown)s'\n"]); + let err_msg = bad_conf.err().unwrap(); + let pat = + r"^Unknown value for placeholder `unknown` in config file .*, section foo, key bad_field$"; + assert!( + Regex::new(pat).unwrap().is_match(&err_msg), + "Error message: {}\nDid not match: {}", + &err_msg, + pat + ); +} diff --git a/src/rust/engine/options/src/lib.rs b/src/rust/engine/options/src/lib.rs index 02f6f1b2856..1d44844d14c 100644 --- a/src/rust/engine/options/src/lib.rs +++ b/src/rust/engine/options/src/lib.rs @@ -27,9 +27,10 @@ mod parse_tests; mod types; -use std::collections::{BTreeMap, HashSet}; +use std::collections::{BTreeMap, HashMap, HashSet}; use std::ops::Deref; use std::os::unix::ffi::OsStrExt; +use std::path; use std::path::Path; use std::rc::Rc; @@ -158,6 +159,22 @@ pub struct OptionParser { impl OptionParser { pub fn new(env: Env, args: Args) -> Result { + let buildroot = BuildRoot::find()?; + let buildroot_string = String::from_utf8(buildroot.as_os_str().as_bytes().to_vec()) + .map_err(|e| { + format!( + "Failed to decode build root path {}: {}", + buildroot.display(), + e + ) + })?; + + let mut seed_values = HashMap::from_iter( + env.env + .iter() + .map(|(k, v)| (format!("env.{k}", k = k), v.clone())), + ); + let mut sources: BTreeMap> = BTreeMap::new(); sources.insert(Source::Env, Rc::new(env)); sources.insert(Source::Flag, Rc::new(args)); @@ -165,20 +182,35 @@ impl OptionParser { sources: sources.clone(), }; - let config_path = BuildRoot::find()?.join("pants.toml"); + fn path_join(a: &str, b: &str) -> String { + format!("{}{}{}", a, path::MAIN_SEPARATOR, b) + } + + let default_config_path = path_join(&buildroot_string, "pants.toml"); let repo_config_files = parser.parse_string_list( &option_id!("pants", "config", "files"), - &[ - std::str::from_utf8(config_path.as_os_str().as_bytes()).map_err(|e| { - format!( - "Failed to decode build root path {}: {}", - config_path.display(), - e - ) - })?, - ], + &[&default_config_path], )?; - let mut config = Config::merged(&repo_config_files)?; + + let subdir = |subdir_name: &str, default: &str| -> Result { + Ok(parser + .parse_string( + &OptionId::new(Scope::Global, ["pants", subdir_name].iter(), None)?, + &path_join(&buildroot_string, default), + )? + .value + .clone()) + }; + + seed_values.extend([ + ("buildroot".to_string(), buildroot_string.clone()), + ("homedir".to_string(), shellexpand::tilde("~").into_owned()), + ("user".to_string(), whoami::username()), + ("pants_workdir".to_string(), subdir("workdir", ".pants.d")?), + ("pants_distdir".to_string(), subdir("distdir", "dist")?), + ]); + + let mut config = Config::merged(&repo_config_files, &seed_values)?; sources.insert(Source::Config, Rc::new(config.clone())); parser = OptionParser { sources: sources.clone(), @@ -191,7 +223,7 @@ impl OptionParser { )? { let rcfile_path = Path::new(&rcfile); if rcfile_path.exists() { - let rc_config = Config::parse(rcfile_path)?; + let rc_config = Config::parse(rcfile_path, &seed_values)?; config = config.merge(rc_config); } } From 9d95c1d0a56fd2b53665898096b6cb45e228afa3 Mon Sep 17 00:00:00 2001 From: Andreas Stenius Date: Fri, 22 Dec 2023 17:27:38 -0500 Subject: [PATCH 07/18] Update team.md (#20329) Add @krishnan-chandra to the list of Contributors. --- docs/markdown/Getting Help/the-pants-community/team.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/markdown/Getting Help/the-pants-community/team.md b/docs/markdown/Getting Help/the-pants-community/team.md index 913f5a24193..5d49af5d672 100644 --- a/docs/markdown/Getting Help/the-pants-community/team.md +++ b/docs/markdown/Getting Help/the-pants-community/team.md @@ -24,13 +24,13 @@ Pants open source project has had many team members in over a decade. The curren | **Benjy Weinberger** | "The options system — unifying our flags, config and env vars into a coherent, extensible framework." | | **Carina C. Zona** | "Developing the #welcome channel on Pants community chat" | | **Christopher Neugebauer** | Adding multi-architecture Mac support for Apple Silicon (M1s) | -| **Dan Moran** | "Enabling batched `pytest` execution in `pants test`" | +| **Dan Moran** | "Enabling batched `pytest` execution in `pants test`" | | **Daniel Wagner-Hall** | Laid the foundation for remote execution | | **Danny McClanahan** | Implemented bounded runtime polymorphism with union rules and improved plugin UX with async/await | | **Eric Arellano** | "Migrating Pants to Python 3 for my internship project" | | **Henry Fuller** | "Working on replacing watchman with a kernel based file watcher in the V2 engine" | -| **Huon Wilson** | "Improving the experiencing of using `mypy` with Pants" | -| **Jacob Floyd** | Gave the idea for `skip_flake8` et al, which grew into a flagship feature: incremental adoption | +| **Huon Wilson** | "Improving the experiencing of using `mypy` with Pants" | +| **Jacob Floyd** | Gave the idea for `skip_flake8` et al, which grew into a flagship feature: incremental adoption | | **John Sirois** | | | **Josh Reed** | "Remains continually willing to bikeshed about design decisions on the Pants Slack" | | **Joshua Cannon** | "I'm proud and thankful my voice can and is being used to shape Pants inside and out" | @@ -53,6 +53,7 @@ Pants open source project has had many team members in over a decade. The curren | **Doron Somech** | "Improving Scala dependency inference to support our codebase" | | **Gautham Nair** | | | **Jonas Stendahl** | "Adding formatting and linting support for Protobuf" | +| **Krishnan Chandra** | "Adding support for the Ruff formatter to Pants" | | **Marcelo Trylesinski** | "Improving onboarding experience" | | **Nick Grisafi** | "Participating in podcasts with maintainers (Eric and Josh) on developer experience and Pants!" | | **Raúl Cuza** | "The first time I was able to help someone else with pants on the Slack ." | From 353b7486eaca903539df46ea938f51996f99787e Mon Sep 17 00:00:00 2001 From: Benjy Weinberger Date: Sat, 23 Dec 2023 20:10:44 -0800 Subject: [PATCH 08/18] Support allow_pantsrc to turn off pantsrc processing in rust (#20330) This is part of the Python config implementation, this change brings the Rust implementation up to par. This is needed in tests, so that they don't consume a real pantsrc by mistake. Also adds `.pants.rc` as a default pantsrc file, which exists in the `pantsrc` option's registration on the Python side, but was not used on the Rust side (which has to repeat the registration defaults since it cannot access them from the Python side). --- src/rust/engine/client/src/client_tests.rs | 1 + src/rust/engine/client/src/main.rs | 2 +- src/rust/engine/options/src/lib.rs | 10 +++++++--- src/rust/engine/pantsd/src/pantsd_testing.rs | 2 +- src/rust/engine/src/externs/pantsd.rs | 4 ++-- 5 files changed, 12 insertions(+), 7 deletions(-) diff --git a/src/rust/engine/client/src/client_tests.rs b/src/rust/engine/client/src/client_tests.rs index 519e3ba7cb4..8aa5e0ce27a 100644 --- a/src/rust/engine/client/src/client_tests.rs +++ b/src/rust/engine/client/src/client_tests.rs @@ -38,6 +38,7 @@ async fn test_client_fingerprint_mismatch() { "--pants-subprocessdir={}", tmpdir.path().display() )]), + true, ) .unwrap(); let error = pantsd::find_pantsd(&build_root, &options_parser) diff --git a/src/rust/engine/client/src/main.rs b/src/rust/engine/client/src/main.rs index 43642ce5b23..f8462c6295e 100644 --- a/src/rust/engine/client/src/main.rs +++ b/src/rust/engine/client/src/main.rs @@ -32,7 +32,7 @@ async fn execute(start: SystemTime) -> Result { let (env, dropped) = Env::capture_lossy(); let env_items = (&env).into(); let argv = env::args().collect::>(); - let options_parser = OptionParser::new(env, Args::argv())?; + let options_parser = OptionParser::new(env, Args::argv(), true)?; let use_pantsd = options_parser.parse_bool(&option_id!("pantsd"), true)?; if !use_pantsd.value { diff --git a/src/rust/engine/options/src/lib.rs b/src/rust/engine/options/src/lib.rs index 1d44844d14c..680dd4484c6 100644 --- a/src/rust/engine/options/src/lib.rs +++ b/src/rust/engine/options/src/lib.rs @@ -158,7 +158,7 @@ pub struct OptionParser { } impl OptionParser { - pub fn new(env: Env, args: Args) -> Result { + pub fn new(env: Env, args: Args, allow_pantsrc: bool) -> Result { let buildroot = BuildRoot::find()?; let buildroot_string = String::from_utf8(buildroot.as_os_str().as_bytes().to_vec()) .map_err(|e| { @@ -216,10 +216,14 @@ impl OptionParser { sources: sources.clone(), }; - if *parser.parse_bool(&option_id!("pantsrc"), true)? { + if allow_pantsrc && *parser.parse_bool(&option_id!("pantsrc"), true)? { for rcfile in parser.parse_string_list( &option_id!("pantsrc", "files"), - &["/etc/pantsrc", shellexpand::tilde("~/.pants.rc").as_ref()], + &[ + "/etc/pantsrc", + shellexpand::tilde("~/.pants.rc").as_ref(), + ".pants.rc", + ], )? { let rcfile_path = Path::new(&rcfile); if rcfile_path.exists() { diff --git a/src/rust/engine/pantsd/src/pantsd_testing.rs b/src/rust/engine/pantsd/src/pantsd_testing.rs index 696cd33d913..bb8849fcff2 100644 --- a/src/rust/engine/pantsd/src/pantsd_testing.rs +++ b/src/rust/engine/pantsd/src/pantsd_testing.rs @@ -26,7 +26,7 @@ pub fn launch_pantsd() -> (BuildRoot, OptionParser, TempDir) { "-V".to_owned(), ]; let options_parser = - OptionParser::new(Env::new(HashMap::new()), Args::new(args.clone())).unwrap(); + OptionParser::new(Env::new(HashMap::new()), Args::new(args.clone()), true).unwrap(); let mut cmd = Command::new(build_root.join("pants")); cmd.current_dir(build_root.as_path()) diff --git a/src/rust/engine/src/externs/pantsd.rs b/src/rust/engine/src/externs/pantsd.rs index b0d72ee5ab2..979434cb112 100644 --- a/src/rust/engine/src/externs/pantsd.rs +++ b/src/rust/engine/src/externs/pantsd.rs @@ -21,8 +21,8 @@ pub fn register(_py: Python, m: &PyModule) -> PyResult<()> { #[pyfunction] fn pantsd_fingerprint_compute(expected_option_names: HashSet) -> PyResult { let build_root = BuildRoot::find().map_err(PyException::new_err)?; - let options_parser = - OptionParser::new(Env::capture_lossy().0, Args::argv()).map_err(PyException::new_err)?; + let options_parser = OptionParser::new(Env::capture_lossy().0, Args::argv(), true) + .map_err(PyException::new_err)?; let options = pantsd::fingerprinted_options(&build_root).map_err(PyException::new_err)?; let actual_option_names = options From 54ec52a2ee2c62c4169970b8b930ca3a6dcdd9c6 Mon Sep 17 00:00:00 2001 From: Benjy Weinberger Date: Sun, 24 Dec 2023 07:29:15 -0800 Subject: [PATCH 09/18] Fix algorithm for gathering prebuilt Go object files (#20332) The previous algorithm did not check if a package had already been traversed, which can lead to exponential blowup of the queue. This is an identical fix to the one in #20030, which was for coverage build requests. These are the only two places in the go backend (that I can find) where a transitive walk like this is done. Unlike #20030 this one is not known to have caused a problem for any users in practice. --- src/python/pants/backend/go/util_rules/build_pkg.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/python/pants/backend/go/util_rules/build_pkg.py b/src/python/pants/backend/go/util_rules/build_pkg.py index 1b639c7d965..c0f34277e73 100644 --- a/src/python/pants/backend/go/util_rules/build_pkg.py +++ b/src/python/pants/backend/go/util_rules/build_pkg.py @@ -471,9 +471,12 @@ async def _gather_transitive_prebuilt_object_files( prebuilt_objects: list[tuple[Digest, list[str]]] = [] queue: deque[BuildGoPackageRequest] = deque([build_request]) + seen: set[BuildGoPackageRequest] = {build_request} while queue: pkg = queue.popleft() - queue.extend(pkg.direct_dependencies) + unseen = [dd for dd in build_request.direct_dependencies if dd not in seen] + queue.extend(unseen) + seen.update(unseen) if pkg.prebuilt_object_files: prebuilt_objects.append( ( @@ -695,7 +698,7 @@ async def build_go_package( ) symabis_path = symabis_result.symabis_path - # Build the arguments for compiling the Go coe in this package. + # Build the arguments for compiling the Go code in this package. compile_args = [ "tool", "compile", From 550cc244505953459337504aa5a4cf85f8fff66c Mon Sep 17 00:00:00 2001 From: Josh Cannon Date: Tue, 26 Dec 2023 15:02:37 -0600 Subject: [PATCH 10/18] Make docker_environment's Pants-provided-PBS an absolute path (#20314) Fixes #20313 by ensuring that the `PEX_PYTHON` used for Pex CLI invocations inside a docker environment is an absolute path. I also did some drive-by improvements of the test code. --- .../pants/core/util_rules/adhoc_binaries.py | 5 ++- .../core/util_rules/adhoc_binaries_test.py | 42 +++++++++---------- 2 files changed, 24 insertions(+), 23 deletions(-) diff --git a/src/python/pants/core/util_rules/adhoc_binaries.py b/src/python/pants/core/util_rules/adhoc_binaries.py index d69b4932fcf..b0d3642e614 100644 --- a/src/python/pants/core/util_rules/adhoc_binaries.py +++ b/src/python/pants/core/util_rules/adhoc_binaries.py @@ -104,10 +104,11 @@ async def download_python_binary( cp -r python "{installation_root}" touch "{installation_root}/DONE" fi + echo "$(realpath "{installation_root}")/bin/python3" """ ) - await Get( + result = await Get( ProcessResult, Process( [bash_binary.path, "-c", installation_script], @@ -123,7 +124,7 @@ async def download_python_binary( ), ) - return _PythonBuildStandaloneBinary(f"{installation_root}/bin/python3") + return _PythonBuildStandaloneBinary(result.stdout.decode().splitlines()[-1].strip()) @dataclass(frozen=True) diff --git a/src/python/pants/core/util_rules/adhoc_binaries_test.py b/src/python/pants/core/util_rules/adhoc_binaries_test.py index 3a4e3bb2f3a..f70acbbcc60 100644 --- a/src/python/pants/core/util_rules/adhoc_binaries_test.py +++ b/src/python/pants/core/util_rules/adhoc_binaries_test.py @@ -6,14 +6,17 @@ import pytest from pants.build_graph.address import Address -from pants.core.target_types import FileTarget from pants.core.util_rules import adhoc_binaries from pants.core.util_rules.adhoc_binaries import ( - PythonBuildStandaloneBinary, _DownloadPythonBuildStandaloneBinaryRequest, _PythonBuildStandaloneBinary, ) -from pants.core.util_rules.environments import EnvironmentTarget, LocalEnvironmentTarget +from pants.core.util_rules.environments import ( + DockerEnvironmentTarget, + EnvironmentTarget, + LocalEnvironmentTarget, +) +from pants.engine.environment import EnvironmentName from pants.testutil.rule_runner import MockGet, QueryRule, RuleRunner, run_rule_with_mocks @@ -27,7 +30,7 @@ def rule_runner() -> RuleRunner: [_DownloadPythonBuildStandaloneBinaryRequest], ), ], - target_types=[LocalEnvironmentTarget, FileTarget], + target_types=[LocalEnvironmentTarget, DockerEnvironmentTarget], ) @@ -47,32 +50,29 @@ def test_local(env_tgt) -> None: assert result == adhoc_binaries.PythonBuildStandaloneBinary(sys.executable) -def test_docker_uses_helper() -> None: - result = run_rule_with_mocks( - adhoc_binaries.get_python_for_scripts, - rule_args=[EnvironmentTarget("docker", FileTarget({"source": ""}, address=Address("")))], - mock_gets=[ - MockGet( - output_type=_PythonBuildStandaloneBinary, - input_types=(_DownloadPythonBuildStandaloneBinaryRequest,), - mock=lambda _: _PythonBuildStandaloneBinary(""), - ) +def test_docker_uses_helper(rule_runner: RuleRunner) -> None: + rule_runner = RuleRunner( + rules=[ + *adhoc_binaries.rules(), + QueryRule( + _PythonBuildStandaloneBinary, + [_DownloadPythonBuildStandaloneBinaryRequest], + ), ], + target_types=[DockerEnvironmentTarget], + inherent_environment=EnvironmentName("docker"), ) - assert result == PythonBuildStandaloneBinary("") - - -def test_docker_helper(rule_runner: RuleRunner): rule_runner.write_files( { - "BUILD": "local_environment(name='local')", + "BUILD": "docker_environment(name='docker', image='ubuntu:latest')", } ) rule_runner.set_options( - ["--environments-preview-names={'local': '//:local'}"], env_inherit={"PATH"} + ["--environments-preview-names={'docker': '//:docker'}"], env_inherit={"PATH"} ) pbs = rule_runner.request( _PythonBuildStandaloneBinary, [_DownloadPythonBuildStandaloneBinaryRequest()], ) - assert not pbs.path.startswith("/") + assert pbs.path.startswith("/pants-named-caches") + assert pbs.path.endswith("/bin/python3") From a0ae9500df6b9a05ed1364d76a89f3c4afa4ba09 Mon Sep 17 00:00:00 2001 From: "A. Alonso Dominguez" <2269440+alonsodomin@users.noreply.github.com> Date: Fri, 29 Dec 2023 10:23:58 +0100 Subject: [PATCH 11/18] Support dependency inference with JVM codegen backends (#20285) Implements some basic support for dependency inference in the JVM-related codegen backends. --- .../backend/codegen/protobuf/java/rules.py | 3 +- .../protobuf/java/rules_integration_test.py | 29 ++++- .../codegen/protobuf/java/symbol_mapper.py | 30 +++++ .../codegen/protobuf/jvm_symbol_mapper.py | 106 ++++++++++++++++++ .../backend/codegen/protobuf/scala/rules.py | 3 +- .../protobuf/scala/rules_integration_test.py | 30 ++++- .../codegen/protobuf/scala/symbol_mapper.py | 30 +++++ .../codegen/soap/java/dependency_inference.py | 88 +++++++++++++++ .../pants/backend/codegen/soap/java/rules.py | 4 +- .../soap/java/rules_integration_test.py | 4 +- .../codegen/soap/java/symbol_mapper.py | 54 +++++++++ .../codegen/thrift/apache/java/rules.py | 3 +- .../apache/java/rules_integration_test.py | 36 ++++-- .../thrift/apache/java/symbol_mapper.py | 28 +++++ .../codegen/thrift/jvm_symbol_mapper.py | 66 +++++++++++ .../codegen/thrift/scrooge/java/rules.py | 2 + .../scrooge/java/rules_integration_test.py | 31 +++-- .../thrift/scrooge/java/symbol_mapper.py | 28 +++++ .../codegen/thrift/scrooge/scala/rules.py | 2 + .../scrooge/scala/rules_integration_test.py | 39 +++++-- .../thrift/scrooge/scala/scrooge.test.lock | 5 +- .../thrift/scrooge/scala/symbol_mapper.py | 30 +++++ .../backend/codegen/thrift/thrift_parser.py | 14 ++- .../codegen/thrift/thrift_parser_test.py | 23 +++- .../openapi/codegen/java/register.py | 9 +- .../backend/experimental/openapi/register.py | 6 +- .../backend/openapi/codegen/java/rules.py | 3 +- .../codegen/java/rules_integration_test.py | 74 +++++++++++- .../openapi/codegen/java/symbol_mapper.py | 61 ++++++++++ .../pants/backend/openapi/target_types.py | 28 +++++ 30 files changed, 805 insertions(+), 64 deletions(-) create mode 100644 src/python/pants/backend/codegen/protobuf/java/symbol_mapper.py create mode 100644 src/python/pants/backend/codegen/protobuf/jvm_symbol_mapper.py create mode 100644 src/python/pants/backend/codegen/protobuf/scala/symbol_mapper.py create mode 100644 src/python/pants/backend/codegen/soap/java/dependency_inference.py create mode 100644 src/python/pants/backend/codegen/soap/java/symbol_mapper.py create mode 100644 src/python/pants/backend/codegen/thrift/apache/java/symbol_mapper.py create mode 100644 src/python/pants/backend/codegen/thrift/jvm_symbol_mapper.py create mode 100644 src/python/pants/backend/codegen/thrift/scrooge/java/symbol_mapper.py create mode 100644 src/python/pants/backend/codegen/thrift/scrooge/scala/symbol_mapper.py create mode 100644 src/python/pants/backend/openapi/codegen/java/symbol_mapper.py diff --git a/src/python/pants/backend/codegen/protobuf/java/rules.py b/src/python/pants/backend/codegen/protobuf/java/rules.py index 747e8c20a9d..76d97fd60c0 100644 --- a/src/python/pants/backend/codegen/protobuf/java/rules.py +++ b/src/python/pants/backend/codegen/protobuf/java/rules.py @@ -5,7 +5,7 @@ from dataclasses import dataclass from pathlib import PurePath -from pants.backend.codegen.protobuf.java import dependency_inference +from pants.backend.codegen.protobuf.java import dependency_inference, symbol_mapper from pants.backend.codegen.protobuf.java.subsystem import JavaProtobufGrpcSubsystem from pants.backend.codegen.protobuf.protoc import Protoc from pants.backend.codegen.protobuf.target_types import ( @@ -212,6 +212,7 @@ def rules(): return [ *collect_rules(), *dependency_inference.rules(), + *symbol_mapper.rules(), UnionRule(GenerateSourcesRequest, GenerateJavaFromProtobufRequest), UnionRule(GenerateToolLockfileSentinel, GrpcJavaToolLockfileSentinel), ProtobufSourceTarget.register_plugin_field(PrefixedJvmJdkField), diff --git a/src/python/pants/backend/codegen/protobuf/java/rules_integration_test.py b/src/python/pants/backend/codegen/protobuf/java/rules_integration_test.py index ad04c74e76a..afa90903ea7 100644 --- a/src/python/pants/backend/codegen/protobuf/java/rules_integration_test.py +++ b/src/python/pants/backend/codegen/protobuf/java/rules_integration_test.py @@ -21,14 +21,21 @@ from pants.backend.experimental.java.register import rules as java_backend_rules from pants.backend.java.compile.javac import CompileJavaSourceRequest from pants.backend.java.target_types import JavaSourcesGeneratorTarget, JavaSourceTarget -from pants.engine.addresses import Address -from pants.engine.target import GeneratedSources, HydratedSources, HydrateSourcesRequest +from pants.engine.addresses import Address, Addresses +from pants.engine.target import ( + Dependencies, + DependenciesRequest, + GeneratedSources, + HydratedSources, + HydrateSourcesRequest, +) from pants.jvm import testutil from pants.jvm.target_types import JvmArtifactTarget from pants.jvm.testutil import ( RenderedClasspath, expect_single_expanded_coarsened_target, make_resolve, + maybe_skip_jdk_test, ) from pants.testutil.rule_runner import QueryRule, RuleRunner @@ -81,6 +88,7 @@ def rule_runner() -> RuleRunner: QueryRule(HydratedSources, [HydrateSourcesRequest]), QueryRule(GeneratedSources, [GenerateJavaFromProtobufRequest]), QueryRule(RenderedClasspath, (CompileJavaSourceRequest,)), + QueryRule(Addresses, (DependenciesRequest,)), ], target_types=[ ProtobufSourcesGeneratorTarget, @@ -112,6 +120,7 @@ def assert_files_generated( assert set(generated_sources.snapshot.files) == set(expected_files) +@maybe_skip_jdk_test def test_generates_java( rule_runner: RuleRunner, protobuf_java_lockfile: JVMLockfileFixture ) -> None: @@ -120,6 +129,7 @@ def test_generates_java( # * Protobuf files can import other protobuf files, and those can import others # (transitive dependencies). We'll only generate the requested target, though. # * We can handle multiple source roots, which need to be preserved in the final output. + # * Dependency inference between Java and Protobuf sources. rule_runner.write_files( { "src/protobuf/dir1/f.proto": dedent( @@ -144,7 +154,7 @@ def test_generates_java( """ ), "src/protobuf/dir1/BUILD": "protobuf_sources()", - "src/protobuf/dir2/f.proto": dedent( + "src/protobuf/dir2/f3.proto": dedent( """\ syntax = "proto3"; @@ -161,7 +171,7 @@ def test_generates_java( package test_protos; - import "dir2/f.proto"; + import "dir2/f3.proto"; """ ), "tests/protobuf/test_protos/BUILD": ( @@ -169,11 +179,13 @@ def test_generates_java( ), "3rdparty/jvm/default.lock": protobuf_java_lockfile.serialized_lockfile, "3rdparty/jvm/BUILD": protobuf_java_lockfile.requirements_as_jvm_artifact_targets(), - "src/jvm/BUILD": "java_sources(dependencies=['src/protobuf/dir1'])", + "src/jvm/BUILD": "java_sources()", "src/jvm/TestJavaProtobuf.java": dedent( """\ package org.pantsbuild.java.example; + import org.pantsbuild.java.proto.F.Person; + public class TestJavaProtobuf { Person person; } @@ -198,13 +210,17 @@ def assert_gen(addr: Address, expected: str) -> None: Address("src/protobuf/dir1", relative_file_path="f2.proto"), "src/protobuf/dir1/F2.java" ) assert_gen( - Address("src/protobuf/dir2", relative_file_path="f.proto"), "src/protobuf/dir2/F.java" + Address("src/protobuf/dir2", relative_file_path="f3.proto"), "src/protobuf/dir2/F3.java" ) assert_gen( Address("tests/protobuf/test_protos", relative_file_path="f.proto"), "tests/protobuf/test_protos/F.java", ) + tgt = rule_runner.get_target(Address("src/jvm", relative_file_path="TestJavaProtobuf.java")) + dependencies = rule_runner.request(Addresses, [DependenciesRequest(tgt[Dependencies])]) + assert Address("src/protobuf/dir1", relative_file_path="f.proto") in dependencies + request = CompileJavaSourceRequest( component=expect_single_expanded_coarsened_target( rule_runner, Address(spec_path="src/jvm") @@ -235,6 +251,7 @@ def protobuf_java_grpc_lockfile( return protobuf_java_grpc_lockfile_def.load(request) +@maybe_skip_jdk_test def test_generates_grpc_java( rule_runner: RuleRunner, protobuf_java_grpc_lockfile: JVMLockfileFixture ) -> None: diff --git a/src/python/pants/backend/codegen/protobuf/java/symbol_mapper.py b/src/python/pants/backend/codegen/protobuf/java/symbol_mapper.py new file mode 100644 index 00000000000..fe49c87948b --- /dev/null +++ b/src/python/pants/backend/codegen/protobuf/java/symbol_mapper.py @@ -0,0 +1,30 @@ +# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). +from __future__ import annotations + +from pants.backend.codegen.protobuf import jvm_symbol_mapper +from pants.backend.codegen.protobuf.jvm_symbol_mapper import FirstPartyProtobufJvmMappingRequest +from pants.engine.rules import collect_rules, rule +from pants.engine.unions import UnionRule +from pants.jvm.dependency_inference import symbol_mapper +from pants.jvm.dependency_inference.symbol_mapper import FirstPartyMappingRequest + + +class FirstPartyProtobufJavaTargetsMappingRequest(FirstPartyMappingRequest): + pass + + +@rule +async def map_first_party_protobuf_java_targets_to_symbols( + _: FirstPartyProtobufJavaTargetsMappingRequest, +) -> FirstPartyProtobufJvmMappingRequest: + return FirstPartyProtobufJvmMappingRequest(capitalize_base_name=True) + + +def rules(): + return [ + *collect_rules(), + *symbol_mapper.rules(), + *jvm_symbol_mapper.rules(), + UnionRule(FirstPartyMappingRequest, FirstPartyProtobufJavaTargetsMappingRequest), + ] diff --git a/src/python/pants/backend/codegen/protobuf/jvm_symbol_mapper.py b/src/python/pants/backend/codegen/protobuf/jvm_symbol_mapper.py new file mode 100644 index 00000000000..ca1b0454d2e --- /dev/null +++ b/src/python/pants/backend/codegen/protobuf/jvm_symbol_mapper.py @@ -0,0 +1,106 @@ +# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). +from __future__ import annotations + +import os +import re +from collections import defaultdict +from dataclasses import dataclass +from typing import DefaultDict, Mapping + +from pants.backend.codegen.protobuf.target_types import AllProtobufTargets, ProtobufSourceField +from pants.engine.addresses import Address +from pants.engine.fs import Digest, DigestContents, FileContent +from pants.engine.rules import Get, MultiGet, collect_rules, rule +from pants.engine.target import HydratedSources, HydrateSourcesRequest +from pants.jvm.dependency_inference.artifact_mapper import MutableTrieNode +from pants.jvm.dependency_inference.symbol_mapper import SymbolMap +from pants.jvm.subsystems import JvmSubsystem +from pants.jvm.target_types import JvmResolveField +from pants.util.ordered_set import OrderedSet + +_ResolveName = str + + +@dataclass(frozen=True) +class FirstPartyProtobufJvmMappingRequest: + capitalize_base_name: bool + + +@rule +async def map_first_party_protobuf_jvm_targets_to_symbols( + request: FirstPartyProtobufJvmMappingRequest, + all_protobuf_targets: AllProtobufTargets, + jvm: JvmSubsystem, +) -> SymbolMap: + sources = await MultiGet( + Get( + HydratedSources, + HydrateSourcesRequest( + tgt[ProtobufSourceField], + for_sources_types=(ProtobufSourceField,), + enable_codegen=True, + ), + ) + for tgt in all_protobuf_targets + ) + + all_contents = await MultiGet( + Get(DigestContents, Digest, source.snapshot.digest) for source in sources + ) + + namespace_mapping: DefaultDict[tuple[_ResolveName, str], OrderedSet[Address]] = defaultdict( + OrderedSet + ) + for tgt, contents in zip(all_protobuf_targets, all_contents): + if not contents: + continue + if len(contents) > 1: + raise AssertionError( + f"Protobuf target `{tgt.address}` mapped to more than one source file." + ) + + resolve = tgt[JvmResolveField].normalized_value(jvm) + namespace = _determine_namespace( + contents[0], capitalize_base_name=request.capitalize_base_name + ) + namespace_mapping[(resolve, namespace)].add(tgt.address) + + mapping: Mapping[str, MutableTrieNode] = defaultdict(MutableTrieNode) + for (resolve, namespace), addresses in namespace_mapping.items(): + mapping[resolve].insert(namespace, addresses, first_party=True, recursive=True) + + return SymbolMap((resolve, node.frozen()) for resolve, node in mapping.items()) + + +# Determine generated Java/Scala package name +# * https://grpc.io/docs/languages/java/generated-code +# * https://scalapb.github.io/docs/generated-code +def _determine_namespace(file: FileContent, *, capitalize_base_name: bool) -> str: + base_name, _, _ = os.path.basename(file.path).partition(".") + base_name = base_name.capitalize() if capitalize_base_name else base_name + package_definition = _parse_package_definition(file.content) + return f"{package_definition}.{base_name}" if package_definition else base_name + + +_QUOTE_CHAR = r"(?:'|\")" +_JAVA_PACKAGE_OPTION_RE = re.compile( + rf"^\s*option\s+java_package\s+=\s+{_QUOTE_CHAR}(.+){_QUOTE_CHAR};" +) +_PACKAGE_RE = re.compile(r"^\s*package\s+(.+);") + + +def _parse_package_definition(content_raw: bytes) -> str | None: + content = content_raw.decode() + for line in content.splitlines(): + m = _JAVA_PACKAGE_OPTION_RE.match(line) + if m: + return m.group(1) + m = _PACKAGE_RE.match(line) + if m: + return m.group(1) + return None + + +def rules(): + return collect_rules() diff --git a/src/python/pants/backend/codegen/protobuf/scala/rules.py b/src/python/pants/backend/codegen/protobuf/scala/rules.py index 30a9552fb29..5ae4b529e41 100644 --- a/src/python/pants/backend/codegen/protobuf/scala/rules.py +++ b/src/python/pants/backend/codegen/protobuf/scala/rules.py @@ -6,7 +6,7 @@ from dataclasses import dataclass from pants.backend.codegen.protobuf.protoc import Protoc -from pants.backend.codegen.protobuf.scala import dependency_inference +from pants.backend.codegen.protobuf.scala import dependency_inference, symbol_mapper from pants.backend.codegen.protobuf.scala.subsystem import PluginArtifactSpec, ScalaPBSubsystem from pants.backend.codegen.protobuf.target_types import ( ProtobufSourceField, @@ -322,6 +322,7 @@ def rules(): *collect_rules(), *lockfile.rules(), *dependency_inference.rules(), + *symbol_mapper.rules(), UnionRule(GenerateSourcesRequest, GenerateScalaFromProtobufRequest), UnionRule(GenerateToolLockfileSentinel, ScalapbcToolLockfileSentinel), ProtobufSourceTarget.register_plugin_field(PrefixedJvmJdkField), diff --git a/src/python/pants/backend/codegen/protobuf/scala/rules_integration_test.py b/src/python/pants/backend/codegen/protobuf/scala/rules_integration_test.py index c131f6c333f..f97735f8747 100644 --- a/src/python/pants/backend/codegen/protobuf/scala/rules_integration_test.py +++ b/src/python/pants/backend/codegen/protobuf/scala/rules_integration_test.py @@ -26,9 +26,16 @@ from pants.build_graph.address import Address from pants.core.util_rules import config_files, distdir, source_files, stripped_source_files from pants.core.util_rules.external_tool import rules as external_tool_rules +from pants.engine.addresses import Addresses from pants.engine.fs import Digest, DigestContents from pants.engine.rules import QueryRule -from pants.engine.target import GeneratedSources, HydratedSources, HydrateSourcesRequest +from pants.engine.target import ( + Dependencies, + DependenciesRequest, + GeneratedSources, + HydratedSources, + HydrateSourcesRequest, +) from pants.jvm import classpath, testutil from pants.jvm.dependency_inference import artifact_mapper from pants.jvm.jdk_rules import rules as jdk_rules @@ -40,6 +47,7 @@ RenderedClasspath, expect_single_expanded_coarsened_target, make_resolve, + maybe_skip_jdk_test, ) from pants.jvm.util_rules import rules as util_rules from pants.testutil.rule_runner import PYTHON_BOOTSTRAP_ENV, RuleRunner @@ -111,6 +119,7 @@ def rule_runner() -> RuleRunner: QueryRule(GeneratedSources, [GenerateScalaFromProtobufRequest]), QueryRule(DigestContents, (Digest,)), QueryRule(RenderedClasspath, (CompileScalaSourceRequest,)), + QueryRule(Addresses, (DependenciesRequest,)), ], target_types=[ ScalaSourceTarget, @@ -147,12 +156,14 @@ def assert_files_generated( assert set(generated_sources.snapshot.files) == set(expected_files) +@maybe_skip_jdk_test def test_generates_scala(rule_runner: RuleRunner, scalapb_lockfile: JVMLockfileFixture) -> None: # This tests a few things: # * We generate the correct file names. # * Protobuf files can import other protobuf files, and those can import others # (transitive dependencies). We'll only generate the requested target, though. # * We can handle multiple source roots, which need to be preserved in the final output. + # * Dependency inference between Scala and Protobuf sources. rule_runner.write_files( { "src/protobuf/dir1/f.proto": dedent( @@ -178,7 +189,7 @@ def test_generates_scala(rule_runner: RuleRunner, scalapb_lockfile: JVMLockfileF """ ), "src/protobuf/dir1/BUILD": "protobuf_sources()", - "src/protobuf/dir2/f.proto": dedent( + "src/protobuf/dir2/f3.proto": dedent( """\ syntax = "proto3"; option java_package = "org.pantsbuild.scala.proto"; @@ -196,7 +207,7 @@ def test_generates_scala(rule_runner: RuleRunner, scalapb_lockfile: JVMLockfileF package test_protos; - import "dir2/f.proto"; + import "dir2/f3.proto"; """ ), "tests/protobuf/test_protos/BUILD": ( @@ -204,11 +215,13 @@ def test_generates_scala(rule_runner: RuleRunner, scalapb_lockfile: JVMLockfileF ), "3rdparty/jvm/default.lock": scalapb_lockfile.serialized_lockfile, "3rdparty/jvm/BUILD": scalapb_lockfile.requirements_as_jvm_artifact_targets(), - "src/jvm/BUILD": "scala_sources(dependencies=['src/protobuf/dir1'])", + "src/jvm/BUILD": "scala_sources()", "src/jvm/ScalaPBExample.scala": dedent( """\ package org.pantsbuild.scala.example + import org.pantsbuild.scala.proto.f.Person + trait TestScrooge { val person: Person } @@ -237,14 +250,18 @@ def assert_gen(addr: Address, expected: Iterable[str]) -> None: ["src/protobuf/org/pantsbuild/scala/proto/f2/F2Proto.scala"], ) assert_gen( - Address("src/protobuf/dir2", relative_file_path="f.proto"), - ["src/protobuf/org/pantsbuild/scala/proto/f/FProto.scala"], + Address("src/protobuf/dir2", relative_file_path="f3.proto"), + ["src/protobuf/org/pantsbuild/scala/proto/f3/F3Proto.scala"], ) assert_gen( Address("tests/protobuf/test_protos", relative_file_path="f.proto"), ["tests/protobuf/test_protos/f/FProto.scala"], ) + tgt = rule_runner.get_target(Address("src/jvm", relative_file_path="ScalaPBExample.scala")) + dependencies = rule_runner.request(Addresses, [DependenciesRequest(tgt[Dependencies])]) + assert Address("src/protobuf/dir1", relative_file_path="f.proto") in dependencies + coarsened_target = expect_single_expanded_coarsened_target( rule_runner, Address(spec_path="src/jvm") ) @@ -254,6 +271,7 @@ def assert_gen(addr: Address, expected: Iterable[str]) -> None: ) +@maybe_skip_jdk_test def test_top_level_proto_root( rule_runner: RuleRunner, scalapb_lockfile: JVMLockfileFixture ) -> None: diff --git a/src/python/pants/backend/codegen/protobuf/scala/symbol_mapper.py b/src/python/pants/backend/codegen/protobuf/scala/symbol_mapper.py new file mode 100644 index 00000000000..9ed885456ff --- /dev/null +++ b/src/python/pants/backend/codegen/protobuf/scala/symbol_mapper.py @@ -0,0 +1,30 @@ +# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). +from __future__ import annotations + +from pants.backend.codegen.protobuf import jvm_symbol_mapper +from pants.backend.codegen.protobuf.jvm_symbol_mapper import FirstPartyProtobufJvmMappingRequest +from pants.engine.rules import collect_rules, rule +from pants.engine.unions import UnionRule +from pants.jvm.dependency_inference import symbol_mapper +from pants.jvm.dependency_inference.symbol_mapper import FirstPartyMappingRequest + + +class FirstPartyProtobufScalaTargetsMappingRequest(FirstPartyMappingRequest): + pass + + +@rule +async def map_first_party_protobuf_scala_targets_to_symbols( + _: FirstPartyProtobufScalaTargetsMappingRequest, +) -> FirstPartyProtobufJvmMappingRequest: + return FirstPartyProtobufJvmMappingRequest(capitalize_base_name=False) + + +def rules(): + return [ + *collect_rules(), + *symbol_mapper.rules(), + *jvm_symbol_mapper.rules(), + UnionRule(FirstPartyMappingRequest, FirstPartyProtobufScalaTargetsMappingRequest), + ] diff --git a/src/python/pants/backend/codegen/soap/java/dependency_inference.py b/src/python/pants/backend/codegen/soap/java/dependency_inference.py new file mode 100644 index 00000000000..c2bfca0437d --- /dev/null +++ b/src/python/pants/backend/codegen/soap/java/dependency_inference.py @@ -0,0 +1,88 @@ +# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). +from __future__ import annotations + +from dataclasses import dataclass +from typing import FrozenSet + +from pants.backend.codegen.soap.java.jaxws import JaxWsTools +from pants.backend.codegen.soap.target_types import WsdlDependenciesField +from pants.engine.addresses import Address +from pants.engine.rules import Get, collect_rules, rule +from pants.engine.target import FieldSet, InferDependenciesRequest, InferredDependencies +from pants.engine.unions import UnionRule +from pants.jvm.dependency_inference.artifact_mapper import ( + AllJvmArtifactTargets, + find_jvm_artifacts_or_raise, +) +from pants.jvm.resolve.common import Coordinate +from pants.jvm.subsystems import JvmSubsystem +from pants.jvm.target_types import JvmResolveField + +_JAXWS_RUNTIME_GROUP = "com.sun.xml.ws" +_JAXWS_RUNTIME_ARTIFACT = "jaxws-rt" + + +@dataclass(frozen=True) +class JaxWSRuntimeDependencyInferenceFieldSet(FieldSet): + required_fields = (WsdlDependenciesField, JvmResolveField) + + resolve: JvmResolveField + + +class InferJaxWSRuntimeDependenciesRequest(InferDependenciesRequest): + infer_from = JaxWSRuntimeDependencyInferenceFieldSet + + +@dataclass(frozen=True) +class JaxWSJavaRuntimeForResolveRequest: + resolve_name: str + + +@dataclass(frozen=True) +class JaxWSJavaRuntimeForResolve: + addresses: FrozenSet[Address] + + +@rule +async def resolve_jaxws_runtime_for_resolve( + request: JaxWSJavaRuntimeForResolveRequest, + jvm_artifact_targets: AllJvmArtifactTargets, + jvm: JvmSubsystem, + jaxws: JaxWsTools, +) -> JaxWSJavaRuntimeForResolve: + addresses = find_jvm_artifacts_or_raise( + required_coordinates=[ + Coordinate( + group=_JAXWS_RUNTIME_GROUP, + artifact=_JAXWS_RUNTIME_ARTIFACT, + version=jaxws.version, + ) + ], + resolve=request.resolve_name, + jvm_artifact_targets=jvm_artifact_targets, + jvm=jvm, + subsystem="the JaxWS runtime", + target_type="wsdl_sources", + requirement_source=f"the `[{jaxws.options_scope}].version` option", + ) + return JaxWSJavaRuntimeForResolve(addresses) + + +@rule +async def infer_jaxws_runtime_dependency( + request: InferJaxWSRuntimeDependenciesRequest, jvm: JvmSubsystem +) -> InferredDependencies: + resolve = request.field_set.resolve.normalized_value(jvm) + + jaxws_resolved_runtime = await Get( + JaxWSJavaRuntimeForResolve, JaxWSJavaRuntimeForResolveRequest(resolve) + ) + return InferredDependencies(jaxws_resolved_runtime.addresses) + + +def rules(): + return [ + *collect_rules(), + UnionRule(InferDependenciesRequest, InferJaxWSRuntimeDependenciesRequest), + ] diff --git a/src/python/pants/backend/codegen/soap/java/rules.py b/src/python/pants/backend/codegen/soap/java/rules.py index b4569167cac..d10a94f720c 100644 --- a/src/python/pants/backend/codegen/soap/java/rules.py +++ b/src/python/pants/backend/codegen/soap/java/rules.py @@ -5,7 +5,7 @@ from dataclasses import dataclass -from pants.backend.codegen.soap.java import extra_fields +from pants.backend.codegen.soap.java import dependency_inference, extra_fields, symbol_mapper from pants.backend.codegen.soap.java.extra_fields import JavaModuleField, JavaPackageField from pants.backend.codegen.soap.java.jaxws import JaxWsTools from pants.backend.codegen.soap.target_types import ( @@ -185,6 +185,8 @@ def rules(): return [ *collect_rules(), *extra_fields.rules(), + *dependency_inference.rules(), + *symbol_mapper.rules(), *jvm_tool.rules(), *jdk_rules.rules(), UnionRule(GenerateSourcesRequest, GenerateJavaFromWsdlRequest), diff --git a/src/python/pants/backend/codegen/soap/java/rules_integration_test.py b/src/python/pants/backend/codegen/soap/java/rules_integration_test.py index 7bfa5d0eb6e..16036919876 100644 --- a/src/python/pants/backend/codegen/soap/java/rules_integration_test.py +++ b/src/python/pants/backend/codegen/soap/java/rules_integration_test.py @@ -167,11 +167,11 @@ def test_generate_java_from_wsdl( ) -> None: rule_runner.write_files( { - "src/wsdl/BUILD": "wsdl_sources(dependencies=['3rdparty/jvm:com.sun.xml.ws_jaxws-rt'])", + "src/wsdl/BUILD": "wsdl_sources(java_package='com.example.wsdl.fooservice')", "src/wsdl/FooService.wsdl": _FOO_SERVICE_WSDL, "3rdparty/jvm/default.lock": wsdl_lockfile.serialized_lockfile, "3rdparty/jvm/BUILD": wsdl_lockfile.requirements_as_jvm_artifact_targets(), - "src/jvm/BUILD": "java_sources(dependencies=['src/wsdl', '3rdparty/jvm:com.sun.xml.ws_jaxws-rt'])", + "src/jvm/BUILD": "java_sources()", "src/jvm/FooServiceMain.java": textwrap.dedent( """\ package org.pantsbuild.example; diff --git a/src/python/pants/backend/codegen/soap/java/symbol_mapper.py b/src/python/pants/backend/codegen/soap/java/symbol_mapper.py new file mode 100644 index 00000000000..bdd7afe4612 --- /dev/null +++ b/src/python/pants/backend/codegen/soap/java/symbol_mapper.py @@ -0,0 +1,54 @@ +# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). +from __future__ import annotations + +from collections import defaultdict +from typing import DefaultDict, Mapping, Tuple + +from pants.backend.codegen.soap.java.extra_fields import JavaPackageField +from pants.backend.codegen.soap.target_types import AllWsdlTargets +from pants.engine.addresses import Address +from pants.engine.rules import collect_rules, rule +from pants.engine.unions import UnionRule +from pants.jvm.dependency_inference.artifact_mapper import MutableTrieNode +from pants.jvm.dependency_inference.symbol_mapper import FirstPartyMappingRequest, SymbolMap +from pants.jvm.subsystems import JvmSubsystem +from pants.jvm.target_types import JvmResolveField +from pants.util.ordered_set import OrderedSet + +_ResolveName = str + + +class FirstPartyWsdlJaxWsTargetsMappingRequest(FirstPartyMappingRequest): + pass + + +@rule +async def map_first_party_wsdl_jaxws_targets_to_symbols( + _: FirstPartyWsdlJaxWsTargetsMappingRequest, wsdl_targets: AllWsdlTargets, jvm: JvmSubsystem +) -> SymbolMap: + package_mapping: DefaultDict[Tuple[_ResolveName, str], OrderedSet[Address]] = defaultdict( + OrderedSet + ) + for target in wsdl_targets: + resolve_name = target[JvmResolveField].normalized_value(jvm) + package_name = target[JavaPackageField].value + + # TODO If no explicit package name is given, parse the WSDL and derive it from its namespace + if not package_name: + continue + + package_mapping[(resolve_name, package_name)].add(target.address) + + symbol_map: Mapping[_ResolveName, MutableTrieNode] = defaultdict(MutableTrieNode) + for (resolve, package), addresses in package_mapping.items(): + symbol_map[resolve].insert(package, addresses, first_party=True, recursive=True) + + return SymbolMap((resolve, node.frozen()) for resolve, node in symbol_map.items()) + + +def rules(): + return [ + *collect_rules(), + UnionRule(FirstPartyMappingRequest, FirstPartyWsdlJaxWsTargetsMappingRequest), + ] diff --git a/src/python/pants/backend/codegen/thrift/apache/java/rules.py b/src/python/pants/backend/codegen/thrift/apache/java/rules.py index 10f8685fab8..ed1e51c5320 100644 --- a/src/python/pants/backend/codegen/thrift/apache/java/rules.py +++ b/src/python/pants/backend/codegen/thrift/apache/java/rules.py @@ -4,7 +4,7 @@ from dataclasses import dataclass -from pants.backend.codegen.thrift.apache.java import subsystem +from pants.backend.codegen.thrift.apache.java import subsystem, symbol_mapper from pants.backend.codegen.thrift.apache.java.subsystem import ApacheThriftJavaSubsystem from pants.backend.codegen.thrift.apache.rules import ( GeneratedThriftSources, @@ -137,6 +137,7 @@ def rules(): return ( *collect_rules(), *subsystem.rules(), + *symbol_mapper.rules(), UnionRule(GenerateSourcesRequest, GenerateJavaFromThriftRequest), UnionRule(InferDependenciesRequest, InferApacheThriftJavaDependencies), ThriftSourceTarget.register_plugin_field(PrefixedJvmJdkField), diff --git a/src/python/pants/backend/codegen/thrift/apache/java/rules_integration_test.py b/src/python/pants/backend/codegen/thrift/apache/java/rules_integration_test.py index b0e79f6976c..1a6c4c4655d 100644 --- a/src/python/pants/backend/codegen/thrift/apache/java/rules_integration_test.py +++ b/src/python/pants/backend/codegen/thrift/apache/java/rules_integration_test.py @@ -26,8 +26,15 @@ from pants.build_graph.address import Address from pants.core.util_rules import config_files, source_files, stripped_source_files from pants.core.util_rules.external_tool import rules as external_tool_rules +from pants.engine.addresses import Addresses from pants.engine.rules import QueryRule -from pants.engine.target import GeneratedSources, HydratedSources, HydrateSourcesRequest +from pants.engine.target import ( + Dependencies, + DependenciesRequest, + GeneratedSources, + HydratedSources, + HydrateSourcesRequest, +) from pants.jvm import classpath, jdk_rules, testutil, util_rules from pants.jvm.dependency_inference import artifact_mapper from pants.jvm.resolve import coursier_fetch, coursier_setup @@ -82,6 +89,7 @@ def rule_runner() -> RuleRunner: QueryRule(HydratedSources, [HydrateSourcesRequest]), QueryRule(GeneratedSources, [GenerateJavaFromThriftRequest]), QueryRule(RenderedClasspath, (CompileJavaSourceRequest,)), + QueryRule(Addresses, (DependenciesRequest,)), ], target_types=[ ThriftSourcesGeneratorTarget, @@ -120,6 +128,7 @@ def test_generates_java(rule_runner: RuleRunner, libthrift_lockfile: JVMLockfile # * Thrift files can import other thrift files, and those can import others # (transitive dependencies). We'll only generate the requested target, though. # * We can handle multiple source roots, which need to be preserved in the final output. + # * Dependency inference between Java and Thrift sources. rule_runner.write_files( { "src/thrift/dir1/f.thrift": dedent( @@ -134,7 +143,7 @@ def test_generates_java(rule_runner: RuleRunner, libthrift_lockfile: JVMLockfile ), "src/thrift/dir1/f2.thrift": dedent( """\ - namespace java org.pantsbuild.example + namespace java org.pantsbuild.example.mngt include "dir1/f.thrift" struct ManagedPerson { 1: f.Person employee @@ -145,7 +154,9 @@ def test_generates_java(rule_runner: RuleRunner, libthrift_lockfile: JVMLockfile "src/thrift/dir1/BUILD": "thrift_sources()", "src/thrift/dir2/g.thrift": dedent( """\ + namespace java org.pantsbuild.example.wrapper include "dir1/f2.thrift" + struct ManagedPersonWrapper { 1: f2.ManagedPerson managed_person } @@ -155,7 +166,9 @@ def test_generates_java(rule_runner: RuleRunner, libthrift_lockfile: JVMLockfile # Test another source root. "tests/thrift/test_thrifts/f.thrift": dedent( """\ + namespace java org.pantsbuild.example.test include "dir2/g.thrift" + struct Executive { 1: g.ManagedPersonWrapper managed_person_wrapper } @@ -164,11 +177,12 @@ def test_generates_java(rule_runner: RuleRunner, libthrift_lockfile: JVMLockfile "tests/thrift/test_thrifts/BUILD": "thrift_sources(dependencies=['src/thrift/dir2'])", "3rdparty/jvm/default.lock": libthrift_lockfile.serialized_lockfile, "3rdparty/jvm/BUILD": libthrift_lockfile.requirements_as_jvm_artifact_targets(), - "src/jvm/BUILD": "java_sources(dependencies=['src/thrift/dir1'])", - "src/jvm/TestScroogeThriftJava.java": dedent( + "src/jvm/BUILD": "java_sources()", + "src/jvm/TestApacheThriftJava.java": dedent( """\ package org.pantsbuild.example; - public class TestScroogeThriftJava { + + public class TestApacheThriftJava { Person person; } """ @@ -193,24 +207,26 @@ def assert_gen(addr: Address, expected: list[str]) -> None: assert_gen( Address("src/thrift/dir1", relative_file_path="f2.thrift"), [ - "src/thrift/org/pantsbuild/example/ManagedPerson.java", + "src/thrift/org/pantsbuild/example/mngt/ManagedPerson.java", ], ) - # TODO: Fix package namespacing? assert_gen( Address("src/thrift/dir2", relative_file_path="g.thrift"), [ - "src/thrift/ManagedPersonWrapper.java", + "src/thrift/org/pantsbuild/example/wrapper/ManagedPersonWrapper.java", ], ) - # TODO: Fix namespacing. assert_gen( Address("tests/thrift/test_thrifts", relative_file_path="f.thrift"), [ - "tests/thrift/Executive.java", + "tests/thrift/org/pantsbuild/example/test/Executive.java", ], ) + tgt = rule_runner.get_target(Address("src/jvm", relative_file_path="TestApacheThriftJava.java")) + dependencies = rule_runner.request(Addresses, [DependenciesRequest(tgt[Dependencies])]) + assert Address("src/thrift/dir1", relative_file_path="f.thrift") in dependencies + request = CompileJavaSourceRequest( component=expect_single_expanded_coarsened_target( rule_runner, Address(spec_path="src/jvm") diff --git a/src/python/pants/backend/codegen/thrift/apache/java/symbol_mapper.py b/src/python/pants/backend/codegen/thrift/apache/java/symbol_mapper.py new file mode 100644 index 00000000000..8c939782c30 --- /dev/null +++ b/src/python/pants/backend/codegen/thrift/apache/java/symbol_mapper.py @@ -0,0 +1,28 @@ +# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). +from __future__ import annotations + +from pants.backend.codegen.thrift import jvm_symbol_mapper +from pants.backend.codegen.thrift.jvm_symbol_mapper import FirstPartyThriftJvmMappingRequest +from pants.engine.rules import collect_rules, rule +from pants.engine.unions import UnionRule +from pants.jvm.dependency_inference.symbol_mapper import FirstPartyMappingRequest + + +class FirstPartyThriftJavaTargetsMappingRequest(FirstPartyMappingRequest): + pass + + +@rule +async def map_first_party_thrift_java_targets_to_symbols( + _: FirstPartyThriftJavaTargetsMappingRequest, +) -> FirstPartyThriftJvmMappingRequest: + return FirstPartyThriftJvmMappingRequest() + + +def rules(): + return [ + *collect_rules(), + *jvm_symbol_mapper.rules(), + UnionRule(FirstPartyMappingRequest, FirstPartyThriftJavaTargetsMappingRequest), + ] diff --git a/src/python/pants/backend/codegen/thrift/jvm_symbol_mapper.py b/src/python/pants/backend/codegen/thrift/jvm_symbol_mapper.py new file mode 100644 index 00000000000..7db24587fdf --- /dev/null +++ b/src/python/pants/backend/codegen/thrift/jvm_symbol_mapper.py @@ -0,0 +1,66 @@ +# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). +from __future__ import annotations + +from collections import defaultdict +from dataclasses import dataclass +from typing import DefaultDict, Mapping + +from pants.backend.codegen.thrift.target_types import AllThriftTargets, ThriftSourceField +from pants.backend.codegen.thrift.thrift_parser import ParsedThrift, ParsedThriftRequest +from pants.engine.addresses import Address +from pants.engine.rules import Get, MultiGet, collect_rules, rule +from pants.jvm.dependency_inference.artifact_mapper import MutableTrieNode +from pants.jvm.dependency_inference.symbol_mapper import SymbolMap +from pants.jvm.subsystems import JvmSubsystem +from pants.jvm.target_types import JvmResolveField +from pants.util.ordered_set import OrderedSet + +_ResolveName = str + + +@dataclass(frozen=True) +class FirstPartyThriftJvmMappingRequest: + extra_lang_ids: tuple[str, ...] = () + extra_namespace_directives: tuple[str, ...] = () + + +@rule +async def map_first_party_thrift_targets_to_jvm_symbols( + request: FirstPartyThriftJvmMappingRequest, thrift_targets: AllThriftTargets, jvm: JvmSubsystem +) -> SymbolMap: + jvm_thrift_targets = [tgt for tgt in thrift_targets if tgt.has_field(JvmResolveField)] + + parsed_thrift_sources = await MultiGet( + Get( + ParsedThrift, + ParsedThriftRequest( + sources_field=tgt[ThriftSourceField], + extra_namespace_directives=request.extra_namespace_directives, + ), + ) + for tgt in jvm_thrift_targets + ) + + package_symbols: DefaultDict[tuple[_ResolveName, str], OrderedSet[Address]] = defaultdict( + OrderedSet + ) + lang_ids = {"java", *request.extra_lang_ids} + for target, parsed_thrift in zip(jvm_thrift_targets, parsed_thrift_sources): + for lang_id in lang_ids: + package_name = parsed_thrift.namespaces.get(lang_id) + if not package_name: + continue + + resolve = target[JvmResolveField].normalized_value(jvm) + package_symbols[(resolve, package_name)].add(target.address) + + mapping: Mapping[str, MutableTrieNode] = defaultdict(MutableTrieNode) + for (resolve, package_name), addresses in package_symbols.items(): + mapping[resolve].insert(package_name, addresses, first_party=True, recursive=True) + + return SymbolMap((resolve, node.frozen()) for resolve, node in mapping.items()) + + +def rules(): + return collect_rules() diff --git a/src/python/pants/backend/codegen/thrift/scrooge/java/rules.py b/src/python/pants/backend/codegen/thrift/scrooge/java/rules.py index 220681be126..c1e5c277259 100644 --- a/src/python/pants/backend/codegen/thrift/scrooge/java/rules.py +++ b/src/python/pants/backend/codegen/thrift/scrooge/java/rules.py @@ -4,6 +4,7 @@ from dataclasses import dataclass +from pants.backend.codegen.thrift.scrooge.java import symbol_mapper from pants.backend.codegen.thrift.scrooge.rules import ( GeneratedScroogeThriftSources, GenerateScroogeThriftSourcesRequest, @@ -138,6 +139,7 @@ async def inject_scrooge_thrift_java_dependencies( def rules(): return ( *collect_rules(), + *symbol_mapper.rules(), UnionRule(GenerateSourcesRequest, GenerateJavaFromThriftRequest), UnionRule(InferDependenciesRequest, InferScroogeThriftJavaDependencies), ThriftSourceTarget.register_plugin_field(PrefixedJvmJdkField), diff --git a/src/python/pants/backend/codegen/thrift/scrooge/java/rules_integration_test.py b/src/python/pants/backend/codegen/thrift/scrooge/java/rules_integration_test.py index 9a9b767aeb2..c9f5f7e34c8 100644 --- a/src/python/pants/backend/codegen/thrift/scrooge/java/rules_integration_test.py +++ b/src/python/pants/backend/codegen/thrift/scrooge/java/rules_integration_test.py @@ -27,8 +27,15 @@ from pants.build_graph.address import Address from pants.core.util_rules import config_files, source_files, stripped_source_files from pants.core.util_rules.external_tool import rules as external_tool_rules +from pants.engine.addresses import Addresses from pants.engine.rules import QueryRule -from pants.engine.target import GeneratedSources, HydratedSources, HydrateSourcesRequest +from pants.engine.target import ( + Dependencies, + DependenciesRequest, + GeneratedSources, + HydratedSources, + HydrateSourcesRequest, +) from pants.jvm import classpath, testutil from pants.jvm.dependency_inference import artifact_mapper from pants.jvm.jdk_rules import rules as jdk_rules @@ -71,6 +78,7 @@ def rule_runner() -> RuleRunner: QueryRule(HydratedSources, [HydrateSourcesRequest]), QueryRule(GeneratedSources, [GenerateJavaFromThriftRequest]), QueryRule(RenderedClasspath, (CompileJavaSourceRequest,)), + QueryRule(Addresses, (DependenciesRequest,)), ], target_types=[ JvmArtifactTarget, @@ -129,6 +137,7 @@ def test_generates_java(rule_runner: RuleRunner, scrooge_lockfile: JVMLockfileFi # * Thrift files can import other thrift files, and those can import others # (transitive dependencies). We'll only generate the requested target, though. # * We can handle multiple source roots, which need to be preserved in the final output. + # * Dependency inference between Java and Thrift sources. rule_runner.write_files( { "src/thrift/dir1/f.thrift": dedent( @@ -143,7 +152,7 @@ def test_generates_java(rule_runner: RuleRunner, scrooge_lockfile: JVMLockfileFi ), "src/thrift/dir1/f2.thrift": dedent( """\ - namespace java org.pantsbuild.example + namespace java org.pantsbuild.example.mngt include "dir1/f.thrift" struct ManagedPerson { 1: f.Person employee @@ -154,7 +163,7 @@ def test_generates_java(rule_runner: RuleRunner, scrooge_lockfile: JVMLockfileFi "src/thrift/dir1/BUILD": "thrift_sources()", "src/thrift/dir2/g.thrift": dedent( """\ - namespace java org.pantsbuild.example + namespace java org.pantsbuild.example.wrapper include "dir1/f2.thrift" struct ManagedPersonWrapper { 1: f2.ManagedPerson managed_person @@ -165,7 +174,7 @@ def test_generates_java(rule_runner: RuleRunner, scrooge_lockfile: JVMLockfileFi # Test another source root. "tests/thrift/test_thrifts/f.thrift": dedent( """\ - namespace java org.pantsbuild.example + namespace java org.pantsbuild.example.test include "dir2/g.thrift" struct Executive { 1: g.ManagedPersonWrapper managed_person_wrapper @@ -175,7 +184,7 @@ def test_generates_java(rule_runner: RuleRunner, scrooge_lockfile: JVMLockfileFi "tests/thrift/test_thrifts/BUILD": "thrift_sources(dependencies=['src/thrift/dir2'])", "3rdparty/jvm/default.lock": scrooge_lockfile.serialized_lockfile, "3rdparty/jvm/BUILD": scrooge_lockfile.requirements_as_jvm_artifact_targets(), - "src/jvm/BUILD": "java_sources(dependencies=['src/thrift/dir1'])", + "src/jvm/BUILD": "java_sources()", "src/jvm/TestScroogeThriftJava.java": dedent( """\ package org.pantsbuild.example; @@ -204,22 +213,28 @@ def assert_gen(addr: Address, expected: list[str]) -> None: assert_gen( Address("src/thrift/dir1", relative_file_path="f2.thrift"), [ - "src/thrift/org/pantsbuild/example/ManagedPerson.java", + "src/thrift/org/pantsbuild/example/mngt/ManagedPerson.java", ], ) assert_gen( Address("src/thrift/dir2", relative_file_path="g.thrift"), [ - "src/thrift/org/pantsbuild/example/ManagedPersonWrapper.java", + "src/thrift/org/pantsbuild/example/wrapper/ManagedPersonWrapper.java", ], ) assert_gen( Address("tests/thrift/test_thrifts", relative_file_path="f.thrift"), [ - "tests/thrift/org/pantsbuild/example/Executive.java", + "tests/thrift/org/pantsbuild/example/test/Executive.java", ], ) + tgt = rule_runner.get_target( + Address("src/jvm", relative_file_path="TestScroogeThriftJava.java") + ) + dependencies = rule_runner.request(Addresses, [DependenciesRequest(tgt[Dependencies])]) + assert Address("src/thrift/dir1", relative_file_path="f.thrift") in dependencies + request = CompileJavaSourceRequest( component=expect_single_expanded_coarsened_target( rule_runner, Address(spec_path="src/jvm") diff --git a/src/python/pants/backend/codegen/thrift/scrooge/java/symbol_mapper.py b/src/python/pants/backend/codegen/thrift/scrooge/java/symbol_mapper.py new file mode 100644 index 00000000000..78acee8cfc2 --- /dev/null +++ b/src/python/pants/backend/codegen/thrift/scrooge/java/symbol_mapper.py @@ -0,0 +1,28 @@ +# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). +from __future__ import annotations + +from pants.backend.codegen.thrift import jvm_symbol_mapper +from pants.backend.codegen.thrift.jvm_symbol_mapper import FirstPartyThriftJvmMappingRequest +from pants.engine.rules import collect_rules, rule +from pants.engine.unions import UnionRule +from pants.jvm.dependency_inference.symbol_mapper import FirstPartyMappingRequest + + +class FirstPartyThriftScroogeJavaTargetsMappingRequest(FirstPartyMappingRequest): + pass + + +@rule +async def map_first_party_thrift_scrooge_java_targets_to_symbols( + _: FirstPartyThriftScroogeJavaTargetsMappingRequest, +) -> FirstPartyThriftJvmMappingRequest: + return FirstPartyThriftJvmMappingRequest() + + +def rules(): + return [ + *collect_rules(), + *jvm_symbol_mapper.rules(), + UnionRule(FirstPartyMappingRequest, FirstPartyThriftScroogeJavaTargetsMappingRequest), + ] diff --git a/src/python/pants/backend/codegen/thrift/scrooge/scala/rules.py b/src/python/pants/backend/codegen/thrift/scrooge/scala/rules.py index 1c4eb43cd9c..1438a63c87d 100644 --- a/src/python/pants/backend/codegen/thrift/scrooge/scala/rules.py +++ b/src/python/pants/backend/codegen/thrift/scrooge/scala/rules.py @@ -8,6 +8,7 @@ GeneratedScroogeThriftSources, GenerateScroogeThriftSourcesRequest, ) +from pants.backend.codegen.thrift.scrooge.scala import symbol_mapper from pants.backend.codegen.thrift.target_types import ( ThriftDependenciesField, ThriftSourceField, @@ -138,6 +139,7 @@ async def inject_scrooge_thrift_scala_dependencies( def rules(): return ( *collect_rules(), + *symbol_mapper.rules(), UnionRule(GenerateSourcesRequest, GenerateScalaFromThriftRequest), UnionRule(InferDependenciesRequest, InferScroogeThriftScalaDependencies), ThriftSourceTarget.register_plugin_field(PrefixedJvmJdkField), diff --git a/src/python/pants/backend/codegen/thrift/scrooge/scala/rules_integration_test.py b/src/python/pants/backend/codegen/thrift/scrooge/scala/rules_integration_test.py index 629201dc177..e7c7b4c53a7 100644 --- a/src/python/pants/backend/codegen/thrift/scrooge/scala/rules_integration_test.py +++ b/src/python/pants/backend/codegen/thrift/scrooge/scala/rules_integration_test.py @@ -21,12 +21,20 @@ from pants.backend.scala import target_types from pants.backend.scala.compile.scalac import CompileScalaSourceRequest from pants.backend.scala.compile.scalac import rules as scalac_rules +from pants.backend.scala.dependency_inference.rules import rules as scala_dep_inf_rules from pants.backend.scala.target_types import ScalaSourcesGeneratorTarget, ScalaSourceTarget from pants.build_graph.address import Address from pants.core.util_rules import config_files, source_files, stripped_source_files from pants.core.util_rules.external_tool import rules as external_tool_rules +from pants.engine.addresses import Addresses from pants.engine.rules import QueryRule -from pants.engine.target import GeneratedSources, HydratedSources, HydrateSourcesRequest +from pants.engine.target import ( + Dependencies, + DependenciesRequest, + GeneratedSources, + HydratedSources, + HydrateSourcesRequest, +) from pants.jvm import classpath, testutil from pants.jvm.jdk_rules import rules as jdk_rules from pants.jvm.resolve.coursier_fetch import rules as coursier_fetch_rules @@ -61,10 +69,12 @@ def rule_runner() -> RuleRunner: *jdk_rules(), *target_types.rules(), *stripped_source_files.rules(), + *scala_dep_inf_rules(), *testutil.rules(), QueryRule(HydratedSources, [HydrateSourcesRequest]), QueryRule(GeneratedSources, [GenerateScalaFromThriftRequest]), QueryRule(RenderedClasspath, (CompileScalaSourceRequest,)), + QueryRule(Addresses, (DependenciesRequest,)), ], target_types=[ ScalaSourceTarget, @@ -84,7 +94,11 @@ def rule_runner() -> RuleRunner: def scrooge_lockfile_def() -> JVMLockfileFixtureDefinition: return JVMLockfileFixtureDefinition( "scrooge.test.lock", - ["org.apache.thrift:libthrift:0.15.0", "com.twitter:scrooge-core_2.13:21.12.0"], + [ + "org.apache.thrift:libthrift:0.15.0", + "com.twitter:scrooge-core_2.13:21.12.0", + "org.scala-lang:scala-library:2.13.6", + ], ) @@ -123,6 +137,7 @@ def test_generates_scala(rule_runner: RuleRunner, scrooge_lockfile: JVMLockfileF # * Thrift files can import other thrift files, and those can import others # (transitive dependencies). We'll only generate the requested target, though. # * We can handle multiple source roots, which need to be preserved in the final output. + # * Dependency inference between Scala and Thrift sources. rule_runner.write_files( { "src/thrift/dir1/f.thrift": dedent( @@ -137,7 +152,7 @@ def test_generates_scala(rule_runner: RuleRunner, scrooge_lockfile: JVMLockfileF ), "src/thrift/dir1/f2.thrift": dedent( """\ - #@namespace scala org.pantsbuild.example + #@namespace scala org.pantsbuild.example.mngt include "dir1/f.thrift" struct ManagedPerson { 1: f.Person employee @@ -148,7 +163,7 @@ def test_generates_scala(rule_runner: RuleRunner, scrooge_lockfile: JVMLockfileF "src/thrift/dir1/BUILD": "thrift_sources()", "src/thrift/dir2/g.thrift": dedent( """\ - #@namespace scala org.pantsbuild.example + #@namespace scala org.pantsbuild.example.wrapper include "dir1/f2.thrift" struct ManagedPersonWrapper { 1: f2.ManagedPerson managed_person @@ -159,7 +174,7 @@ def test_generates_scala(rule_runner: RuleRunner, scrooge_lockfile: JVMLockfileF # Test another source root. "tests/thrift/test_thrifts/f.thrift": dedent( """\ - #@namespace scala org.pantsbuild.example + #@namespace scala org.pantsbuild.example.test include "dir2/g.thrift" struct Executive { 1: g.ManagedPersonWrapper managed_person_wrapper @@ -169,7 +184,7 @@ def test_generates_scala(rule_runner: RuleRunner, scrooge_lockfile: JVMLockfileF "tests/thrift/test_thrifts/BUILD": "thrift_sources(dependencies=['src/thrift/dir2'])", "3rdparty/jvm/default.lock": scrooge_lockfile.serialized_lockfile, "3rdparty/jvm/BUILD": scrooge_lockfile.requirements_as_jvm_artifact_targets(), - "src/jvm/BUILD": "scala_sources(dependencies=['src/thrift/dir1'])", + "src/jvm/BUILD": "scala_sources()", "src/jvm/TestScroogeThriftScala.scala": dedent( """\ package org.pantsbuild.example @@ -198,22 +213,28 @@ def assert_gen(addr: Address, expected: list[str]) -> None: assert_gen( Address("src/thrift/dir1", relative_file_path="f2.thrift"), [ - "src/thrift/org/pantsbuild/example/ManagedPerson.scala", + "src/thrift/org/pantsbuild/example/mngt/ManagedPerson.scala", ], ) assert_gen( Address("src/thrift/dir2", relative_file_path="g.thrift"), [ - "src/thrift/org/pantsbuild/example/ManagedPersonWrapper.scala", + "src/thrift/org/pantsbuild/example/wrapper/ManagedPersonWrapper.scala", ], ) assert_gen( Address("tests/thrift/test_thrifts", relative_file_path="f.thrift"), [ - "tests/thrift/org/pantsbuild/example/Executive.scala", + "tests/thrift/org/pantsbuild/example/test/Executive.scala", ], ) + tgt = rule_runner.get_target( + Address("src/jvm", relative_file_path="TestScroogeThriftScala.scala") + ) + dependencies = rule_runner.request(Addresses, [DependenciesRequest(tgt[Dependencies])]) + assert Address("src/thrift/dir1", relative_file_path="f.thrift") in dependencies + request = CompileScalaSourceRequest( component=expect_single_expanded_coarsened_target( rule_runner, Address(spec_path="src/jvm") diff --git a/src/python/pants/backend/codegen/thrift/scrooge/scala/scrooge.test.lock b/src/python/pants/backend/codegen/thrift/scrooge/scala/scrooge.test.lock index fd101a30269..feaecd67d27 100644 --- a/src/python/pants/backend/codegen/thrift/scrooge/scala/scrooge.test.lock +++ b/src/python/pants/backend/codegen/thrift/scrooge/scala/scrooge.test.lock @@ -1,13 +1,14 @@ # This lockfile was autogenerated by Pants. To regenerate, run: # -# ./pants internal-generate-test-lockfile-fixtures :: +# pants internal-generate-test-lockfile-fixtures :: # # --- BEGIN PANTS LOCKFILE METADATA: DO NOT EDIT OR REMOVE --- # { # "version": 1, # "generated_with_requirements": [ # "com.twitter:scrooge-core_2.13:21.12.0,url=not_provided,jar=not_provided", -# "org.apache.thrift:libthrift:0.15.0,url=not_provided,jar=not_provided" +# "org.apache.thrift:libthrift:0.15.0,url=not_provided,jar=not_provided", +# "org.scala-lang:scala-library:2.13.6,url=not_provided,jar=not_provided" # ] # } # --- END PANTS LOCKFILE METADATA --- diff --git a/src/python/pants/backend/codegen/thrift/scrooge/scala/symbol_mapper.py b/src/python/pants/backend/codegen/thrift/scrooge/scala/symbol_mapper.py new file mode 100644 index 00000000000..757b2cba7d9 --- /dev/null +++ b/src/python/pants/backend/codegen/thrift/scrooge/scala/symbol_mapper.py @@ -0,0 +1,30 @@ +# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). +from __future__ import annotations + +from pants.backend.codegen.thrift import jvm_symbol_mapper +from pants.backend.codegen.thrift.jvm_symbol_mapper import FirstPartyThriftJvmMappingRequest +from pants.engine.rules import collect_rules, rule +from pants.engine.unions import UnionRule +from pants.jvm.dependency_inference.symbol_mapper import FirstPartyMappingRequest + + +class FirstPartyThriftScroogeScalaTargetsMappingRequest(FirstPartyMappingRequest): + pass + + +@rule +async def map_first_party_thrift_scrooge_scala_targets_to_symbols( + _: FirstPartyThriftScroogeScalaTargetsMappingRequest, +) -> FirstPartyThriftJvmMappingRequest: + return FirstPartyThriftJvmMappingRequest( + extra_lang_ids=("scala",), extra_namespace_directives=("#@namespace",) + ) + + +def rules(): + return [ + *collect_rules(), + *jvm_symbol_mapper.rules(), + UnionRule(FirstPartyMappingRequest, FirstPartyThriftScroogeScalaTargetsMappingRequest), + ] diff --git a/src/python/pants/backend/codegen/thrift/thrift_parser.py b/src/python/pants/backend/codegen/thrift/thrift_parser.py index ba40c73111e..1216dc417e2 100644 --- a/src/python/pants/backend/codegen/thrift/thrift_parser.py +++ b/src/python/pants/backend/codegen/thrift/thrift_parser.py @@ -5,6 +5,7 @@ import re from dataclasses import dataclass +from typing import Mapping from pants.backend.codegen.thrift.target_types import ThriftSourceField from pants.engine.engine_aware import EngineAwareParameter @@ -36,6 +37,7 @@ class ParsedThrift: @dataclass(frozen=True) class ParsedThriftRequest(EngineAwareParameter): sources_field: ThriftSourceField + extra_namespace_directives: tuple[str, ...] = () def debug_hint(self) -> str: return self.sources_field.file_path @@ -46,10 +48,20 @@ async def parse_thrift_file(request: ParsedThriftRequest) -> ParsedThrift: sources = await Get(HydratedSources, HydrateSourcesRequest(request.sources_field)) digest_contents = await Get(DigestContents, Digest, sources.snapshot.digest) assert len(digest_contents) == 1 + file_content = digest_contents[0].content.decode() + extra_namespaces: Mapping[str, str] = {} + if request.extra_namespace_directives: + for directive in request.extra_namespace_directives: + extra_namespace_pattern = re.compile(rf"{directive}\s+([a-z]+)\s+(.+)\s*") + extra_namespaces = { + **extra_namespaces, + **dict(extra_namespace_pattern.findall(file_content)), + } + return ParsedThrift( imports=FrozenOrderedSet(_IMPORT_REGEX.findall(file_content)), - namespaces=FrozenDict(_NAMESPACE_REGEX.findall(file_content)), + namespaces=FrozenDict({**dict(_NAMESPACE_REGEX.findall(file_content)), **extra_namespaces}), ) diff --git a/src/python/pants/backend/codegen/thrift/thrift_parser_test.py b/src/python/pants/backend/codegen/thrift/thrift_parser_test.py index 25e2b2b4d7b..6ecfe500c70 100644 --- a/src/python/pants/backend/codegen/thrift/thrift_parser_test.py +++ b/src/python/pants/backend/codegen/thrift/thrift_parser_test.py @@ -4,6 +4,7 @@ from __future__ import annotations from textwrap import dedent +from typing import Iterable import pytest @@ -14,14 +15,19 @@ from pants.testutil.rule_runner import QueryRule, RuleRunner -def parse(content: str) -> ParsedThrift: +def parse(content: str, *, extra_namespace_directives: Iterable[str] = ()) -> ParsedThrift: rule_runner = RuleRunner( rules=[*thrift_parser.rules(), QueryRule(ParsedThrift, [ParsedThriftRequest])] ) rule_runner.write_files({"f.thrift": content}) return rule_runner.request( ParsedThrift, - [ParsedThriftRequest(ThriftSourceField("f.thrift", Address("", target_name="t")))], + [ + ParsedThriftRequest( + ThriftSourceField("f.thrift", Address("", target_name="t")), + extra_namespace_directives=tuple(extra_namespace_directives), + ) + ], ) @@ -85,3 +91,16 @@ def test_namespaces_invalid() -> None: ) assert not result.imports assert not result.namespaces + + +def test_extra_namespace_directives() -> None: + result = parse( + dedent( + """\ + #@namespace psd mynamespace + """ + ), + extra_namespace_directives=["#@namespace"], + ) + assert not result.imports + assert dict(result.namespaces) == {"psd": "mynamespace"} diff --git a/src/python/pants/backend/experimental/openapi/codegen/java/register.py b/src/python/pants/backend/experimental/openapi/codegen/java/register.py index 5fedcc4d414..911aad89506 100644 --- a/src/python/pants/backend/experimental/openapi/codegen/java/register.py +++ b/src/python/pants/backend/experimental/openapi/codegen/java/register.py @@ -4,12 +4,15 @@ from __future__ import annotations from pants.backend.experimental.java.register import rules as java_rules -from pants.backend.openapi.codegen.java.rules import rules as java_codegen_rules +from pants.backend.experimental.java.register import target_types as java_target_types +from pants.backend.experimental.openapi.register import rules as openapi_rules +from pants.backend.experimental.openapi.register import target_types as openapi_target_types +from pants.backend.openapi.codegen.java.rules import rules as openapi_java_codegen_rules def target_types(): - return [] + return [*java_target_types(), *openapi_target_types()] def rules(): - return [*java_rules(), *java_codegen_rules()] + return [*java_rules(), *openapi_rules(), *openapi_java_codegen_rules()] diff --git a/src/python/pants/backend/experimental/openapi/register.py b/src/python/pants/backend/experimental/openapi/register.py index 60db2725fe6..3610ed4b78a 100644 --- a/src/python/pants/backend/experimental/openapi/register.py +++ b/src/python/pants/backend/experimental/openapi/register.py @@ -13,16 +13,14 @@ OpenApiSourceGeneratorTarget, OpenApiSourceTarget, ) +from pants.backend.openapi.target_types import rules as target_types_rules from pants.engine.rules import Rule from pants.engine.target import Target from pants.engine.unions import UnionRule def rules() -> Iterable[Rule | UnionRule]: - return [ - *dependency_inference.rules(), - *tailor.rules(), - ] + return [*dependency_inference.rules(), *tailor.rules(), *target_types_rules()] def target_types() -> Iterable[type[Target]]: diff --git a/src/python/pants/backend/openapi/codegen/java/rules.py b/src/python/pants/backend/openapi/codegen/java/rules.py index 5f9d0eae4d1..a13647710c4 100644 --- a/src/python/pants/backend/openapi/codegen/java/rules.py +++ b/src/python/pants/backend/openapi/codegen/java/rules.py @@ -6,7 +6,7 @@ from dataclasses import dataclass from pants.backend.java.target_types import JavaSourceField -from pants.backend.openapi.codegen.java import extra_fields +from pants.backend.openapi.codegen.java import extra_fields, symbol_mapper from pants.backend.openapi.codegen.java.extra_fields import ( OpenApiJavaApiPackageField, OpenApiJavaModelPackageField, @@ -263,6 +263,7 @@ def rules(): *generator_process.rules(), *artifact_mapper.rules(), *pom_parser.rules(), + *symbol_mapper.rules(), UnionRule(GenerateSourcesRequest, GenerateJavaFromOpenAPIRequest), UnionRule(InferDependenciesRequest, InferOpenApiJavaRuntimeDependencyRequest), ] diff --git a/src/python/pants/backend/openapi/codegen/java/rules_integration_test.py b/src/python/pants/backend/openapi/codegen/java/rules_integration_test.py index fe116bfb1d3..d52c1ce7eab 100644 --- a/src/python/pants/backend/openapi/codegen/java/rules_integration_test.py +++ b/src/python/pants/backend/openapi/codegen/java/rules_integration_test.py @@ -3,6 +3,7 @@ from __future__ import annotations +from textwrap import dedent from typing import Iterable import pytest @@ -11,6 +12,9 @@ JVMLockfileFixture, JVMLockfileFixtureDefinition, ) +from pants.backend.experimental.java.register import rules as java_backend_rules +from pants.backend.java.compile.javac import CompileJavaSourceRequest +from pants.backend.java.target_types import JavaSourcesGeneratorTarget, JavaSourceTarget from pants.backend.openapi.codegen.java.rules import GenerateJavaFromOpenAPIRequest from pants.backend.openapi.codegen.java.rules import rules as java_codegen_rules from pants.backend.openapi.sample.resources import PETSTORE_SAMPLE_SPEC @@ -22,16 +26,23 @@ OpenApiSourceGeneratorTarget, OpenApiSourceTarget, ) -from pants.core.util_rules import config_files, external_tool, source_files, system_binaries +from pants.backend.openapi.target_types import rules as target_types_rules from pants.engine.addresses import Address, Addresses from pants.engine.target import ( + Dependencies, DependenciesRequest, GeneratedSources, HydratedSources, HydrateSourcesRequest, ) +from pants.jvm import testutil from pants.jvm.target_types import JvmArtifactTarget -from pants.jvm.testutil import maybe_skip_jdk_test +from pants.jvm.testutil import ( + RenderedClasspath, + expect_single_expanded_coarsened_target, + make_resolve, + maybe_skip_jdk_test, +) from pants.testutil.rule_runner import PYTHON_BOOTSTRAP_ENV, QueryRule, RuleRunner @@ -40,20 +51,22 @@ def rule_runner() -> RuleRunner: rule_runner = RuleRunner( target_types=[ JvmArtifactTarget, + JavaSourceTarget, + JavaSourcesGeneratorTarget, OpenApiSourceTarget, OpenApiSourceGeneratorTarget, OpenApiDocumentTarget, OpenApiDocumentGeneratorTarget, ], rules=[ + *java_backend_rules(), *java_codegen_rules(), - *config_files.rules(), - *source_files.rules(), - *external_tool.rules(), - *system_binaries.rules(), + *target_types_rules(), + *testutil.rules(), QueryRule(HydratedSources, (HydrateSourcesRequest,)), QueryRule(GeneratedSources, (GenerateJavaFromOpenAPIRequest,)), QueryRule(Addresses, (DependenciesRequest,)), + QueryRule(RenderedClasspath, (CompileJavaSourceRequest,)), ], ) rule_runner.set_options(args=[], env_inherit=PYTHON_BOOTSTRAP_ENV) @@ -220,3 +233,52 @@ def assert_gen(address: Address, expected: Iterable[str]) -> None: "src/openapi/org/mycompany/PetsApi.java", ], ) + + +@maybe_skip_jdk_test +def test_java_dependency_inference( + rule_runner: RuleRunner, openapi_lockfile: JVMLockfileFixture +) -> None: + rule_runner.write_files( + { + "3rdparty/jvm/default.lock": openapi_lockfile.serialized_lockfile, + "3rdparty/jvm/BUILD": openapi_lockfile.requirements_as_jvm_artifact_targets(), + "src/openapi/BUILD": dedent( + """\ + openapi_document( + name="petstore", + source="petstore_spec.yaml", + java_api_package="org.mycompany.api", + java_model_package="org.mycompany.model", + ) + """ + ), + "src/openapi/petstore_spec.yaml": PETSTORE_SAMPLE_SPEC, + "src/java/BUILD": "java_sources()", + "src/java/Example.java": dedent( + """\ + package org.pantsbuild.java.example; + + import org.mycompany.api.PetsApi; + import org.mycompany.model.Pet; + + public class Example { + PetsApi api; + Pet pet; + } + """ + ), + } + ) + + tgt = rule_runner.get_target(Address("src/java", relative_file_path="Example.java")) + dependencies = rule_runner.request(Addresses, [DependenciesRequest(tgt[Dependencies])]) + assert Address("src/openapi", target_name="petstore") in dependencies + + coarsened_target = expect_single_expanded_coarsened_target( + rule_runner, Address(spec_path="src/java") + ) + _ = rule_runner.request( + RenderedClasspath, + [CompileJavaSourceRequest(component=coarsened_target, resolve=make_resolve(rule_runner))], + ) diff --git a/src/python/pants/backend/openapi/codegen/java/symbol_mapper.py b/src/python/pants/backend/openapi/codegen/java/symbol_mapper.py new file mode 100644 index 00000000000..d67d98ee55a --- /dev/null +++ b/src/python/pants/backend/openapi/codegen/java/symbol_mapper.py @@ -0,0 +1,61 @@ +# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). +from __future__ import annotations + +from collections import defaultdict +from typing import DefaultDict, Mapping, Tuple + +from pants.backend.openapi.codegen.java.extra_fields import ( + OpenApiJavaApiPackageField, + OpenApiJavaModelPackageField, +) +from pants.backend.openapi.target_types import AllOpenApiDocumentTargets +from pants.engine.addresses import Address +from pants.engine.rules import collect_rules, rule +from pants.engine.unions import UnionRule +from pants.jvm.dependency_inference.artifact_mapper import MutableTrieNode +from pants.jvm.dependency_inference.symbol_mapper import FirstPartyMappingRequest, SymbolMap +from pants.jvm.subsystems import JvmSubsystem +from pants.jvm.target_types import JvmResolveField +from pants.util.ordered_set import OrderedSet + +_ResolveName = str + + +class FirstPartyOpenAPIJavaTargetsMappingRequest(FirstPartyMappingRequest): + pass + + +_DEFAULT_API_PACKAGE = "org.openapitools.client.api" +_DEFAULT_MODEL_PACKAGE = "org.openapitools.client.model" + + +@rule +async def map_first_party_openapi_java_targets_to_symbols( + _: FirstPartyOpenAPIJavaTargetsMappingRequest, + all_openapi_document_targets: AllOpenApiDocumentTargets, + jvm: JvmSubsystem, +) -> SymbolMap: + package_mapping: DefaultDict[Tuple[_ResolveName, str], OrderedSet[Address]] = defaultdict( + OrderedSet + ) + for target in all_openapi_document_targets: + resolve_name = target[JvmResolveField].normalized_value(jvm) + api_package = target[OpenApiJavaApiPackageField].value or _DEFAULT_API_PACKAGE + model_package = target[OpenApiJavaModelPackageField].value or _DEFAULT_MODEL_PACKAGE + + package_mapping[(resolve_name, api_package)].add(target.address) + package_mapping[(resolve_name, model_package)].add(target.address) + + symbol_map: Mapping[_ResolveName, MutableTrieNode] = defaultdict(MutableTrieNode) + for (resolve, package), addresses in package_mapping.items(): + symbol_map[resolve].insert(package, addresses, first_party=True, recursive=True) + + return SymbolMap((resolve, node.frozen()) for resolve, node in symbol_map.items()) + + +def rules(): + return [ + *collect_rules(), + UnionRule(FirstPartyMappingRequest, FirstPartyOpenAPIJavaTargetsMappingRequest), + ] diff --git a/src/python/pants/backend/openapi/target_types.py b/src/python/pants/backend/openapi/target_types.py index fa10dc40e68..4d25b667158 100644 --- a/src/python/pants/backend/openapi/target_types.py +++ b/src/python/pants/backend/openapi/target_types.py @@ -3,15 +3,19 @@ from __future__ import annotations +from pants.engine.rules import collect_rules, rule from pants.engine.target import ( COMMON_TARGET_FIELDS, + AllTargets, Dependencies, MultipleSourcesField, SingleSourceField, Target, TargetFilesGenerator, + Targets, generate_multiple_sources_field_help_message, ) +from pants.util.logging import LogLevel OPENAPI_FILE_EXTENSIONS = (".json", ".yaml", ".yml") @@ -64,6 +68,17 @@ class OpenApiDocumentGeneratorTarget(TargetFilesGenerator): help = "Generate an `openapi_document` target for each file in the `sources` field." +class AllOpenApiDocumentTargets(Targets): + pass + + +@rule(desc="Find all OpenAPI Document targets in project", level=LogLevel.DEBUG) +def find_all_openapi_document_targets(all_targets: AllTargets) -> AllOpenApiDocumentTargets: + return AllOpenApiDocumentTargets( + tgt for tgt in all_targets if tgt.has_field(OpenApiDocumentField) + ) + + # ----------------------------------------------------------------------------------------------- # `openapi_source` and `openapi_sources` targets # ----------------------------------------------------------------------------------------------- @@ -102,3 +117,16 @@ class OpenApiSourceGeneratorTarget(TargetFilesGenerator): copied_fields = COMMON_TARGET_FIELDS moved_fields = (OpenApiSourceDependenciesField,) help = "Generate an `openapi_source` target for each file in the `sources` field." + + +class AllOpenApiSourceTargets(Targets): + pass + + +@rule(desc="Find all OpenAPI source targets in project", level=LogLevel.DEBUG) +def find_all_openapi_source_targets(all_targets: AllTargets) -> AllOpenApiSourceTargets: + return AllOpenApiSourceTargets(tgt for tgt in all_targets if tgt.has_field(OpenApiSourceField)) + + +def rules(): + return collect_rules() From 8decaa62c62f6c7658e1c209aae9f2d2713f86f1 Mon Sep 17 00:00:00 2001 From: "A. Alonso Dominguez" <2269440+alonsodomin@users.noreply.github.com> Date: Fri, 29 Dec 2023 20:55:55 +0100 Subject: [PATCH 12/18] DRY gather config files by directories (#20346) Refactors the gathering and indexing of config files by the given file paths. --- .../backend/scala/lint/scalafmt/rules.py | 55 ++++----------- .../lint/scalafmt/rules_integration_test.py | 62 +--------------- .../backend/scala/lint/scalafmt/subsystem.py | 29 +++++++- .../pants/backend/tools/yamllint/rules.py | 62 ++++------------ .../pants/backend/tools/yamllint/subsystem.py | 20 +++++- .../pants/core/util_rules/config_files.py | 70 +++++++++++++++++++ .../core/util_rules/config_files_test.py | 51 +++++++++++++- src/python/pants/util/dirutil_test.py | 26 +++++++ 8 files changed, 219 insertions(+), 156 deletions(-) diff --git a/src/python/pants/backend/scala/lint/scalafmt/rules.py b/src/python/pants/backend/scala/lint/scalafmt/rules.py index 05d448bc13f..76d0d75a8b8 100644 --- a/src/python/pants/backend/scala/lint/scalafmt/rules.py +++ b/src/python/pants/backend/scala/lint/scalafmt/rules.py @@ -12,6 +12,10 @@ from pants.backend.scala.target_types import ScalaSourceField from pants.core.goals.fmt import FmtResult, FmtTargetsRequest, Partitions from pants.core.goals.generate_lockfiles import GenerateToolLockfileSentinel +from pants.core.util_rules.config_files import ( + GatherConfigFilesByDirectoriesRequest, + GatheredConfigFilesByDirectories, +) from pants.core.util_rules.partitions import Partition from pants.engine.fs import Digest, DigestSubset, MergeDigests, PathGlobs, Snapshot from pants.engine.internals.selectors import Get, MultiGet @@ -23,13 +27,11 @@ from pants.jvm.jdk_rules import InternalJdk, JvmProcess from pants.jvm.resolve.coursier_fetch import ToolClasspath, ToolClasspathRequest from pants.jvm.resolve.jvm_tool import GenerateJvmLockfileFromTool, GenerateJvmToolLockfileSentinel -from pants.util.dirutil import find_nearest_ancestor_file, group_by_dir +from pants.util.dirutil import group_by_dir from pants.util.frozendict import FrozenDict from pants.util.logging import LogLevel from pants.util.strutil import pluralize -_SCALAFMT_CONF_FILENAME = ".scalafmt.conf" - @dataclass(frozen=True) class ScalafmtFieldSet(FieldSet): @@ -73,44 +75,6 @@ def description(self) -> str: return self.config_snapshot.files[0] -# @TODO: This logic is very similar, but not identical to the one for yamllint. It should be generalized and shared. -@rule -async def gather_scalafmt_config_files( - request: GatherScalafmtConfigFilesRequest, -) -> ScalafmtConfigFiles: - """Gather scalafmt config files and identify which config files to use for each source - directory.""" - source_dirs = frozenset(os.path.dirname(path) for path in request.filepaths) - - source_dirs_with_ancestors = {"", *source_dirs} - for source_dir in source_dirs: - source_dir_parts = source_dir.split(os.path.sep) - source_dir_parts.pop() - while source_dir_parts: - source_dirs_with_ancestors.add(os.path.sep.join(source_dir_parts)) - source_dir_parts.pop() - - config_file_globs = [ - os.path.join(dir, _SCALAFMT_CONF_FILENAME) for dir in source_dirs_with_ancestors - ] - config_files_snapshot = await Get(Snapshot, PathGlobs(config_file_globs)) - config_files_set = set(config_files_snapshot.files) - - source_dir_to_config_file: dict[str, str] = {} - for source_dir in source_dirs: - config_file = find_nearest_ancestor_file( - config_files_set, source_dir, _SCALAFMT_CONF_FILENAME - ) - if not config_file: - raise ValueError( - f"No scalafmt config file (`{_SCALAFMT_CONF_FILENAME}`) found for " - f"source directory '{source_dir}'" - ) - source_dir_to_config_file[source_dir] = config_file - - return ScalafmtConfigFiles(config_files_snapshot, FrozenDict(source_dir_to_config_file)) - - @rule async def partition_scalafmt( request: ScalafmtRequest.PartitionRequest, tool: ScalafmtSubsystem @@ -125,8 +89,13 @@ async def partition_scalafmt( tool_classpath, config_files = await MultiGet( Get(ToolClasspath, ToolClasspathRequest(lockfile=lockfile_request)), Get( - ScalafmtConfigFiles, - GatherScalafmtConfigFilesRequest(filepaths), + GatheredConfigFilesByDirectories, + GatherConfigFilesByDirectoriesRequest( + tool_name=tool.name, + config_filename=tool.config_file_name, + filepaths=filepaths, + orphan_filepath_behavior=tool.orphan_files_behavior, + ), ), ) diff --git a/src/python/pants/backend/scala/lint/scalafmt/rules_integration_test.py b/src/python/pants/backend/scala/lint/scalafmt/rules_integration_test.py index 9a1bb2b27b1..9d60be57324 100644 --- a/src/python/pants/backend/scala/lint/scalafmt/rules_integration_test.py +++ b/src/python/pants/backend/scala/lint/scalafmt/rules_integration_test.py @@ -10,13 +10,7 @@ from pants.backend.scala import target_types from pants.backend.scala.compile.scalac import rules as scalac_rules from pants.backend.scala.lint.scalafmt import skip_field -from pants.backend.scala.lint.scalafmt.rules import ( - GatherScalafmtConfigFilesRequest, - PartitionInfo, - ScalafmtConfigFiles, - ScalafmtFieldSet, - ScalafmtRequest, -) +from pants.backend.scala.lint.scalafmt.rules import PartitionInfo, ScalafmtFieldSet, ScalafmtRequest from pants.backend.scala.lint.scalafmt.rules import rules as scalafmt_rules from pants.backend.scala.target_types import ScalaSourcesGeneratorTarget, ScalaSourceTarget from pants.build_graph.address import Address @@ -33,7 +27,6 @@ from pants.jvm.strip_jar import strip_jar from pants.jvm.util_rules import rules as util_rules from pants.testutil.rule_runner import PYTHON_BOOTSTRAP_ENV, RuleRunner -from pants.util.dirutil import find_nearest_ancestor_file @pytest.fixture @@ -56,7 +49,6 @@ def rule_runner() -> RuleRunner: QueryRule(Partitions, (ScalafmtRequest.PartitionRequest,)), QueryRule(FmtResult, (ScalafmtRequest.Batch,)), QueryRule(Snapshot, (PathGlobs,)), - QueryRule(ScalafmtConfigFiles, (GatherScalafmtConfigFilesRequest,)), ], target_types=[ScalaSourceTarget, ScalaSourcesGeneratorTarget], ) @@ -241,55 +233,3 @@ def test_multiple_config_files(rule_runner: RuleRunner) -> None: assert fmt_results[1].output == rule_runner.make_snapshot( {"foo/bar/Bar.scala": FIXED_BAD_FILE_INDENT_4} ) - - -def test_find_nearest_ancestor_file() -> None: - files = {"grok.conf", "foo/bar/grok.conf", "hello/world/grok.conf"} - assert find_nearest_ancestor_file(files, "foo/bar", "grok.conf") == "foo/bar/grok.conf" - assert find_nearest_ancestor_file(files, "foo/bar/", "grok.conf") == "foo/bar/grok.conf" - assert find_nearest_ancestor_file(files, "foo", "grok.conf") == "grok.conf" - assert find_nearest_ancestor_file(files, "foo/", "grok.conf") == "grok.conf" - assert find_nearest_ancestor_file(files, "foo/xyzzy", "grok.conf") == "grok.conf" - assert find_nearest_ancestor_file(files, "foo/xyzzy", "grok.conf") == "grok.conf" - assert find_nearest_ancestor_file(files, "", "grok.conf") == "grok.conf" - assert find_nearest_ancestor_file(files, "hello", "grok.conf") == "grok.conf" - assert find_nearest_ancestor_file(files, "hello/", "grok.conf") == "grok.conf" - assert ( - find_nearest_ancestor_file(files, "hello/world/foo", "grok.conf") == "hello/world/grok.conf" - ) - assert ( - find_nearest_ancestor_file(files, "hello/world/foo/", "grok.conf") - == "hello/world/grok.conf" - ) - - files2 = {"foo/bar/grok.conf", "hello/world/grok.conf"} - assert find_nearest_ancestor_file(files2, "foo", "grok.conf") is None - assert find_nearest_ancestor_file(files2, "foo/", "grok.conf") is None - assert find_nearest_ancestor_file(files2, "", "grok.conf") is None - - -def test_gather_scalafmt_config_files(rule_runner: RuleRunner) -> None: - rule_runner.write_files( - { - SCALAFMT_CONF_FILENAME: "", - f"foo/bar/{SCALAFMT_CONF_FILENAME}": "", - f"hello/{SCALAFMT_CONF_FILENAME}": "", - "hello/Foo.scala": "", - "hello/world/Foo.scala": "", - "foo/bar/Foo.scala": "", - "foo/bar/xyyzzy/Foo.scala": "", - "foo/blah/Foo.scala": "", - } - ) - - snapshot = rule_runner.request(Snapshot, [PathGlobs(["**/*.scala"])]) - request = rule_runner.request( - ScalafmtConfigFiles, [GatherScalafmtConfigFilesRequest(snapshot.files)] - ) - assert sorted(request.source_dir_to_config_file.items()) == [ - ("foo/bar", "foo/bar/.scalafmt.conf"), - ("foo/bar/xyyzzy", "foo/bar/.scalafmt.conf"), - ("foo/blah", ".scalafmt.conf"), - ("hello", "hello/.scalafmt.conf"), - ("hello/world", "hello/.scalafmt.conf"), - ] diff --git a/src/python/pants/backend/scala/lint/scalafmt/subsystem.py b/src/python/pants/backend/scala/lint/scalafmt/subsystem.py index 66f677fc33b..fed173b7a3a 100644 --- a/src/python/pants/backend/scala/lint/scalafmt/subsystem.py +++ b/src/python/pants/backend/scala/lint/scalafmt/subsystem.py @@ -1,8 +1,12 @@ # Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). +from pants.core.util_rules.config_files import OrphanFilepathConfigBehavior from pants.jvm.resolve.jvm_tool import JvmToolBase -from pants.option.option_types import SkipOption +from pants.option.option_types import EnumOption, SkipOption, StrOption +from pants.util.strutil import softwrap + +DEFAULT_SCALAFMT_CONF_FILENAME = ".scalafmt.conf" class ScalafmtSubsystem(JvmToolBase): @@ -17,4 +21,27 @@ class ScalafmtSubsystem(JvmToolBase): "scalafmt.default.lockfile.txt", ) + config_file_name = StrOption( + "--config-file-name", + default=DEFAULT_SCALAFMT_CONF_FILENAME, + advanced=True, + help=softwrap( + """ + Name of a config file understood by scalafmt (https://scalameta.org/scalafmt/docs/configuration.html). + The plugin will search the ancestors of each directory in which Scala files are found for a config file of this name. + """ + ), + ) + + orphan_files_behavior = EnumOption( + default=OrphanFilepathConfigBehavior.ERROR, + advanced=True, + help=softwrap( + f""" + Whether to ignore, error or show a warning when files are found that are not + covered by the config file provided in `[{options_scope}].config_file_name` setting. + """ + ), + ) + skip = SkipOption("fmt", "lint") diff --git a/src/python/pants/backend/tools/yamllint/rules.py b/src/python/pants/backend/tools/yamllint/rules.py index a82e323f2a7..9bff99905c3 100644 --- a/src/python/pants/backend/tools/yamllint/rules.py +++ b/src/python/pants/backend/tools/yamllint/rules.py @@ -10,13 +10,16 @@ from pants.backend.python.util_rules.pex import Pex, PexProcess, PexRequest from pants.backend.tools.yamllint.subsystem import Yamllint from pants.core.goals.lint import LintFilesRequest, LintResult +from pants.core.util_rules.config_files import ( + GatherConfigFilesByDirectoriesRequest, + GatheredConfigFilesByDirectories, +) from pants.core.util_rules.partitions import Partition, Partitions from pants.engine.fs import Digest, DigestSubset, MergeDigests, PathGlobs from pants.engine.internals.native_engine import FilespecMatcher, Snapshot from pants.engine.process import FallibleProcessResult from pants.engine.rules import Get, MultiGet, collect_rules, rule -from pants.util.dirutil import find_nearest_ancestor_file, group_by_dir -from pants.util.frozendict import FrozenDict +from pants.util.dirutil import group_by_dir from pants.util.logging import LogLevel from pants.util.strutil import pluralize @@ -37,49 +40,6 @@ def description(self) -> str: return "" -@dataclass(frozen=True) -class YamllintConfigFilesRequest: - filepaths: tuple[str, ...] - - -@dataclass(frozen=True) -class YamllintConfigFiles: - snapshot: Snapshot - source_dir_to_config_files: FrozenDict[str, str] - - -# @TODO: This logic is very similar, but not identical to the one for scalafmt. It should be generalized and shared. -@rule -async def gather_config_files( - request: YamllintConfigFilesRequest, yamllint: Yamllint -) -> YamllintConfigFiles: - """Gather yamllint configuration files.""" - source_dirs = frozenset(os.path.dirname(path) for path in request.filepaths) - source_dirs_with_ancestors = {"", *source_dirs} - for source_dir in source_dirs: - source_dir_parts = source_dir.split(os.path.sep) - source_dir_parts.pop() - while source_dir_parts: - source_dirs_with_ancestors.add(os.path.sep.join(source_dir_parts)) - source_dir_parts.pop() - - config_file_globs = [ - os.path.join(dir, yamllint.config_file_name) for dir in source_dirs_with_ancestors - ] - config_files_snapshot = await Get(Snapshot, PathGlobs(config_file_globs)) - config_files_set = set(config_files_snapshot.files) - - source_dir_to_config_file: dict[str, str] = {} - for source_dir in source_dirs: - config_file = find_nearest_ancestor_file( - config_files_set, source_dir, yamllint.config_file_name - ) - if config_file: - source_dir_to_config_file[source_dir] = config_file - - return YamllintConfigFiles(config_files_snapshot, FrozenDict(source_dir_to_config_file)) - - @rule async def partition_inputs( request: YamllintRequest.PartitionRequest, yamllint: Yamllint @@ -92,15 +52,21 @@ async def partition_inputs( ).matches(request.files) config_files = await Get( - YamllintConfigFiles, YamllintConfigFilesRequest(filepaths=tuple(sorted(matching_filepaths))) + GatheredConfigFilesByDirectories, + GatherConfigFilesByDirectoriesRequest( + tool_name=yamllint.name, + config_filename=yamllint.config_file_name, + filepaths=tuple(sorted(matching_filepaths)), + orphan_filepath_behavior=yamllint.orphan_files_behavior, + ), ) default_source_files: set[str] = set() source_files_by_config_file: dict[str, set[str]] = defaultdict(set) for source_dir, files_in_source_dir in group_by_dir(matching_filepaths).items(): files = (os.path.join(source_dir, name) for name in files_in_source_dir) - if source_dir in config_files.source_dir_to_config_files: - config_file = config_files.source_dir_to_config_files[source_dir] + if source_dir in config_files.source_dir_to_config_file: + config_file = config_files.source_dir_to_config_file[source_dir] source_files_by_config_file[config_file].update(files) else: default_source_files.update(files) diff --git a/src/python/pants/backend/tools/yamllint/subsystem.py b/src/python/pants/backend/tools/yamllint/subsystem.py index b24d90cf53f..5e65e37960c 100644 --- a/src/python/pants/backend/tools/yamllint/subsystem.py +++ b/src/python/pants/backend/tools/yamllint/subsystem.py @@ -7,9 +7,16 @@ from pants.backend.python.subsystems.python_tool_base import PythonToolBase from pants.backend.python.target_types import ConsoleScript +from pants.core.util_rules.config_files import OrphanFilepathConfigBehavior from pants.engine.rules import Rule, collect_rules from pants.engine.unions import UnionRule -from pants.option.option_types import ArgsListOption, SkipOption, StrListOption, StrOption +from pants.option.option_types import ( + ArgsListOption, + EnumOption, + SkipOption, + StrListOption, + StrOption, +) from pants.util.strutil import softwrap @@ -37,6 +44,17 @@ class Yamllint(PythonToolBase): ), ) + orphan_files_behavior = EnumOption( + default=OrphanFilepathConfigBehavior.IGNORE, + advanced=True, + help=softwrap( + f""" + Whether to ignore, error or show a warning when files are found that are not + covered by the config file provided in `[{options_scope}].config_file_name` setting. + """ + ), + ) + file_glob_include = StrListOption( "--include", default=["**/*.yml", "**/*.yaml"], diff --git a/src/python/pants/core/util_rules/config_files.py b/src/python/pants/core/util_rules/config_files.py index d2cbe3918b1..b19b9b19a5b 100644 --- a/src/python/pants/core/util_rules/config_files.py +++ b/src/python/pants/core/util_rules/config_files.py @@ -4,15 +4,19 @@ from __future__ import annotations import logging +import os from dataclasses import dataclass +from enum import Enum from typing import Iterable, Mapping from pants.base.glob_match_error_behavior import GlobMatchErrorBehavior from pants.engine.fs import EMPTY_SNAPSHOT, DigestContents, PathGlobs, Snapshot from pants.engine.rules import Get, collect_rules, rule from pants.util.collections import ensure_str_list +from pants.util.dirutil import find_nearest_ancestor_file from pants.util.frozendict import FrozenDict from pants.util.logging import LogLevel +from pants.util.strutil import softwrap logger = logging.getLogger(__name__) @@ -83,5 +87,71 @@ async def find_config_file(request: ConfigFilesRequest) -> ConfigFiles: return ConfigFiles(config_snapshot) +class OrphanFilepathConfigBehavior(Enum): + IGNORE = "ignore" + ERROR = "error" + WARN = "warn" + + +@dataclass(frozen=True) +class GatheredConfigFilesByDirectories: + config_filename: str + snapshot: Snapshot + source_dir_to_config_file: FrozenDict[str, str] + + +@dataclass(frozen=True) +class GatherConfigFilesByDirectoriesRequest: + tool_name: str + config_filename: str + filepaths: tuple[str, ...] + orphan_filepath_behavior: OrphanFilepathConfigBehavior = OrphanFilepathConfigBehavior.ERROR + + +@rule +async def gather_config_files_by_workspace_dir( + request: GatherConfigFilesByDirectoriesRequest, +) -> GatheredConfigFilesByDirectories: + """Gathers config files from the workspace and indexes them by the directories relative to + them.""" + + source_dirs = frozenset(os.path.dirname(path) for path in request.filepaths) + source_dirs_with_ancestors = {"", *source_dirs} + for source_dir in source_dirs: + source_dir_parts = source_dir.split(os.path.sep) + source_dir_parts.pop() + while source_dir_parts: + source_dirs_with_ancestors.add(os.path.sep.join(source_dir_parts)) + source_dir_parts.pop() + + config_file_globs = [ + os.path.join(dir, request.config_filename) for dir in source_dirs_with_ancestors + ] + config_files_snapshot = await Get(Snapshot, PathGlobs(config_file_globs)) + config_files_set = set(config_files_snapshot.files) + source_dir_to_config_file: dict[str, str] = {} + for source_dir in source_dirs: + config_file = find_nearest_ancestor_file( + config_files_set, source_dir, request.config_filename + ) + if config_file: + source_dir_to_config_file[source_dir] = config_file + else: + msg = softwrap( + f""" + No {request.tool_name} file (`{request.config_filename}`) found for + source directory '{source_dir}'. + """ + ) + if request.orphan_filepath_behavior == OrphanFilepathConfigBehavior.ERROR: + raise ValueError(msg) + else: + logger.warn(msg) + + return GatheredConfigFilesByDirectories( + request.config_filename, config_files_snapshot, FrozenDict(source_dir_to_config_file) + ) + + def rules(): return collect_rules() diff --git a/src/python/pants/core/util_rules/config_files_test.py b/src/python/pants/core/util_rules/config_files_test.py index fb813b2b674..a2525b93794 100644 --- a/src/python/pants/core/util_rules/config_files_test.py +++ b/src/python/pants/core/util_rules/config_files_test.py @@ -6,14 +6,26 @@ import pytest from pants.core.util_rules import config_files -from pants.core.util_rules.config_files import ConfigFiles, ConfigFilesRequest +from pants.core.util_rules.config_files import ( + ConfigFiles, + ConfigFilesRequest, + GatherConfigFilesByDirectoriesRequest, + GatheredConfigFilesByDirectories, +) +from pants.engine.fs import PathGlobs, Snapshot from pants.engine.internals.scheduler import ExecutionError from pants.testutil.rule_runner import QueryRule, RuleRunner @pytest.fixture def rule_runner() -> RuleRunner: - return RuleRunner(rules=[*config_files.rules(), QueryRule(ConfigFiles, [ConfigFilesRequest])]) + return RuleRunner( + rules=[ + *config_files.rules(), + QueryRule(ConfigFiles, [ConfigFilesRequest]), + QueryRule(GatheredConfigFilesByDirectories, [GatherConfigFilesByDirectoriesRequest]), + ] + ) def test_resolve_if_specified(rule_runner: RuleRunner) -> None: @@ -57,3 +69,38 @@ def discover( assert discover(["fake"], {"c4": b"bad"}) == () # Explicitly specifying turns off auto-discovery. assert discover(["c1"], {}, specified="c2") == ("c2",) + + +TEST_CONFIG_FILENAME = "myconfig.cfg" + + +def test_gather_config_files(rule_runner: RuleRunner) -> None: + rule_runner.write_files( + { + TEST_CONFIG_FILENAME: "", + f"foo/bar/{TEST_CONFIG_FILENAME}": "", + f"hello/{TEST_CONFIG_FILENAME}": "", + "hello/Foo.x": "", + "hello/world/Foo.x": "", + "foo/bar/Foo.x": "", + "foo/bar/xyyzzy/Foo.x": "", + "foo/blah/Foo.x": "", + } + ) + + snapshot = rule_runner.request(Snapshot, [PathGlobs(["**/*.x"])]) + request = rule_runner.request( + GatheredConfigFilesByDirectories, + [ + GatherConfigFilesByDirectoriesRequest( + tool_name="test", config_filename=TEST_CONFIG_FILENAME, filepaths=snapshot.files + ) + ], + ) + assert sorted(request.source_dir_to_config_file.items()) == [ + ("foo/bar", f"foo/bar/{TEST_CONFIG_FILENAME}"), + ("foo/bar/xyyzzy", f"foo/bar/{TEST_CONFIG_FILENAME}"), + ("foo/blah", TEST_CONFIG_FILENAME), + ("hello", f"hello/{TEST_CONFIG_FILENAME}"), + ("hello/world", f"hello/{TEST_CONFIG_FILENAME}"), + ] diff --git a/src/python/pants/util/dirutil_test.py b/src/python/pants/util/dirutil_test.py index ebc1d9fb898..2b472ceb6a0 100644 --- a/src/python/pants/util/dirutil_test.py +++ b/src/python/pants/util/dirutil_test.py @@ -19,6 +19,7 @@ _mkdtemp_unregister_cleaner, absolute_symlink, fast_relpath, + find_nearest_ancestor_file, group_by_dir, longest_dir_prefix, read_file, @@ -434,3 +435,28 @@ def test_overwrite_file(self) -> None: def test_overwrite_dir(self) -> None: os.makedirs(os.path.join(self.link, "a", "b", "c")) self._create_and_check_link(self.source, self.link) + + +def test_find_nearest_ancestor_file() -> None: + files = {"grok.conf", "foo/bar/grok.conf", "hello/world/grok.conf"} + assert find_nearest_ancestor_file(files, "foo/bar", "grok.conf") == "foo/bar/grok.conf" + assert find_nearest_ancestor_file(files, "foo/bar/", "grok.conf") == "foo/bar/grok.conf" + assert find_nearest_ancestor_file(files, "foo", "grok.conf") == "grok.conf" + assert find_nearest_ancestor_file(files, "foo/", "grok.conf") == "grok.conf" + assert find_nearest_ancestor_file(files, "foo/xyzzy", "grok.conf") == "grok.conf" + assert find_nearest_ancestor_file(files, "foo/xyzzy", "grok.conf") == "grok.conf" + assert find_nearest_ancestor_file(files, "", "grok.conf") == "grok.conf" + assert find_nearest_ancestor_file(files, "hello", "grok.conf") == "grok.conf" + assert find_nearest_ancestor_file(files, "hello/", "grok.conf") == "grok.conf" + assert ( + find_nearest_ancestor_file(files, "hello/world/foo", "grok.conf") == "hello/world/grok.conf" + ) + assert ( + find_nearest_ancestor_file(files, "hello/world/foo/", "grok.conf") + == "hello/world/grok.conf" + ) + + files2 = {"foo/bar/grok.conf", "hello/world/grok.conf"} + assert find_nearest_ancestor_file(files2, "foo", "grok.conf") is None + assert find_nearest_ancestor_file(files2, "foo/", "grok.conf") is None + assert find_nearest_ancestor_file(files2, "", "grok.conf") is None From 4119004158db33a4ed311c04c2f4d31bd831dd0e Mon Sep 17 00:00:00 2001 From: cburroughs Date: Sat, 30 Dec 2023 00:03:44 -0500 Subject: [PATCH 13/18] use pyupgrade to modernize syntax forward to 3.8 (#20317) What, Python 3.8 you say? Hasn't Pants long been on 3.9? Yes, but 3.8 is the last version for which pyupgrade supports `--keep-runtime-typing`. Said runtime typing needs to be preserved for now because Pants itself doesn't yet support the spiffy "new" typing (`foo | None` instead of `Optional`). --- build-support/bin/generate_docs.py | 2 +- pants.toml | 2 +- .../pants/backend/docker/goals/package_image.py | 4 +--- src/python/pants/backend/go/util_rules/cgo.py | 8 ++++---- .../pants/backend/go/util_rules/coverage_html.py | 4 ++-- .../pants/backend/javascript/package_json.py | 3 +-- src/python/pants/backend/project_info/peek.py | 4 +--- .../framework/django/dependency_inference.py | 2 +- .../python/util_rules/interpreter_constraints.py | 3 +-- .../pants/backend/python/util_rules/partition.py | 4 +--- .../pants/backend/python/util_rules/scripts/BUILD | 5 ++++- src/python/pants/base/parse_context.py | 4 +--- src/python/pants/base/specs.py | 4 +--- src/python/pants/bsp/protocol.py | 8 +------- src/python/pants/core/goals/fix.py | 15 ++++++++++++--- src/python/pants/core/goals/generate_lockfiles.py | 4 +--- src/python/pants/core/goals/lint.py | 4 ++-- .../pants/core/goals/multi_tool_goal_helper.py | 4 +--- src/python/pants/core/util_rules/partitions.py | 4 +--- src/python/pants/engine/internals/dep_rules.py | 3 +-- .../pants/engine/internals/native_engine.pyi | 3 ++- src/python/pants/engine/rules.py | 3 ++- src/python/pants/engine/target.py | 3 ++- src/python/pants/help/help_printer.py | 4 +--- src/python/pants/option/config.py | 3 +-- src/python/pants/pantsd/pants_daemon_core.py | 4 +--- .../testutil/python_interpreter_selection.py | 2 +- src/python/pants/testutil/skip_utils.py | 2 +- src/python/pants/util/dirutil.py | 4 +--- src/python/pants_release/reversion.py | 12 +++++------- .../src/python/coordinated_runs/waiter.py | 2 +- testprojects/src/python/hello/greet/greet.py | 2 +- 32 files changed, 58 insertions(+), 77 deletions(-) diff --git a/build-support/bin/generate_docs.py b/build-support/bin/generate_docs.py index a672e369ef0..b5d919a72d3 100644 --- a/build-support/bin/generate_docs.py +++ b/build-support/bin/generate_docs.py @@ -234,7 +234,7 @@ def run_pants_help_all() -> dict[str, Any]: "--no-verify-config", "help-all", ] - run = subprocess.run(argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding="utf-8") + run = subprocess.run(argv, capture_output=True, encoding="utf-8") try: run.check_returncode() except subprocess.CalledProcessError: diff --git a/pants.toml b/pants.toml index 47baff727fd..d6a901e7b2f 100644 --- a/pants.toml +++ b/pants.toml @@ -207,7 +207,7 @@ template_by_globs = "@build-support/preambles/config.yaml" diff = true [pyupgrade] -args = ["--keep-runtime-typing"] +args = ["--keep-runtime-typing", "--py38-plus"] [jvm] diff --git a/src/python/pants/backend/docker/goals/package_image.py b/src/python/pants/backend/docker/goals/package_image.py index b035e5f8270..7b8c0f5d087 100644 --- a/src/python/pants/backend/docker/goals/package_image.py +++ b/src/python/pants/backend/docker/goals/package_image.py @@ -9,9 +9,7 @@ from dataclasses import asdict, dataclass from functools import partial from itertools import chain -from typing import Iterator, cast - -from typing_extensions import Literal +from typing import Iterator, Literal, cast # Re-exporting BuiltDockerImage here, as it has its natural home here, but has moved out to resolve # a dependency cycle from docker_build_context. diff --git a/src/python/pants/backend/go/util_rules/cgo.py b/src/python/pants/backend/go/util_rules/cgo.py index 4150131b71d..c5db9d0ed64 100644 --- a/src/python/pants/backend/go/util_rules/cgo.py +++ b/src/python/pants/backend/go/util_rules/cgo.py @@ -831,7 +831,7 @@ async def cgo_compile_request( # C files cflags = [*flags.cppflags, *flags.cflags] for gcc_file in gcc_files: - ofile = os.path.join(obj_dir_path, "_x{:03}.o".format(oseq)) + ofile = os.path.join(obj_dir_path, f"_x{oseq:03}.o") oseq = oseq + 1 out_obj_files.append(ofile) @@ -850,7 +850,7 @@ async def cgo_compile_request( # C++ files cxxflags = [*flags.cppflags, *flags.cxxflags] for cxx_file in (os.path.join(dir_path, cxx_file) for cxx_file in request.cxx_files): - ofile = os.path.join(obj_dir_path, "_x{:03}.o".format(oseq)) + ofile = os.path.join(obj_dir_path, f"_x{oseq:03}.o") oseq = oseq + 1 out_obj_files.append(ofile) @@ -868,7 +868,7 @@ async def cgo_compile_request( # Objective-C files for objc_file in (os.path.join(dir_path, objc_file) for objc_file in request.objc_files): - ofile = os.path.join(obj_dir_path, "_x{:03}.o".format(oseq)) + ofile = os.path.join(obj_dir_path, f"_x{oseq:03}.o") oseq = oseq + 1 out_obj_files.append(ofile) @@ -888,7 +888,7 @@ async def cgo_compile_request( for fortran_file in ( os.path.join(dir_path, fortran_file) for fortran_file in request.fortran_files ): - ofile = os.path.join(obj_dir_path, "_x{:03}.o".format(oseq)) + ofile = os.path.join(obj_dir_path, f"_x{oseq:03}.o") oseq = oseq + 1 out_obj_files.append(ofile) diff --git a/src/python/pants/backend/go/util_rules/coverage_html.py b/src/python/pants/backend/go/util_rules/coverage_html.py index f608df3fa04..2ad2a7a7246 100644 --- a/src/python/pants/backend/go/util_rules/coverage_html.py +++ b/src/python/pants/backend/go/util_rules/coverage_html.py @@ -89,7 +89,7 @@ def _render_source_file(content: bytes, boundaries: Sequence[GoCoverageBoundary] n = 0 if b.count > 0: n = int(math.floor(b.norm * 9)) + 1 - rendered.write(''.format(n, b.count)) + rendered.write(f'') else: rendered.write("") boundaries = boundaries[1:] @@ -154,7 +154,7 @@ async def render_go_coverage_profile_to_html( { "i": i, "name": file.name, - "coverage": "{:.1f}".format(file.coverage), + "coverage": f"{file.coverage:.1f}", "body": file.body, } for i, file in enumerate(files) diff --git a/src/python/pants/backend/javascript/package_json.py b/src/python/pants/backend/javascript/package_json.py index 86b99552511..a406ec2eaeb 100644 --- a/src/python/pants/backend/javascript/package_json.py +++ b/src/python/pants/backend/javascript/package_json.py @@ -8,10 +8,9 @@ import os.path from abc import ABC from dataclasses import dataclass, field -from typing import Any, ClassVar, Iterable, Mapping, Optional, Tuple +from typing import Any, ClassVar, Iterable, Literal, Mapping, Optional, Tuple import yaml -from typing_extensions import Literal from pants.backend.project_info import dependencies from pants.base.glob_match_error_behavior import GlobMatchErrorBehavior diff --git a/src/python/pants/backend/project_info/peek.py b/src/python/pants/backend/project_info/peek.py index d6eaeda481c..67249a7bc21 100644 --- a/src/python/pants/backend/project_info/peek.py +++ b/src/python/pants/backend/project_info/peek.py @@ -6,9 +6,7 @@ import collections import json from dataclasses import dataclass, fields, is_dataclass -from typing import Any, Iterable, Mapping - -from typing_extensions import Protocol, runtime_checkable +from typing import Any, Iterable, Mapping, Protocol, runtime_checkable from pants.engine.addresses import Addresses from pants.engine.collection import Collection diff --git a/src/python/pants/backend/python/framework/django/dependency_inference.py b/src/python/pants/backend/python/framework/django/dependency_inference.py index 7c806282961..dee0294223d 100644 --- a/src/python/pants/backend/python/framework/django/dependency_inference.py +++ b/src/python/pants/backend/python/framework/django/dependency_inference.py @@ -86,7 +86,7 @@ async def _django_migration_dependencies( # default for decode(), we make that explicit here for emphasis. process_output = process_result.stdout.decode("utf8") or "{}" modules = [ - "{}.migrations.{}".format(django_apps.label_to_name[label], migration) + f"{django_apps.label_to_name[label]}.migrations.{migration}" for label, migration in json.loads(process_output) if label in django_apps.label_to_name ] diff --git a/src/python/pants/backend/python/util_rules/interpreter_constraints.py b/src/python/pants/backend/python/util_rules/interpreter_constraints.py index 9b331890604..af525782a71 100644 --- a/src/python/pants/backend/python/util_rules/interpreter_constraints.py +++ b/src/python/pants/backend/python/util_rules/interpreter_constraints.py @@ -6,10 +6,9 @@ import itertools import re from collections import defaultdict -from typing import Iterable, Iterator, Sequence, Tuple, TypeVar +from typing import Iterable, Iterator, Protocol, Sequence, Tuple, TypeVar from pkg_resources import Requirement -from typing_extensions import Protocol from pants.backend.python.subsystems.setup import PythonSetup from pants.backend.python.target_types import InterpreterConstraintsField diff --git a/src/python/pants/backend/python/util_rules/partition.py b/src/python/pants/backend/python/util_rules/partition.py index 23f11a6b263..e9c64c3ed52 100644 --- a/src/python/pants/backend/python/util_rules/partition.py +++ b/src/python/pants/backend/python/util_rules/partition.py @@ -5,9 +5,7 @@ import itertools from collections import defaultdict -from typing import Iterable, Mapping, Sequence, TypeVar - -from typing_extensions import Protocol +from typing import Iterable, Mapping, Protocol, Sequence, TypeVar from pants.backend.python.subsystems.setup import PythonSetup from pants.backend.python.target_types import InterpreterConstraintsField, PythonResolveField diff --git a/src/python/pants/backend/python/util_rules/scripts/BUILD b/src/python/pants/backend/python/util_rules/scripts/BUILD index 752b6d575e9..7fc0737b959 100644 --- a/src/python/pants/backend/python/util_rules/scripts/BUILD +++ b/src/python/pants/backend/python/util_rules/scripts/BUILD @@ -1,3 +1,6 @@ # Copyright 2023 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). -python_sources() +python_sources( + # Used for Python 2.7 distributions + skip_pyupgrade=True, +) diff --git a/src/python/pants/base/parse_context.py b/src/python/pants/base/parse_context.py index 3ff094346c5..062fae97fad 100644 --- a/src/python/pants/base/parse_context.py +++ b/src/python/pants/base/parse_context.py @@ -2,9 +2,7 @@ # Licensed under the Apache License, Version 2.0 (see LICENSE). import os -from typing import Any, Mapping - -from typing_extensions import Protocol +from typing import Any, Mapping, Protocol class FilePathOracle(Protocol): diff --git a/src/python/pants/base/specs.py b/src/python/pants/base/specs.py index 62a12708c35..db9450f2b69 100644 --- a/src/python/pants/base/specs.py +++ b/src/python/pants/base/specs.py @@ -6,9 +6,7 @@ import os from abc import ABC, abstractmethod from dataclasses import dataclass -from typing import ClassVar, Iterable, Iterator, cast - -from typing_extensions import Protocol +from typing import ClassVar, Iterable, Iterator, Protocol, cast from pants.base.glob_match_error_behavior import GlobMatchErrorBehavior from pants.build_graph.address import Address diff --git a/src/python/pants/bsp/protocol.py b/src/python/pants/bsp/protocol.py index 3742438af3d..0c6464fba01 100644 --- a/src/python/pants/bsp/protocol.py +++ b/src/python/pants/bsp/protocol.py @@ -4,7 +4,7 @@ import logging from concurrent.futures import Future -from typing import Any, BinaryIO, ClassVar +from typing import Any, BinaryIO, ClassVar, Protocol from pylsp_jsonrpc.endpoint import Endpoint # type: ignore[import] from pylsp_jsonrpc.exceptions import ( # type: ignore[import] @@ -23,12 +23,6 @@ from pants.engine.internals.selectors import Params from pants.engine.unions import UnionMembership, union -try: - from typing import Protocol # Python 3.8+ -except ImportError: - # See https://github.com/python/mypy/issues/4427 re the ignore - from typing_extensions import Protocol # type: ignore - _logger = logging.getLogger(__name__) diff --git a/src/python/pants/core/goals/fix.py b/src/python/pants/core/goals/fix.py index 117728642bf..1bcfd384bbf 100644 --- a/src/python/pants/core/goals/fix.py +++ b/src/python/pants/core/goals/fix.py @@ -7,9 +7,18 @@ import logging from collections import defaultdict from dataclasses import dataclass -from typing import Any, Callable, Iterable, Iterator, NamedTuple, Sequence, Tuple, Type, TypeVar - -from typing_extensions import Protocol +from typing import ( + Any, + Callable, + Iterable, + Iterator, + NamedTuple, + Protocol, + Sequence, + Tuple, + Type, + TypeVar, +) from pants.base.specs import Specs from pants.core.goals.lint import ( diff --git a/src/python/pants/core/goals/generate_lockfiles.py b/src/python/pants/core/goals/generate_lockfiles.py index d6ce1434507..a7cd490e74b 100644 --- a/src/python/pants/core/goals/generate_lockfiles.py +++ b/src/python/pants/core/goals/generate_lockfiles.py @@ -8,9 +8,7 @@ from collections import defaultdict from dataclasses import dataclass, replace from enum import Enum -from typing import Callable, ClassVar, Iterable, Iterator, Mapping, Sequence, Tuple, cast - -from typing_extensions import Protocol +from typing import Callable, ClassVar, Iterable, Iterator, Mapping, Protocol, Sequence, Tuple, cast from pants.engine.collection import Collection from pants.engine.console import Console diff --git a/src/python/pants/core/goals/lint.py b/src/python/pants/core/goals/lint.py index 40f4b707be8..9545d41c558 100644 --- a/src/python/pants/core/goals/lint.py +++ b/src/python/pants/core/goals/lint.py @@ -6,9 +6,9 @@ import logging from collections import defaultdict from dataclasses import dataclass -from typing import Any, Callable, ClassVar, Iterable, Iterator, Sequence, TypeVar, cast +from typing import Any, Callable, ClassVar, Iterable, Iterator, Protocol, Sequence, TypeVar, cast -from typing_extensions import Protocol, final +from typing_extensions import final from pants.base.specs import Specs from pants.core.goals.multi_tool_goal_helper import ( diff --git a/src/python/pants/core/goals/multi_tool_goal_helper.py b/src/python/pants/core/goals/multi_tool_goal_helper.py index 18bc3552aa7..5f6ff8ebb85 100644 --- a/src/python/pants/core/goals/multi_tool_goal_helper.py +++ b/src/python/pants/core/goals/multi_tool_goal_helper.py @@ -5,9 +5,7 @@ import logging import os.path -from typing import Iterable, Mapping, Sequence, TypeVar - -from typing_extensions import Protocol +from typing import Iterable, Mapping, Protocol, Sequence, TypeVar from pants.core.util_rules.distdir import DistDir from pants.engine.fs import EMPTY_DIGEST, Digest, Workspace diff --git a/src/python/pants/core/util_rules/partitions.py b/src/python/pants/core/util_rules/partitions.py index cc9d3def0f3..b5c3ba0a7fa 100644 --- a/src/python/pants/core/util_rules/partitions.py +++ b/src/python/pants/core/util_rules/partitions.py @@ -8,9 +8,7 @@ import itertools from dataclasses import dataclass from enum import Enum -from typing import Generic, Iterable, TypeVar, overload - -from typing_extensions import Protocol +from typing import Generic, Iterable, Protocol, TypeVar, overload from pants.core.goals.multi_tool_goal_helper import SkippableSubsystem from pants.engine.collection import Collection diff --git a/src/python/pants/engine/internals/dep_rules.py b/src/python/pants/engine/internals/dep_rules.py index 6b47b526109..ce071ca2ddb 100644 --- a/src/python/pants/engine/internals/dep_rules.py +++ b/src/python/pants/engine/internals/dep_rules.py @@ -6,8 +6,7 @@ from abc import ABC, abstractmethod from dataclasses import dataclass from enum import Enum - -from typing_extensions import Protocol +from typing import Protocol from pants.engine.addresses import Address from pants.engine.internals.target_adaptor import TargetAdaptor diff --git a/src/python/pants/engine/internals/native_engine.pyi b/src/python/pants/engine/internals/native_engine.pyi index 51f80812c94..70fb907c301 100644 --- a/src/python/pants/engine/internals/native_engine.pyi +++ b/src/python/pants/engine/internals/native_engine.pyi @@ -12,6 +12,7 @@ from typing import ( Generic, Iterable, Mapping, + Protocol, Sequence, TextIO, Tuple, @@ -19,7 +20,7 @@ from typing import ( overload, ) -from typing_extensions import Protocol, Self +from typing_extensions import Self from pants.engine.internals.scheduler import Workunit, _PathGlobsAndRootCollection from pants.engine.internals.session import SessionValues diff --git a/src/python/pants/engine/rules.py b/src/python/pants/engine/rules.py index 4ce8f2a1016..0118a28695c 100644 --- a/src/python/pants/engine/rules.py +++ b/src/python/pants/engine/rules.py @@ -16,6 +16,7 @@ Iterable, Mapping, Optional, + Protocol, Sequence, Tuple, Type, @@ -26,7 +27,7 @@ overload, ) -from typing_extensions import ParamSpec, Protocol +from typing_extensions import ParamSpec from pants.engine.engine_aware import SideEffecting from pants.engine.goal import Goal diff --git a/src/python/pants/engine/target.py b/src/python/pants/engine/target.py index 401d0315f98..c4de1b598f8 100644 --- a/src/python/pants/engine/target.py +++ b/src/python/pants/engine/target.py @@ -29,6 +29,7 @@ KeysView, Mapping, Optional, + Protocol, Sequence, Set, Tuple, @@ -39,7 +40,7 @@ get_type_hints, ) -from typing_extensions import Protocol, Self, final +from typing_extensions import Self, final from pants.base.deprecated import warn_or_error from pants.engine.addresses import Address, Addresses, UnparsedAddressInputs, assert_single_address diff --git a/src/python/pants/help/help_printer.py b/src/python/pants/help/help_printer.py index 3f3ec9e7365..a80d1b74a82 100644 --- a/src/python/pants/help/help_printer.py +++ b/src/python/pants/help/help_printer.py @@ -6,9 +6,7 @@ import re import textwrap from itertools import cycle -from typing import Callable, Dict, Iterable, List, Optional, Set, Tuple, cast - -from typing_extensions import Literal +from typing import Callable, Dict, Iterable, List, Literal, Optional, Set, Tuple, cast from pants.base.build_environment import pants_version from pants.help.help_formatter import HelpFormatter diff --git a/src/python/pants/option/config.py b/src/python/pants/option/config.py index b3ee236effb..180a2ca911a 100644 --- a/src/python/pants/option/config.py +++ b/src/python/pants/option/config.py @@ -8,10 +8,9 @@ import re from dataclasses import dataclass from types import SimpleNamespace -from typing import Any, Dict, Iterable, List, Mapping, Union, cast +from typing import Any, Dict, Iterable, List, Mapping, Protocol, Union, cast import toml -from typing_extensions import Protocol from pants.base.build_environment import get_buildroot from pants.option.errors import ConfigError, ConfigValidationError, InterpolationMissingOptionError diff --git a/src/python/pants/pantsd/pants_daemon_core.py b/src/python/pants/pantsd/pants_daemon_core.py index e4abdd04515..158be42d4c0 100644 --- a/src/python/pants/pantsd/pants_daemon_core.py +++ b/src/python/pants/pantsd/pants_daemon_core.py @@ -6,9 +6,7 @@ import logging import threading from contextlib import contextmanager -from typing import Iterator - -from typing_extensions import Protocol +from typing import Iterator, Protocol from pants.build_graph.build_configuration import BuildConfiguration from pants.engine.env_vars import CompleteEnvironmentVars diff --git a/src/python/pants/testutil/python_interpreter_selection.py b/src/python/pants/testutil/python_interpreter_selection.py index 04067ef10be..ff2e4b1ddd9 100644 --- a/src/python/pants/testutil/python_interpreter_selection.py +++ b/src/python/pants/testutil/python_interpreter_selection.py @@ -34,7 +34,7 @@ def has_python_version(version): return python_interpreter_path(version) is not None -@lru_cache() +@lru_cache def python_interpreter_path(version): """Returns the interpreter path if the current system has the specified version of python. diff --git a/src/python/pants/testutil/skip_utils.py b/src/python/pants/testutil/skip_utils.py index 9d65ef951a1..05742846ac6 100644 --- a/src/python/pants/testutil/skip_utils.py +++ b/src/python/pants/testutil/skip_utils.py @@ -7,7 +7,7 @@ import pytest -@lru_cache() +@lru_cache def skip_if_command_errors(*args: str): def empty_decorator(func): return func diff --git a/src/python/pants/util/dirutil.py b/src/python/pants/util/dirutil.py index bf0a6377b73..9a1d6c2bd82 100644 --- a/src/python/pants/util/dirutil.py +++ b/src/python/pants/util/dirutil.py @@ -14,9 +14,7 @@ from collections import defaultdict from contextlib import contextmanager from pathlib import Path -from typing import Any, Callable, DefaultDict, Iterable, Iterator, Sequence, overload - -from typing_extensions import Literal +from typing import Any, Callable, DefaultDict, Iterable, Iterator, Literal, Sequence, overload from pants.util.strutil import ensure_text diff --git a/src/python/pants_release/reversion.py b/src/python/pants_release/reversion.py index 02d76ab586d..a556fea2b35 100644 --- a/src/python/pants_release/reversion.py +++ b/src/python/pants_release/reversion.py @@ -48,9 +48,9 @@ def locate_dist_info_dir(workspace): dir_suffix = "*.dist-info" matches = glob.glob(os.path.join(workspace, dir_suffix)) if not matches: - raise Exception("Unable to locate `{}` directory in input whl.".format(dir_suffix)) + raise Exception(f"Unable to locate `{dir_suffix}` directory in input whl.") if len(matches) > 1: - raise Exception("Too many `{}` directories in input whl: {}".format(dir_suffix, matches)) + raise Exception(f"Too many `{dir_suffix}` directories in input whl: {matches}") return os.path.relpath(matches[0], workspace) @@ -81,9 +81,7 @@ def rewrite_record_file(workspace, src_record_file, mutated_file_tuples): else: mutated_files.add(dst) if not dst_record_file: - raise Exception( - "Malformed whl or bad globs: `{}` was not rewritten.".format(src_record_file) - ) + raise Exception(f"Malformed whl or bad globs: `{src_record_file}` was not rewritten.") output_records = [] file_name = os.path.join(workspace, dst_record_file) @@ -128,7 +126,7 @@ def reversion( input_version = mo.group("version") break if not input_version: - raise Exception("Could not find `Version:` line in {}".format(metadata_file)) + raise Exception(f"Could not find `Version:` line in {metadata_file}") # Rewrite and move all files (including the RECORD file), recording which files need to be # re-fingerprinted due to content changes. @@ -160,7 +158,7 @@ def reversion( os.mkdir(check_dst) subprocess.run(args=["wheel", "unpack", "-d", check_dst, tmp_whl_file], check=True) shutil.move(tmp_whl_file, dst_whl_file) - print("Wrote whl with version {} to {}.\n".format(target_version, dst_whl_file)) + print(f"Wrote whl with version {target_version} to {dst_whl_file}.\n") def create_parser() -> argparse.ArgumentParser: diff --git a/testprojects/src/python/coordinated_runs/waiter.py b/testprojects/src/python/coordinated_runs/waiter.py index 3c9c63225fe..88c5d74e123 100644 --- a/testprojects/src/python/coordinated_runs/waiter.py +++ b/testprojects/src/python/coordinated_runs/waiter.py @@ -35,7 +35,7 @@ def main(): if attempts <= 0: raise Exception("File was never written.") attempts -= 1 - sys.stderr.write("Waiting for file {}\n".format(waiting_for_file)) + sys.stderr.write(f"Waiting for file {waiting_for_file}\n") sys.stderr.flush() time.sleep(1) diff --git a/testprojects/src/python/hello/greet/greet.py b/testprojects/src/python/hello/greet/greet.py index 5549ab066e6..6a410f70d59 100644 --- a/testprojects/src/python/hello/greet/greet.py +++ b/testprojects/src/python/hello/greet/greet.py @@ -9,4 +9,4 @@ def greet(greetee): """Given the name, return a greeting for a person of that name.""" greeting = pkgutil.get_data(__name__, "greeting.txt").decode().strip() - return green("{}, {}!".format(greeting, greetee)) + return green(f"{greeting}, {greetee}!") From 6d8078b563ea4da57e48f4f0cdd22c9ed15679ca Mon Sep 17 00:00:00 2001 From: cburroughs Date: Sat, 30 Dec 2023 00:05:09 -0500 Subject: [PATCH 14/18] add grpcio-health-checking to default module mappings (#20295) See https://pypi.org/project/grpcio-health-checking --------- Co-authored-by: Tom Solberg --- .../python/dependency_inference/default_module_mapping.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/python/pants/backend/python/dependency_inference/default_module_mapping.py b/src/python/pants/backend/python/dependency_inference/default_module_mapping.py index 803fa3929e2..7e59bc46ed2 100644 --- a/src/python/pants/backend/python/dependency_inference/default_module_mapping.py +++ b/src/python/pants/backend/python/dependency_inference/default_module_mapping.py @@ -96,6 +96,7 @@ def two_groups_hyphens_two_replacements_with_suffix( "gitpython": ("git",), "graphql-core": ("graphql",), "grpcio": ("grpc",), + "grpcio-health-checking": ("grpc_health",), "honeycomb-opentelemetry": ("honeycomb.opentelemetry",), "ipython": ("IPython",), "jack-client": ("jack",), From 3ffbba355e7844a3c54edf0ac46075d766105d9a Mon Sep 17 00:00:00 2001 From: Gautham Nair Date: Fri, 29 Dec 2023 23:34:29 -0600 Subject: [PATCH 15/18] Refactor: DRY adhoc_tool code_quality_tool (#20255) Before moving to step 2 of the plan described in https://github.com/pantsbuild/pants/issues/17729#issuecomment-1787677211 , cleaning up a gross duplication of rule code that I introduced in https://github.com/pantsbuild/pants/pull/20135 between `adhoc_tool` and the new `code_quality_tool`. This PR extracts the shared logic into the concept of a ToolRunner and a rule to hydrate it in `adhoc_process_support`. Both `adhoc_tool` and `code_quality_tool` have the latent idea of a tool runner and a considerable machinery to build it. Starting from something like ```python @dataclass(frozen=True) class ToolRunnerRequest: runnable_address_str: str args: tuple[str, ...] execution_dependencies: tuple[str, ...] runnable_dependencies: tuple[str, ...] target: Target ``` they need to assemble things like locate the actual runnable by str and figure out what should be its base digest, args, env, etc. and also co-locate the execution and runnable dependencies. We now capture that information as a "runner": ```python @dataclass(frozen=True) class ToolRunner: digest: Digest args: tuple[str, ...] extra_env: Mapping[str, str] append_only_caches: Mapping[str, str] immutable_input_digests: Mapping[str, Digest] ``` After this, `adhoc_tool` and `code_quality_tool` diverge in what they do with it. `adhoc_tool` uses this runner to generate code and code_quality_tool uses it to run batches of lint/fmt/fix on source files. ## Food for thought ... It should not escape our attention that this `ToolRunner` could also be surfaced as a Target, to be used by `adhoc_tool` and `code_quality_tool` rather than each specifying all these fields together. It would also help to reduce confusion when handling all the kinds of 'dependencies' arguments that `adhoc_tool` takes. --- src/python/pants/backend/adhoc/adhoc_tool.py | 115 +++------------- .../pants/backend/adhoc/code_quality_tool.py | 126 +++--------------- .../core/util_rules/adhoc_process_support.py | 120 ++++++++++++++++- 3 files changed, 153 insertions(+), 208 deletions(-) diff --git a/src/python/pants/backend/adhoc/adhoc_tool.py b/src/python/pants/backend/adhoc/adhoc_tool.py index 2db3ec19dae..89cedc3a2be 100644 --- a/src/python/pants/backend/adhoc/adhoc_tool.py +++ b/src/python/pants/backend/adhoc/adhoc_tool.py @@ -20,31 +20,19 @@ AdhocToolStdoutFilenameField, AdhocToolWorkdirField, ) -from pants.build_graph.address import Address, AddressInput -from pants.core.goals.run import RunFieldSet, RunInSandboxRequest from pants.core.target_types import FileSourceField from pants.core.util_rules.adhoc_process_support import ( AdhocProcessRequest, AdhocProcessResult, - ExtraSandboxContents, - MergeExtraSandboxContents, - ResolvedExecutionDependencies, - ResolveExecutionDependenciesRequest, + ToolRunner, + ToolRunnerRequest, ) from pants.core.util_rules.adhoc_process_support import rules as adhoc_process_support_rules from pants.core.util_rules.environments import EnvironmentNameRequest -from pants.engine.addresses import Addresses from pants.engine.environment import EnvironmentName -from pants.engine.fs import Digest, MergeDigests, Snapshot -from pants.engine.internals.native_engine import EMPTY_DIGEST +from pants.engine.fs import Digest, Snapshot from pants.engine.rules import Get, collect_rules, rule -from pants.engine.target import ( - FieldSetsPerTarget, - FieldSetsPerTargetRequest, - GeneratedSources, - GenerateSourcesRequest, - Targets, -) +from pants.engine.target import GeneratedSources, GenerateSourcesRequest from pants.engine.unions import UnionRule from pants.util.frozendict import FrozenDict from pants.util.logging import LogLevel @@ -63,6 +51,7 @@ async def run_in_sandbox_request( ) -> GeneratedSources: target = request.protocol_target description = f"the `{target.alias}` at {target.address}" + environment_name = await Get( EnvironmentName, EnvironmentNameRequest, EnvironmentNameRequest.from_target(target) ) @@ -71,104 +60,38 @@ async def run_in_sandbox_request( if not runnable_address_str: raise Exception(f"Must supply a value for `runnable` for {description}.") - runnable_address = await Get( - Address, - AddressInput, - AddressInput.parse( - runnable_address_str, - relative_to=target.address.spec_path, - description_of_origin=f"The `{AdhocToolRunnableField.alias}` field of {description}", + tool_runner = await Get( + ToolRunner, + ToolRunnerRequest( + runnable_address_str=runnable_address_str, + args=target.get(AdhocToolArgumentsField).value or (), + execution_dependencies=target.get(AdhocToolExecutionDependenciesField).value or (), + runnable_dependencies=target.get(AdhocToolRunnableDependenciesField).value or (), + target=request.protocol_target, + named_caches=FrozenDict(target.get(AdhocToolNamedCachesField).value or {}), ), ) - addresses = Addresses((runnable_address,)) - addresses.expect_single() - - runnable_targets = await Get(Targets, Addresses, addresses) - field_sets = await Get( - FieldSetsPerTarget, FieldSetsPerTargetRequest(RunFieldSet, runnable_targets) - ) - run_field_set: RunFieldSet = field_sets.field_sets[0] - working_directory = target[AdhocToolWorkdirField].value or "" root_output_directory = target[AdhocToolOutputRootDirField].value or "" - # Must be run in target environment so that the binaries/envvars match the execution - # environment when we actually run the process. - run_request = await Get( - RunInSandboxRequest, {environment_name: EnvironmentName, run_field_set: RunFieldSet} - ) - - execution_environment = await Get( - ResolvedExecutionDependencies, - ResolveExecutionDependenciesRequest( - target.address, - target.get(AdhocToolExecutionDependenciesField).value, - target.get(AdhocToolRunnableDependenciesField).value, - ), - ) - dependencies_digest = execution_environment.digest - runnable_dependencies = execution_environment.runnable_dependencies - - extra_env: dict[str, str] = dict(run_request.extra_env or {}) - extra_path = extra_env.pop("PATH", None) - - extra_sandbox_contents = [] - - extra_sandbox_contents.append( - ExtraSandboxContents( - EMPTY_DIGEST, - extra_path, - run_request.immutable_input_digests or FrozenDict(), - run_request.append_only_caches or FrozenDict(), - run_request.extra_env or FrozenDict(), - ) - ) - - if runnable_dependencies: - extra_sandbox_contents.append( - ExtraSandboxContents( - EMPTY_DIGEST, - f"{{chroot}}/{runnable_dependencies.path_component}", - runnable_dependencies.immutable_input_digests, - runnable_dependencies.append_only_caches, - runnable_dependencies.extra_env, - ) - ) - - merged_extras = await Get( - ExtraSandboxContents, MergeExtraSandboxContents(tuple(extra_sandbox_contents)) - ) - extra_env = dict(merged_extras.extra_env) - if merged_extras.path: - extra_env["PATH"] = merged_extras.path - - input_digest = await Get(Digest, MergeDigests((dependencies_digest, run_request.digest))) - output_files = target.get(AdhocToolOutputFilesField).value or () output_directories = target.get(AdhocToolOutputDirectoriesField).value or () - extra_args = target.get(AdhocToolArgumentsField).value or () - - append_only_caches = { - **merged_extras.append_only_caches, - **(target.get(AdhocToolNamedCachesField).value or {}), - } - process_request = AdhocProcessRequest( description=description, address=target.address, working_directory=working_directory, root_output_directory=root_output_directory, - argv=tuple(run_request.args + extra_args), + argv=tool_runner.args, timeout=None, - input_digest=input_digest, - immutable_input_digests=FrozenDict.frozen(merged_extras.immutable_input_digests), - append_only_caches=FrozenDict(append_only_caches), + input_digest=tool_runner.digest, + immutable_input_digests=FrozenDict.frozen(tool_runner.immutable_input_digests), + append_only_caches=FrozenDict(tool_runner.append_only_caches), output_files=output_files, output_directories=output_directories, fetch_env_vars=target.get(AdhocToolExtraEnvVarsField).value or (), - supplied_env_var_values=FrozenDict(extra_env), + supplied_env_var_values=FrozenDict(tool_runner.extra_env), log_on_process_errors=None, log_output=target[AdhocToolLogOutputField].value, capture_stderr_file=target[AdhocToolStderrFilenameField].value, diff --git a/src/python/pants/backend/adhoc/code_quality_tool.py b/src/python/pants/backend/adhoc/code_quality_tool.py index 9a1c4c556a8..7de26a59c83 100644 --- a/src/python/pants/backend/adhoc/code_quality_tool.py +++ b/src/python/pants/backend/adhoc/code_quality_tool.py @@ -1,27 +1,18 @@ # Copyright 2023 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). from dataclasses import dataclass -from typing import ClassVar, Iterable, Mapping +from typing import ClassVar, Iterable from pants.core.goals.fix import Fix, FixFilesRequest, FixResult from pants.core.goals.fmt import Fmt, FmtFilesRequest, FmtResult from pants.core.goals.lint import Lint, LintFilesRequest, LintResult -from pants.core.goals.run import RunFieldSet, RunInSandboxRequest -from pants.core.util_rules.adhoc_process_support import ( - ExtraSandboxContents, - MergeExtraSandboxContents, - ResolvedExecutionDependencies, - ResolveExecutionDependenciesRequest, -) +from pants.core.util_rules.adhoc_process_support import ToolRunner, ToolRunnerRequest from pants.core.util_rules.adhoc_process_support import rules as adhoc_process_support_rules -from pants.core.util_rules.environments import EnvironmentNameRequest from pants.core.util_rules.partitions import Partitions from pants.engine.addresses import Addresses -from pants.engine.environment import EnvironmentName from pants.engine.fs import PathGlobs from pants.engine.goal import Goal from pants.engine.internals.native_engine import ( - EMPTY_DIGEST, Address, AddressInput, Digest, @@ -34,8 +25,6 @@ from pants.engine.rules import Rule, collect_rules, rule from pants.engine.target import ( COMMON_TARGET_FIELDS, - FieldSetsPerTarget, - FieldSetsPerTargetRequest, SpecialCasedDependencies, StringField, StringSequenceField, @@ -199,18 +188,9 @@ async def find_code_quality_tool(request: CodeQualityToolAddressString) -> CodeQ ) -@dataclass(frozen=True) -class CodeQualityToolBatchRunner: - digest: Digest - args: tuple[str, ...] - extra_env: Mapping[str, str] - append_only_caches: Mapping[str, str] - immutable_input_digests: Mapping[str, Digest] - - @dataclass(frozen=True) class CodeQualityToolBatch: - runner: CodeQualityToolBatchRunner + runner: ToolRunner sources_snapshot: Snapshot output_files: tuple[str, ...] @@ -237,91 +217,15 @@ async def process_files(batch: CodeQualityToolBatch) -> FallibleProcessResult: @rule -async def hydrate_code_quality_tool( - request: CodeQualityToolAddressString, -) -> CodeQualityToolBatchRunner: - cqt = await Get(CodeQualityTool, CodeQualityToolAddressString, request) - - runnable_address = await Get( - Address, - AddressInput, - AddressInput.parse( - cqt.runnable_address_str, - relative_to=cqt.target.address.spec_path, - description_of_origin=f"Runnable target for code quality tool {cqt.target.address.spec_path}", - ), - ) - - addresses = Addresses((runnable_address,)) - addresses.expect_single() - - runnable_targets = await Get(Targets, Addresses, addresses) - - target = runnable_targets[0] - - run_field_sets, environment_name, execution_environment = await MultiGet( - Get(FieldSetsPerTarget, FieldSetsPerTargetRequest(RunFieldSet, runnable_targets)), - Get(EnvironmentName, EnvironmentNameRequest, EnvironmentNameRequest.from_target(target)), - Get( - ResolvedExecutionDependencies, - ResolveExecutionDependenciesRequest( - address=runnable_address, - execution_dependencies=cqt.execution_dependencies, - runnable_dependencies=cqt.runnable_dependencies, - ), - ), - ) - - run_field_set: RunFieldSet = run_field_sets.field_sets[0] - - run_request = await Get( - RunInSandboxRequest, {environment_name: EnvironmentName, run_field_set: RunFieldSet} - ) - - dependencies_digest = execution_environment.digest - runnable_dependencies = execution_environment.runnable_dependencies - - extra_env: dict[str, str] = dict(run_request.extra_env or {}) - extra_path = extra_env.pop("PATH", None) - - extra_sandbox_contents = [] - - extra_sandbox_contents.append( - ExtraSandboxContents( - EMPTY_DIGEST, - extra_path, - run_request.immutable_input_digests or FrozenDict(), - run_request.append_only_caches or FrozenDict(), - run_request.extra_env or FrozenDict(), - ) - ) - - if runnable_dependencies: - extra_sandbox_contents.append( - ExtraSandboxContents( - EMPTY_DIGEST, - f"{{chroot}}/{runnable_dependencies.path_component}", - runnable_dependencies.immutable_input_digests, - runnable_dependencies.append_only_caches, - runnable_dependencies.extra_env, - ) - ) - - merged_extras, main_digest = await MultiGet( - Get(ExtraSandboxContents, MergeExtraSandboxContents(tuple(extra_sandbox_contents))), - Get(Digest, MergeDigests((dependencies_digest, run_request.digest))), - ) - - extra_env = dict(merged_extras.extra_env) - if merged_extras.path: - extra_env["PATH"] = merged_extras.path - - return CodeQualityToolBatchRunner( - digest=main_digest, - args=run_request.args + tuple(cqt.args), - extra_env=FrozenDict(extra_env), - append_only_caches=merged_extras.append_only_caches, - immutable_input_digests=merged_extras.immutable_input_digests, +async def runner_request_for_code_quality_tool( + cqt: CodeQualityTool, +) -> ToolRunnerRequest: + return ToolRunnerRequest( + runnable_address_str=cqt.runnable_address_str, + args=cqt.args, + execution_dependencies=cqt.execution_dependencies, + runnable_dependencies=cqt.runnable_dependencies, + target=cqt.target, ) @@ -391,7 +295,7 @@ async def partition_inputs( async def run_code_quality(request: CodeQualityProcessingRequest.Batch) -> LintResult: sources_snapshot, code_quality_tool_runner = await MultiGet( Get(Snapshot, PathGlobs(request.elements)), - Get(CodeQualityToolBatchRunner, CodeQualityToolAddressString(address=self.target)), + Get(ToolRunner, CodeQualityToolAddressString(address=self.target)), ) proc_result = await Get( @@ -445,7 +349,7 @@ async def run_code_quality(request: CodeQualityProcessingRequest.Batch) -> FmtRe sources_snapshot = request.snapshot code_quality_tool_runner = await Get( - CodeQualityToolBatchRunner, CodeQualityToolAddressString(address=self.target) + ToolRunner, CodeQualityToolAddressString(address=self.target) ) proc_result = await Get( @@ -507,7 +411,7 @@ async def run_code_quality(request: CodeQualityProcessingRequest.Batch) -> FixRe sources_snapshot = request.snapshot code_quality_tool_runner = await Get( - CodeQualityToolBatchRunner, CodeQualityToolAddressString(address=self.target) + ToolRunner, CodeQualityToolAddressString(address=self.target) ) proc_result = await Get( diff --git a/src/python/pants/core/util_rules/adhoc_process_support.py b/src/python/pants/core/util_rules/adhoc_process_support.py index 6da4ed57b5f..58d2e2f0a67 100644 --- a/src/python/pants/core/util_rules/adhoc_process_support.py +++ b/src/python/pants/core/util_rules/adhoc_process_support.py @@ -14,10 +14,12 @@ from pants.core.goals.package import BuiltPackage, EnvironmentAwarePackageRequest, PackageFieldSet from pants.core.goals.run import RunFieldSet, RunInSandboxRequest from pants.core.target_types import FileSourceField +from pants.core.util_rules.environments import EnvironmentNameRequest from pants.core.util_rules.source_files import SourceFiles, SourceFilesRequest from pants.core.util_rules.system_binaries import BashBinary from pants.engine.addresses import Addresses, UnparsedAddressInputs from pants.engine.env_vars import EnvironmentVars, EnvironmentVarsRequest +from pants.engine.environment import EnvironmentName from pants.engine.fs import ( EMPTY_DIGEST, CreateDigest, @@ -27,13 +29,14 @@ MergeDigests, Snapshot, ) -from pants.engine.internals.native_engine import RemovePrefix +from pants.engine.internals.native_engine import AddressInput, RemovePrefix from pants.engine.process import FallibleProcessResult, Process, ProcessResult, ProductDescription from pants.engine.rules import Get, MultiGet, collect_rules, rule from pants.engine.target import ( FieldSetsPerTarget, FieldSetsPerTargetRequest, SourcesField, + Target, Targets, TransitiveTargets, TransitiveTargetsRequest, @@ -91,6 +94,25 @@ class RunnableDependencies: extra_env: Mapping[str, str] +@dataclass(frozen=True) +class ToolRunnerRequest: + runnable_address_str: str + args: tuple[str, ...] + execution_dependencies: tuple[str, ...] + runnable_dependencies: tuple[str, ...] + target: Target + named_caches: FrozenDict[str, str] | None = None + + +@dataclass(frozen=True) +class ToolRunner: + digest: Digest + args: tuple[str, ...] + extra_env: FrozenDict[str, str] + append_only_caches: FrozenDict[str, str] + immutable_input_digests: FrozenDict[str, Digest] + + # # Things that need a home # @@ -360,6 +382,102 @@ def _quote(s: str) -> str: ).encode() +@rule +async def create_tool_runner( + request: ToolRunnerRequest, +) -> ToolRunner: + runnable_address = await Get( + Address, + AddressInput, + AddressInput.parse( + request.runnable_address_str, + relative_to=request.target.address.spec_path, + description_of_origin=f"Runnable target for {request.target.address.spec_path}", + ), + ) + + addresses = Addresses((runnable_address,)) + addresses.expect_single() + + runnable_targets = await Get(Targets, Addresses, addresses) + + run_field_sets, environment_name, execution_environment = await MultiGet( + Get(FieldSetsPerTarget, FieldSetsPerTargetRequest(RunFieldSet, runnable_targets)), + Get( + EnvironmentName, + EnvironmentNameRequest, + EnvironmentNameRequest.from_target(request.target), + ), + Get( + ResolvedExecutionDependencies, + ResolveExecutionDependenciesRequest( + address=runnable_address, + execution_dependencies=request.execution_dependencies, + runnable_dependencies=request.runnable_dependencies, + ), + ), + ) + + run_field_set: RunFieldSet = run_field_sets.field_sets[0] + + # Must be run in target environment so that the binaries/envvars match the execution + # environment when we actually run the process. + run_request = await Get( + RunInSandboxRequest, {environment_name: EnvironmentName, run_field_set: RunFieldSet} + ) + + dependencies_digest = execution_environment.digest + runnable_dependencies = execution_environment.runnable_dependencies + + extra_env: dict[str, str] = dict(run_request.extra_env or {}) + extra_path = extra_env.pop("PATH", None) + + extra_sandbox_contents = [] + + extra_sandbox_contents.append( + ExtraSandboxContents( + EMPTY_DIGEST, + extra_path, + run_request.immutable_input_digests or FrozenDict(), + run_request.append_only_caches or FrozenDict(), + run_request.extra_env or FrozenDict(), + ) + ) + + if runnable_dependencies: + extra_sandbox_contents.append( + ExtraSandboxContents( + EMPTY_DIGEST, + f"{{chroot}}/{runnable_dependencies.path_component}", + runnable_dependencies.immutable_input_digests, + runnable_dependencies.append_only_caches, + runnable_dependencies.extra_env, + ) + ) + + merged_extras, main_digest = await MultiGet( + Get(ExtraSandboxContents, MergeExtraSandboxContents(tuple(extra_sandbox_contents))), + Get(Digest, MergeDigests((dependencies_digest, run_request.digest))), + ) + + extra_env = dict(merged_extras.extra_env) + if merged_extras.path: + extra_env["PATH"] = merged_extras.path + + append_only_caches = { + **merged_extras.append_only_caches, + **(request.named_caches or {}), + } + + return ToolRunner( + digest=main_digest, + args=run_request.args + tuple(request.args), + extra_env=FrozenDict(extra_env), + append_only_caches=FrozenDict(append_only_caches), + immutable_input_digests=FrozenDict(merged_extras.immutable_input_digests), + ) + + @rule async def run_adhoc_process( request: AdhocProcessRequest, From edcafeb0cfb62d09c62d200c584badd340e0b8d8 Mon Sep 17 00:00:00 2001 From: "A. Alonso Dominguez" <2269440+alonsodomin@users.noreply.github.com> Date: Sat, 30 Dec 2023 16:53:46 +0100 Subject: [PATCH 16/18] Addresses TODOs in Java and Kotline parsers (#20350) --- .../java/dependency_inference/java_parser.py | 25 ++++++------------- .../dependency_inference/kotlin_parser.py | 25 ++++++------------- 2 files changed, 16 insertions(+), 34 deletions(-) diff --git a/src/python/pants/backend/java/dependency_inference/java_parser.py b/src/python/pants/backend/java/dependency_inference/java_parser.py index 0841f8c3fc3..dfc17ef14cb 100644 --- a/src/python/pants/backend/java/dependency_inference/java_parser.py +++ b/src/python/pants/backend/java/dependency_inference/java_parser.py @@ -15,13 +15,12 @@ from pants.core.util_rules.source_files import SourceFiles from pants.engine.fs import AddPrefix, CreateDigest, Digest, DigestContents, Directory, FileContent from pants.engine.internals.native_engine import MergeDigests, RemovePrefix -from pants.engine.process import FallibleProcessResult, ProcessExecutionFailure, ProcessResult +from pants.engine.process import FallibleProcessResult, ProcessResult, ProductDescription from pants.engine.rules import Get, MultiGet, collect_rules, rule from pants.engine.unions import UnionRule from pants.jvm.jdk_rules import InternalJdk, JvmProcess from pants.jvm.resolve.coursier_fetch import ToolClasspath, ToolClasspathRequest from pants.jvm.resolve.jvm_tool import GenerateJvmLockfileFromTool, GenerateJvmToolLockfileSentinel -from pants.option.global_options import KeepSandboxes from pants.util.logging import LogLevel from pants.util.ordered_set import FrozenOrderedSet @@ -53,23 +52,15 @@ class JavaParserCompiledClassfiles: @rule(level=LogLevel.DEBUG) async def resolve_fallible_result_to_analysis( fallible_result: FallibleJavaSourceDependencyAnalysisResult, - keep_sandboxes: KeepSandboxes, ) -> JavaSourceDependencyAnalysis: - # TODO(#12725): Just convert directly to a ProcessResult like this: - # result = await Get(ProcessResult, FallibleProcessResult, fallible_result.process_result) - if fallible_result.process_result.exit_code == 0: - analysis_contents = await Get( - DigestContents, Digest, fallible_result.process_result.output_digest - ) - analysis = json.loads(analysis_contents[0].content) - return JavaSourceDependencyAnalysis.from_json_dict(analysis) - raise ProcessExecutionFailure( - fallible_result.process_result.exit_code, - fallible_result.process_result.stdout, - fallible_result.process_result.stderr, - "Java source dependency analysis failed.", - keep_sandboxes=keep_sandboxes, + desc = ProductDescription("Java source dependency analysis failed.") + result = await Get( + ProcessResult, + {fallible_result.process_result: FallibleProcessResult, desc: ProductDescription}, ) + analysis_contents = await Get(DigestContents, Digest, result.output_digest) + analysis = json.loads(analysis_contents[0].content) + return JavaSourceDependencyAnalysis.from_json_dict(analysis) @rule(level=LogLevel.DEBUG) diff --git a/src/python/pants/backend/kotlin/dependency_inference/kotlin_parser.py b/src/python/pants/backend/kotlin/dependency_inference/kotlin_parser.py index cd75b8df751..8e7e4547d95 100644 --- a/src/python/pants/backend/kotlin/dependency_inference/kotlin_parser.py +++ b/src/python/pants/backend/kotlin/dependency_inference/kotlin_parser.py @@ -12,7 +12,7 @@ from pants.engine.fs import CreateDigest, DigestContents, Directory, FileContent from pants.engine.internals.native_engine import AddPrefix, Digest, MergeDigests, RemovePrefix from pants.engine.internals.selectors import Get, MultiGet -from pants.engine.process import FallibleProcessResult, ProcessExecutionFailure, ProcessResult +from pants.engine.process import FallibleProcessResult, ProcessResult, ProductDescription from pants.engine.rules import collect_rules, rule from pants.engine.unions import UnionRule from pants.jvm.compile import ClasspathEntry @@ -20,7 +20,6 @@ from pants.jvm.resolve.common import ArtifactRequirements, Coordinate from pants.jvm.resolve.coursier_fetch import ToolClasspath, ToolClasspathRequest from pants.jvm.resolve.jvm_tool import GenerateJvmLockfileFromTool, GenerateJvmToolLockfileSentinel -from pants.option.global_options import KeepSandboxes from pants.util.frozendict import FrozenDict from pants.util.logging import LogLevel from pants.util.ordered_set import FrozenOrderedSet @@ -210,23 +209,15 @@ async def analyze_kotlin_source_dependencies( @rule(level=LogLevel.DEBUG) async def resolve_fallible_result_to_analysis( fallible_result: FallibleKotlinSourceDependencyAnalysisResult, - keep_sandboxes: KeepSandboxes, ) -> KotlinSourceDependencyAnalysis: - # TODO(#12725): Just convert directly to a ProcessResult like this: - # result = await Get(ProcessResult, FallibleProcessResult, fallible_result.process_result) - if fallible_result.process_result.exit_code == 0: - analysis_contents = await Get( - DigestContents, Digest, fallible_result.process_result.output_digest - ) - analysis = json.loads(analysis_contents[0].content) - return KotlinSourceDependencyAnalysis.from_json_dict(analysis) - raise ProcessExecutionFailure( - fallible_result.process_result.exit_code, - fallible_result.process_result.stdout, - fallible_result.process_result.stderr, - "Kotlin source dependency analysis failed.", - keep_sandboxes=keep_sandboxes, + desc = ProductDescription("Kotlin source dependency analysis failed.") + result = await Get( + ProcessResult, + {fallible_result.process_result: FallibleProcessResult, desc: ProductDescription}, ) + analysis_contents = await Get(DigestContents, Digest, result.output_digest) + analysis = json.loads(analysis_contents[0].content) + return KotlinSourceDependencyAnalysis.from_json_dict(analysis) @rule From 08abad0fec74d1b74fd3bcea90c41397c49819c4 Mon Sep 17 00:00:00 2001 From: Stu Hood Date: Sat, 30 Dec 2023 21:36:06 -0800 Subject: [PATCH 17/18] Prepare 2.20.0.dev4 (#20351) --- src/python/pants/VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/python/pants/VERSION b/src/python/pants/VERSION index 2645c80bfb7..56e8bcf9be0 100644 --- a/src/python/pants/VERSION +++ b/src/python/pants/VERSION @@ -1 +1 @@ -2.20.0.dev3 +2.20.0.dev4 From 7a1cd54d98a43485027d2ac53099565900659811 Mon Sep 17 00:00:00 2001 From: Huon Wilson Date: Sun, 31 Dec 2023 20:48:23 +1100 Subject: [PATCH 18/18] Update all of Pants' unpinned Python deps (#20348) This runs `pants generate-lockfiles --resolve=python-default` to do a period/adhoc update of all of Pants' unpinned (transitive) deps. ``` Lockfile diff: 3rdparty/python/user_reqs.lock [python-default] == Upgraded dependencies == certifi 2023.7.22 --> 2023.11.17 charset-normalizer 3.3.0 --> 3.3.2 cryptography 41.0.4 --> 41.0.7 exceptiongroup 1.1.3 --> 1.2.0 idna 3.4 --> 3.6 pygments 2.16.1 --> 2.17.2 ujson 5.8.0 --> 5.9.0 urllib3 2.0.7 --> 2.1.0 uvloop 0.18.0 --> 0.19.0 websockets 11.0.3 --> 12.0 wrapt 1.15.0 --> 1.16.0 ``` NB. I haven't checked changelogs for new features or things to be aware of. --- 3rdparty/python/user_reqs.lock | 364 ++++++++++++++++----------------- 1 file changed, 180 insertions(+), 184 deletions(-) diff --git a/3rdparty/python/user_reqs.lock b/3rdparty/python/user_reqs.lock index 103c481abf4..042598fa517 100644 --- a/3rdparty/python/user_reqs.lock +++ b/3rdparty/python/user_reqs.lock @@ -223,19 +223,19 @@ "artifacts": [ { "algorithm": "sha256", - "hash": "92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9", - "url": "https://files.pythonhosted.org/packages/4c/dd/2234eab22353ffc7d94e8d13177aaa050113286e93e7b40eae01fbf7c3d9/certifi-2023.7.22-py3-none-any.whl" + "hash": "e036ab49d5b79556f99cfc2d9320b34cfbe5be05c5871b51de9329f0603b0474", + "url": "https://files.pythonhosted.org/packages/64/62/428ef076be88fa93716b576e4a01f919d25968913e817077a386fcbe4f42/certifi-2023.11.17-py3-none-any.whl" }, { "algorithm": "sha256", - "hash": "539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082", - "url": "https://files.pythonhosted.org/packages/98/98/c2ff18671db109c9f10ed27f5ef610ae05b73bd876664139cf95bd1429aa/certifi-2023.7.22.tar.gz" + "hash": "9b469f3a900bf28dc19b8cfbf8019bf47f7fdd1a65a1d4ffb98fc14166beb4d1", + "url": "https://files.pythonhosted.org/packages/d4/91/c89518dd4fe1f3a4e3f6ab7ff23cb00ef2e8c9adf99dacc618ad5e068e28/certifi-2023.11.17.tar.gz" } ], "project_name": "certifi", "requires_dists": [], "requires_python": ">=3.6", - "version": "2023.7.22" + "version": "2023.11.17" }, { "artifacts": [ @@ -301,84 +301,84 @@ "artifacts": [ { "algorithm": "sha256", - "hash": "e46cd37076971c1040fc8c41273a8b3e2c624ce4f2be3f5dfcb7a430c1d3acc2", - "url": "https://files.pythonhosted.org/packages/a3/dc/efab5b27839f04be4b8058c1eb85b7ab7dbc55ef8067250bea0518392756/charset_normalizer-3.3.0-py3-none-any.whl" + "hash": "3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc", + "url": "https://files.pythonhosted.org/packages/28/76/e6222113b83e3622caa4bb41032d0b1bf785250607392e1b778aca0b8a7d/charset_normalizer-3.3.2-py3-none-any.whl" }, { "algorithm": "sha256", - "hash": "aaf7b34c5bc56b38c931a54f7952f1ff0ae77a2e82496583b247f7c969eb1479", - "url": "https://files.pythonhosted.org/packages/0a/71/57bab2955a9da7a0282368824b7ed61c8f1a5e3cb8ffeba8d92185cc3a9a/charset_normalizer-3.3.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl" + "hash": "122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185", + "url": "https://files.pythonhosted.org/packages/1f/8d/33c860a7032da5b93382cbe2873261f81467e7b37f4ed91e25fed62fd49b/charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl" }, { "algorithm": "sha256", - "hash": "7966951325782121e67c81299a031f4c115615e68046f79b85856b86ebffc4cd", - "url": "https://files.pythonhosted.org/packages/0f/b8/e38e04920ac6481538893a0088b069e408dc40a52b1c0c9b2b36e0d697e7/charset_normalizer-3.3.0-cp39-cp39-musllinux_1_1_i686.whl" + "hash": "68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0", + "url": "https://files.pythonhosted.org/packages/2a/9d/a6d15bd1e3e2914af5955c8eb15f4071997e7078419328fee93dfd497eb7/charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl" }, { "algorithm": "sha256", - "hash": "09c77f964f351a7369cc343911e0df63e762e42bac24cd7d18525961c81754f4", - "url": "https://files.pythonhosted.org/packages/1f/36/90a7f37b056b581b592973b79fe44b58c392632f1efac7bcd4568bc59457/charset_normalizer-3.3.0-cp39-cp39-macosx_10_9_x86_64.whl" + "hash": "22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269", + "url": "https://files.pythonhosted.org/packages/3d/85/5b7416b349609d20611a64718bed383b9251b5a601044550f0c8983b8900/charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl" }, { "algorithm": "sha256", - "hash": "a0ac5e7015a5920cfce654c06618ec40c33e12801711da6b4258af59a8eff00a", - "url": "https://files.pythonhosted.org/packages/1f/9b/19f1788f2f4a393de85a4dc243f4c3707307f85b80734d285d8d116b42b1/charset_normalizer-3.3.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl" + "hash": "1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519", + "url": "https://files.pythonhosted.org/packages/44/80/b339237b4ce635b4af1c73742459eee5f97201bd92b2371c53e11958392e/charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl" }, { "algorithm": "sha256", - "hash": "805dfea4ca10411a5296bcc75638017215a93ffb584c9e344731eef0dcfb026a", - "url": "https://files.pythonhosted.org/packages/22/fe/b21d8a7e306baceb1c56835d3730d72f2818fc1092464dec1a0f5ce7ea63/charset_normalizer-3.3.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl" + "hash": "9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458", + "url": "https://files.pythonhosted.org/packages/51/fd/0ee5b1c2860bb3c60236d05b6e4ac240cf702b67471138571dad91bcfed8/charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl" }, { "algorithm": "sha256", - "hash": "e0fc42822278451bc13a2e8626cf2218ba570f27856b536e00cfa53099724828", - "url": "https://files.pythonhosted.org/packages/2e/8f/25c87f81417711d5055c9bb54244a7a321b50e1a692bdc2581918ff96e29/charset_normalizer-3.3.0-cp39-cp39-macosx_10_9_universal2.whl" + "hash": "34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8", + "url": "https://files.pythonhosted.org/packages/53/cd/aa4b8a4d82eeceb872f83237b2d27e43e637cac9ffaef19a1321c3bafb67/charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl" }, { "algorithm": "sha256", - "hash": "12ebea541c44fdc88ccb794a13fe861cc5e35d64ed689513a5c03d05b53b7c82", - "url": "https://files.pythonhosted.org/packages/61/d0/6518049da755ce7dc553170b19944aa186825fd7341459c9c298d89afb78/charset_normalizer-3.3.0-cp39-cp39-macosx_11_0_arm64.whl" + "hash": "ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561", + "url": "https://files.pythonhosted.org/packages/54/7f/cad0b328759630814fcf9d804bfabaf47776816ad4ef2e9938b7e1123d04/charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl" }, { "algorithm": "sha256", - "hash": "93aa7eef6ee71c629b51ef873991d6911b906d7312c6e8e99790c0f33c576f89", - "url": "https://files.pythonhosted.org/packages/65/e1/903bb63dc26f0ae0a5a0a603421dfbf4b6d8458f084ae037fa8864d0087a/charset_normalizer-3.3.0-cp39-cp39-musllinux_1_1_aarch64.whl" + "hash": "f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5", + "url": "https://files.pythonhosted.org/packages/63/09/c1bc53dab74b1816a00d8d030de5bf98f724c52c1635e07681d312f20be8/charset-normalizer-3.3.2.tar.gz" }, { "algorithm": "sha256", - "hash": "c2af80fb58f0f24b3f3adcb9148e6203fa67dd3f61c4af146ecad033024dde43", - "url": "https://files.pythonhosted.org/packages/86/19/8bdc186b6a06c33f36498fd5dd96088073e7b5f367dcd54f42b818097be2/charset_normalizer-3.3.0-cp39-cp39-musllinux_1_1_s390x.whl" + "hash": "5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d", + "url": "https://files.pythonhosted.org/packages/66/fe/c7d3da40a66a6bf2920cce0f436fa1f62ee28aaf92f412f0bf3b84c8ad6c/charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl" }, { "algorithm": "sha256", - "hash": "02673e456dc5ab13659f85196c534dc596d4ef260e4d86e856c3b2773ce09843", - "url": "https://files.pythonhosted.org/packages/94/e4/204960a9390cfd6d11f3385da7580ab660383e3900c38cf1ec55f009d756/charset_normalizer-3.3.0-cp39-cp39-musllinux_1_1_ppc64le.whl" + "hash": "e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c", + "url": "https://files.pythonhosted.org/packages/79/66/8946baa705c588521afe10b2d7967300e49380ded089a62d38537264aece/charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl" }, { "algorithm": "sha256", - "hash": "96c2b49eb6a72c0e4991d62406e365d87067ca14c1a729a870d22354e6f68115", - "url": "https://files.pythonhosted.org/packages/98/13/8e623974018097aa223c58002983a053c51f96f8ee9b79052463d021da4d/charset_normalizer-3.3.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl" + "hash": "b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796", + "url": "https://files.pythonhosted.org/packages/98/69/5d8751b4b670d623aa7a47bef061d69c279e9f922f6705147983aa76c3ce/charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" }, { "algorithm": "sha256", - "hash": "619d1c96099be5823db34fe89e2582b336b5b074a7f47f819d6b3a57ff7bdb86", - "url": "https://files.pythonhosted.org/packages/9e/45/824835a9c165eae015eb7b4a875a581918b9fc96439f8d9a5ca0868f0b7d/charset_normalizer-3.3.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" + "hash": "d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c", + "url": "https://files.pythonhosted.org/packages/c2/65/52aaf47b3dd616c11a19b1052ce7fa6321250a7a0b975f48d8c366733b9f/charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl" }, { "algorithm": "sha256", - "hash": "63563193aec44bce707e0c5ca64ff69fa72ed7cf34ce6e11d5127555756fd2f6", - "url": "https://files.pythonhosted.org/packages/cf/ac/e89b2f2f75f51e9859979b56d2ec162f7f893221975d244d8d5277aa9489/charset-normalizer-3.3.0.tar.gz" + "hash": "7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2", + "url": "https://files.pythonhosted.org/packages/e1/9c/60729bf15dc82e3aaf5f71e81686e42e50715a1399770bcde1a9e43d09db/charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl" }, { "algorithm": "sha256", - "hash": "153e7b6e724761741e0974fc4dcd406d35ba70b92bfe3fedcb497226c93b9da7", - "url": "https://files.pythonhosted.org/packages/f9/b6/6f7f5699bb81152eae18971702b6d18dc3f753380705339ab3f9071fc8f0/charset_normalizer-3.3.0-cp39-cp39-musllinux_1_1_x86_64.whl" + "hash": "c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4", + "url": "https://files.pythonhosted.org/packages/f7/9d/bcf4a449a438ed6f19790eee543a86a740c77508fbc5ddab210ab3ba3a9a/charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl" } ], "project_name": "charset-normalizer", "requires_dists": [], "requires_python": ">=3.7.0", - "version": "3.3.0" + "version": "3.3.2" }, { "artifacts": [ @@ -423,63 +423,63 @@ "artifacts": [ { "algorithm": "sha256", - "hash": "8ac4f9ead4bbd0bc8ab2d318f97d85147167a488be0e08814a37eb2f439d5cf6", - "url": "https://files.pythonhosted.org/packages/f9/ea/342ab02337bdc9f154f187114df73a0beac876c99b2f172aba816d966f66/cryptography-41.0.4-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl" + "hash": "c5ca78485a255e03c32b513f8c2bc39fedb7f5c5f8535545bdc223a03b24f248", + "url": "https://files.pythonhosted.org/packages/79/68/9767a3fb985515d3c34221c3671043cda57b1f691046ad8aae355fb2a8a5/cryptography-41.0.7-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl" }, { "algorithm": "sha256", - "hash": "80907d3faa55dc5434a16579952ac6da800935cd98d14dbd62f6f042c7f5e839", - "url": "https://files.pythonhosted.org/packages/06/5d/f992c40471b60b762dca2b118c0a7837e446bea917f2be54b8f49802fe5e/cryptography-41.0.4-cp37-abi3-macosx_10_12_universal2.whl" + "hash": "c7f3201ec47d5207841402594f1d7950879ef890c0c495052fa62f58283fde1a", + "url": "https://files.pythonhosted.org/packages/0d/bf/e7a1382034c4feaa77b35147138ff2bc8ae47a2fa7e2838fcdd41d2d0f2e/cryptography-41.0.7-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl" }, { "algorithm": "sha256", - "hash": "e40211b4923ba5a6dc9769eab704bdb3fbb58d56c5b336d30996c24fcf12aadb", - "url": "https://files.pythonhosted.org/packages/25/1d/f86ce362aedc580c3f90c0d74fa097289e3af9ba52b8d5a37369c186b0f1/cryptography-41.0.4-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" + "hash": "841df4caa01008bad253bce2a6f7b47f86dc9f08df4b433c404def869f590a15", + "url": "https://files.pythonhosted.org/packages/14/fd/dd5bd6ab0d12476ebca579cbfd48d31bd90fa28fa257b209df585dcf62a0/cryptography-41.0.7-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" }, { "algorithm": "sha256", - "hash": "0d9409894f495d465fe6fda92cb70e8323e9648af912d5b9141d616df40a87b8", - "url": "https://files.pythonhosted.org/packages/81/aa/e972c2a19d6207bda7ee077184b6fa8d6772fca40057b7ce099b691542f2/cryptography-41.0.4-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl" + "hash": "5429ec739a29df2e29e15d082f1d9ad683701f0ec7709ca479b3ff2708dae65a", + "url": "https://files.pythonhosted.org/packages/3e/81/ae2c51ea2b80d57d5756a12df67816230124faea0a762a7a6304fe3c819c/cryptography-41.0.7-cp37-abi3-manylinux_2_28_aarch64.whl" }, { "algorithm": "sha256", - "hash": "5a0f09cefded00e648a127048119f77bc2b2ec61e736660b5789e638f43cc397", - "url": "https://files.pythonhosted.org/packages/a2/5c/b821ad3b2f1506b8042500edfb671c30efb6eca7dc5aa63236342338669f/cryptography-41.0.4-cp37-abi3-musllinux_1_1_aarch64.whl" + "hash": "43f2552a2378b44869fe8827aa19e69512e3245a219104438692385b0ee119d1", + "url": "https://files.pythonhosted.org/packages/62/bd/69628ab50368b1beb900eb1de5c46f8137169b75b2458affe95f2f470501/cryptography-41.0.7-cp37-abi3-manylinux_2_28_x86_64.whl" }, { "algorithm": "sha256", - "hash": "23a25c09dfd0d9f28da2352503b23e086f8e78096b9fd585d1d14eca01613e13", - "url": "https://files.pythonhosted.org/packages/a2/d0/b8cf2c1367f850011d4618348760b23bb1268efba9e6ca03c063803e8763/cryptography-41.0.4-cp37-abi3-manylinux_2_28_aarch64.whl" + "hash": "5a1b41bc97f1ad230a41657d9155113c7521953869ae57ac39ac7f1bb471469a", + "url": "https://files.pythonhosted.org/packages/68/bb/475658ea92653a894589e657d6cea9ae01354db73405d62126ac5e74e2f8/cryptography-41.0.7-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl" }, { "algorithm": "sha256", - "hash": "9eeb77214afae972a00dee47382d2591abe77bdae166bda672fb1e24702a3860", - "url": "https://files.pythonhosted.org/packages/ae/8e/c2466577a0f29421a74c5e0c7731274c3f82504e0dd08a3ef0489822f0cd/cryptography-41.0.4-cp37-abi3-musllinux_1_1_x86_64.whl" + "hash": "928258ba5d6f8ae644e764d0f996d61a8777559f72dfeb2eea7e2fe0ad6e782d", + "url": "https://files.pythonhosted.org/packages/a9/76/d705397d076fcbf5671544eb72a70b5a5ac83462d23dbd2a365a3bf3692a/cryptography-41.0.7-cp37-abi3-macosx_10_12_x86_64.whl" }, { "algorithm": "sha256", - "hash": "c3391bd8e6de35f6f1140e50aaeb3e2b3d6a9012536ca23ab0d9c35ec18c8a91", - "url": "https://files.pythonhosted.org/packages/b0/e1/e72d4092537223101fbb3b496edc0c9434d349291716a57b908709db9594/cryptography-41.0.4-pp39-pypy39_pp73-macosx_10_12_x86_64.whl" + "hash": "af03b32695b24d85a75d40e1ba39ffe7db7ffcb099fe507b39fd41a565f1b157", + "url": "https://files.pythonhosted.org/packages/b6/4a/1808333c5ea79cb6d51102036cbcf698704b1fc7a5ccd139957aeadd2311/cryptography-41.0.7-cp37-abi3-musllinux_1_1_aarch64.whl" }, { "algorithm": "sha256", - "hash": "35c00f637cd0b9d5b6c6bd11b6c3359194a8eba9c46d4e875a3660e3b400005f", - "url": "https://files.pythonhosted.org/packages/bb/c1/e8ca19a3e9ac5c867efa6f23ce0b119ad00a16b6019e49a298b8c1fe6866/cryptography-41.0.4-cp37-abi3-macosx_10_12_x86_64.whl" + "hash": "48a0476626da912a44cc078f9893f292f0b3e4c739caf289268168d8f4702a39", + "url": "https://files.pythonhosted.org/packages/b9/19/75d3e8b9b814c09eef76899fea542473273311ab9bfaa1ca4e22c112e660/cryptography-41.0.7-pp39-pypy39_pp73-macosx_10_12_x86_64.whl" }, { "algorithm": "sha256", - "hash": "cecfefa17042941f94ab54f769c8ce0fe14beff2694e9ac684176a2535bf9714", - "url": "https://files.pythonhosted.org/packages/e2/b5/11bcc59ad5a7121fe1279a716f3e212f6b37ef993726b06c25aa3aefa0a7/cryptography-41.0.4-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl" + "hash": "49f0805fc0b2ac8d4882dd52f4a3b935b210935d500b6b805f321addc8177406", + "url": "https://files.pythonhosted.org/packages/c5/07/826d66b6b03c5bfde8b451bea22c41e68d60aafff0ffa02c5f0819844319/cryptography-41.0.7-cp37-abi3-musllinux_1_1_x86_64.whl" }, { "algorithm": "sha256", - "hash": "2ed09183922d66c4ec5fdaa59b4d14e105c084dd0febd27452de8f6f74704143", - "url": "https://files.pythonhosted.org/packages/eb/4b/f86cc66c632cf0948ca1712aadd255f624deef1cd371ea3bfd30851e188d/cryptography-41.0.4-cp37-abi3-manylinux_2_28_x86_64.whl" + "hash": "13f93ce9bea8016c253b34afc6bd6a75993e5c40672ed5405a9c832f0d4a00bc", + "url": "https://files.pythonhosted.org/packages/ce/b3/13a12ea7edb068de0f62bac88a8ffd92cc2901881b391839851846b84a81/cryptography-41.0.7.tar.gz" }, { "algorithm": "sha256", - "hash": "7febc3094125fc126a7f6fb1f420d0da639f3f32cb15c8ff0dc3997c4549f51a", - "url": "https://files.pythonhosted.org/packages/ef/33/87512644b788b00a250203382e40ee7040ae6fa6b4c4a31dcfeeaa26043b/cryptography-41.0.4.tar.gz" + "hash": "3c78451b78313fa81607fa1b3f1ae0a5ddd8014c38a02d9db0616133987b9cdf", + "url": "https://files.pythonhosted.org/packages/e4/73/5461318abd2fe426855a2f66775c063bbefd377729ece3c3ee048ddf19a5/cryptography-41.0.7-cp37-abi3-macosx_10_12_universal2.whl" } ], "project_name": "cryptography", @@ -505,7 +505,7 @@ "twine>=1.12.0; extra == \"docstest\"" ], "requires_python": ">=3.7", - "version": "41.0.4" + "version": "41.0.7" }, { "artifacts": [ @@ -564,13 +564,13 @@ "artifacts": [ { "algorithm": "sha256", - "hash": "343280667a4585d195ca1cf9cef84a4e178c4b6cf2274caef9859782b567d5e3", - "url": "https://files.pythonhosted.org/packages/ad/83/b71e58666f156a39fb29417e4c8ca4bc7400c0dd4ed9e8842ab54dc8c344/exceptiongroup-1.1.3-py3-none-any.whl" + "hash": "4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14", + "url": "https://files.pythonhosted.org/packages/b8/9a/5028fd52db10e600f1c4674441b968cf2ea4959085bfb5b99fb1250e5f68/exceptiongroup-1.2.0-py3-none-any.whl" }, { "algorithm": "sha256", - "hash": "097acd85d473d75af5bb98e41b61ff7fe35efe6675e4f9370ec6ec5126d160e9", - "url": "https://files.pythonhosted.org/packages/c2/e1/5561ad26f99b7779c28356f73f69a8b468ef491d0f6adf20d7ed0ac98ec1/exceptiongroup-1.1.3.tar.gz" + "hash": "91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68", + "url": "https://files.pythonhosted.org/packages/8e/1c/beef724eaf5b01bb44b6338c8c3494eff7cab376fab4904cfbbc3585dc79/exceptiongroup-1.2.0.tar.gz" } ], "project_name": "exceptiongroup", @@ -578,7 +578,7 @@ "pytest>=6; extra == \"test\"" ], "requires_python": ">=3.7", - "version": "1.1.3" + "version": "1.2.0" }, { "artifacts": [ @@ -772,19 +772,19 @@ "artifacts": [ { "algorithm": "sha256", - "hash": "90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2", - "url": "https://files.pythonhosted.org/packages/fc/34/3030de6f1370931b9dbb4dad48f6ab1015ab1d32447850b9fc94e60097be/idna-3.4-py3-none-any.whl" + "hash": "c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f", + "url": "https://files.pythonhosted.org/packages/c2/e7/a82b05cf63a603df6e68d59ae6a68bf5064484a0718ea5033660af4b54a9/idna-3.6-py3-none-any.whl" }, { "algorithm": "sha256", - "hash": "814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4", - "url": "https://files.pythonhosted.org/packages/8b/e1/43beb3d38dba6cb420cefa297822eac205a277ab43e5ba5d5c46faf96438/idna-3.4.tar.gz" + "hash": "9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca", + "url": "https://files.pythonhosted.org/packages/bf/3f/ea4b9117521a1e9c50344b909be7886dd00a519552724809bb1f486986c2/idna-3.6.tar.gz" } ], "project_name": "idna", "requires_dists": [], "requires_python": ">=3.5", - "version": "3.4" + "version": "3.6" }, { "artifacts": [ @@ -1112,21 +1112,22 @@ "artifacts": [ { "algorithm": "sha256", - "hash": "13fc09fa63bc8d8671a6d247e1eb303c4b343eaee81d861f3404db2935653692", - "url": "https://files.pythonhosted.org/packages/43/88/29adf0b44ba6ac85045e63734ae0997d3c58d8b1a91c914d240828d0d73d/Pygments-2.16.1-py3-none-any.whl" + "hash": "b27c2826c47d0f3219f29554824c30c5e8945175d888647acd804ddd04af846c", + "url": "https://files.pythonhosted.org/packages/97/9c/372fef8377a6e340b1704768d20daaded98bf13282b5327beb2e2fe2c7ef/pygments-2.17.2-py3-none-any.whl" }, { "algorithm": "sha256", - "hash": "1daff0494820c69bc8941e407aa20f577374ee88364ee10a98fdbe0aece96e29", - "url": "https://files.pythonhosted.org/packages/d6/f7/4d461ddf9c2bcd6a4d7b2b139267ca32a69439387cc1f02a924ff8883825/Pygments-2.16.1.tar.gz" + "hash": "da46cec9fd2de5be3a8a784f434e4c4ab670b4ff54d605c4c2717e9d49c4c367", + "url": "https://files.pythonhosted.org/packages/55/59/8bccf4157baf25e4aa5a0bb7fa3ba8600907de105ebc22b0c78cfbf6f565/pygments-2.17.2.tar.gz" } ], "project_name": "pygments", "requires_dists": [ + "colorama>=0.4.6; extra == \"windows-terminal\"", "importlib-metadata; python_version < \"3.8\" and extra == \"plugins\"" ], "requires_python": ">=3.7", - "version": "2.16.1" + "version": "2.17.2" }, { "artifacts": [ @@ -1868,102 +1869,97 @@ "artifacts": [ { "algorithm": "sha256", - "hash": "f504117a39cb98abba4153bf0b46b4954cc5d62f6351a14660201500ba31fe7f", - "url": "https://files.pythonhosted.org/packages/ab/70/898a7a82a4792089715ff5ed425a7f685b80b75bb511b4133edcb70a4403/ujson-5.8.0-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl" + "hash": "2a8ea0f55a1396708e564595aaa6696c0d8af532340f477162ff6927ecc46e21", + "url": "https://files.pythonhosted.org/packages/40/da/4eeda413bad5a5d3222076210283b1f2bb0fbf91c751702ad8361498c4ef/ujson-5.9.0-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl" }, { "algorithm": "sha256", - "hash": "407d60eb942c318482bbfb1e66be093308bb11617d41c613e33b4ce5be789adc", - "url": "https://files.pythonhosted.org/packages/0f/bf/32441baf63c8f04330c0927fec4ef59594ed6d3ac77fd00d8742f40cf764/ujson-5.8.0-cp39-cp39-musllinux_1_1_x86_64.whl" + "hash": "bdf7fc21a03bafe4ba208dafa84ae38e04e5d36c0e1c746726edf5392e9f9f36", + "url": "https://files.pythonhosted.org/packages/02/2d/4d4956140a1c92f06ef8aa1a62a8eb7e99dd2f7f32aa5d2e4a963a4bcf7c/ujson-5.9.0-cp39-cp39-macosx_11_0_arm64.whl" }, { "algorithm": "sha256", - "hash": "78e318def4ade898a461b3d92a79f9441e7e0e4d2ad5419abed4336d702c7425", - "url": "https://files.pythonhosted.org/packages/15/16/ff0a051f9a6e122f07630ed1e9cbe0e0b769273e123673f0d2aa17fe3a36/ujson-5.8.0.tar.gz" + "hash": "32bba5870c8fa2a97f4a68f6401038d3f1922e66c34280d710af00b14a3ca562", + "url": "https://files.pythonhosted.org/packages/0b/28/ddbd1f3e7b81be954961bc9c54d5b7594367a6fcd3362ffbd3822514d3b3/ujson-5.9.0-cp39-cp39-musllinux_1_1_aarch64.whl" }, { "algorithm": "sha256", - "hash": "fb87decf38cc82bcdea1d7511e73629e651bdec3a43ab40985167ab8449b769c", - "url": "https://files.pythonhosted.org/packages/2d/94/67960e910c66cab19c36f9b9dc9999fb89281da4534f751e77a669771512/ujson-5.8.0-cp39-cp39-musllinux_1_1_i686.whl" + "hash": "e2f909bc08ce01f122fd9c24bc6f9876aa087188dfaf3c4116fe6e4daf7e194f", + "url": "https://files.pythonhosted.org/packages/22/fb/e5531dd0d0de2d5d1aff2e6a0b78299f2f9b611d2cd67954c1dfe064aae6/ujson-5.9.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl" }, { "algorithm": "sha256", - "hash": "d6f84a7a175c75beecde53a624881ff618e9433045a69fcfb5e154b73cdaa377", - "url": "https://files.pythonhosted.org/packages/38/c7/2088ea60e55ee8e98ac2b6189649b35c76a2e0d55e832c307017576aea95/ujson-5.8.0-pp39-pypy39_pp73-macosx_10_9_x86_64.whl" + "hash": "f91719c6abafe429c1a144cfe27883eace9fb1c09a9c5ef1bcb3ae80a3076a4e", + "url": "https://files.pythonhosted.org/packages/35/84/e8ef8d94e18182ecf75949d04406b5ba1433b2fe9cd9b83cc6fae4d30182/ujson-5.9.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl" }, { "algorithm": "sha256", - "hash": "b748797131ac7b29826d1524db1cc366d2722ab7afacc2ce1287cdafccddbf1f", - "url": "https://files.pythonhosted.org/packages/a0/bb/6a1f0e0ec003800402a722511633d9dead569f2050eeef8d20716bedf9b6/ujson-5.8.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl" + "hash": "f69f16b8f1c69da00e38dc5f2d08a86b0e781d0ad3e4cc6a13ea033a439c4844", + "url": "https://files.pythonhosted.org/packages/37/70/f7a455225de729763c4cd34b06828bbb08478b39bb1409be0b5ec416d8a5/ujson-5.9.0-cp39-cp39-musllinux_1_1_x86_64.whl" }, { "algorithm": "sha256", - "hash": "2e72ba76313d48a1a3a42e7dc9d1db32ea93fac782ad8dde6f8b13e35c229130", - "url": "https://files.pythonhosted.org/packages/a3/30/12ba1b8e54f7869617e1f57beecfcaa304e1c093650003f0e38bf516a5a3/ujson-5.8.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" + "hash": "473fb8dff1d58f49912323d7cb0859df5585cfc932e4b9c053bf8cf7f2d7c5c4", + "url": "https://files.pythonhosted.org/packages/3c/30/950218fb10fb6c9dd3b50ac6f922805827885fdf358748c2f0aa4a76df1d/ujson-5.9.0-pp39-pypy39_pp73-macosx_10_9_x86_64.whl" }, { "algorithm": "sha256", - "hash": "f3554eaadffe416c6f543af442066afa6549edbc34fe6a7719818c3e72ebfe95", - "url": "https://files.pythonhosted.org/packages/ac/c6/11cecc6e72121af011462667761142364d7d7691459c0ad29f5abe8296b8/ujson-5.8.0-cp39-cp39-musllinux_1_1_aarch64.whl" + "hash": "63fb2e6599d96fdffdb553af0ed3f76b85fda63281063f1cb5b1141a6fcd0617", + "url": "https://files.pythonhosted.org/packages/49/64/c563bc163154714a128a7e7403bc3df5e826e8936bf1f5ef602c19626eed/ujson-5.9.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl" }, { "algorithm": "sha256", - "hash": "69b3104a2603bab510497ceabc186ba40fef38ec731c0ccaa662e01ff94a985c", - "url": "https://files.pythonhosted.org/packages/b1/ab/ba7ccd41bcc13a1bb5c8f680b0aa935eec668ce38b45e39b500f34068e53/ujson-5.8.0-cp39-cp39-macosx_10_9_x86_64.whl" + "hash": "37ef92e42535a81bf72179d0e252c9af42a4ed966dc6be6967ebfb929a87bc60", + "url": "https://files.pythonhosted.org/packages/50/4f/9541c36bc1342dbea0853d6e75b91094f44f1e5709bca3c16e1a35f6bf84/ujson-5.9.0-cp39-cp39-musllinux_1_1_i686.whl" }, { "algorithm": "sha256", - "hash": "7a42baa647a50fa8bed53d4e242be61023bd37b93577f27f90ffe521ac9dc7a3", - "url": "https://files.pythonhosted.org/packages/ca/29/ab7a93b6304c20a847e0046d090d103d827ab4b108a1cd235a76adc9e94e/ujson-5.8.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl" + "hash": "89cc92e73d5501b8a7f48575eeb14ad27156ad092c2e9fc7e3cf949f07e75532", + "url": "https://files.pythonhosted.org/packages/6e/54/6f2bdac7117e89a47de4511c9f01732a283457ab1bf856e1e51aa861619e/ujson-5.9.0.tar.gz" }, { "algorithm": "sha256", - "hash": "9249fdefeb021e00b46025e77feed89cd91ffe9b3a49415239103fc1d5d9c29a", - "url": "https://files.pythonhosted.org/packages/e2/a5/3e4a004c2626340b6149d74dd529027d7166cfd86cadd27decf8480ac149/ujson-5.8.0-cp39-cp39-macosx_11_0_arm64.whl" + "hash": "7b1c0991c4fe256f5fdb19758f7eac7f47caac29a6c57d0de16a19048eb86bad", + "url": "https://files.pythonhosted.org/packages/84/79/e8751f45fe1b9da65f48888dd1f15d9244f667d4d1d9293a4a092d0dd7bf/ujson-5.9.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" }, { "algorithm": "sha256", - "hash": "2873d196725a8193f56dde527b322c4bc79ed97cd60f1d087826ac3290cf9207", - "url": "https://files.pythonhosted.org/packages/e3/82/7019db84bfa1833e954b64450c18a6226c3e9847298e1bf2d99ffb0502d4/ujson-5.8.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl" + "hash": "d0fd2eba664a22447102062814bd13e63c6130540222c0aa620701dd01f4be81", + "url": "https://files.pythonhosted.org/packages/b2/2c/4500b6c1e99e01e2a902ddd8a14d0972d18c05f670c42a64ed65c6361eee/ujson-5.9.0-cp39-cp39-macosx_10_9_x86_64.whl" }, { "algorithm": "sha256", - "hash": "6a4dafa9010c366589f55afb0fd67084acd8added1a51251008f9ff2c3e44042", - "url": "https://files.pythonhosted.org/packages/ed/2f/04fb635a03e11630ae8fd0dff8617442251a4845b7622e359fdf1256e172/ujson-5.8.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" + "hash": "bd4ea86c2afd41429751d22a3ccd03311c067bd6aeee2d054f83f97e41e11d8f", + "url": "https://files.pythonhosted.org/packages/bd/39/bacd7004191d2d9bc8aaf0af102cbc761ab2af7dca649df67888041f84cd/ujson-5.9.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" } ], "project_name": "ujson", "requires_dists": [], "requires_python": ">=3.8", - "version": "5.8.0" + "version": "5.9.0" }, { "artifacts": [ { "algorithm": "sha256", - "hash": "fdb6d215c776278489906c2f8916e6e7d4f5a9b602ccbcfdf7f016fc8da0596e", - "url": "https://files.pythonhosted.org/packages/d2/b2/b157855192a68541a91ba7b2bbcb91f1b4faa51f8bae38d8005c034be524/urllib3-2.0.7-py3-none-any.whl" + "hash": "55901e917a5896a349ff771be919f8bd99aff50b79fe58fec595eb37bbc56bb3", + "url": "https://files.pythonhosted.org/packages/96/94/c31f58c7a7f470d5665935262ebd7455c7e4c7782eb525658d3dbf4b9403/urllib3-2.1.0-py3-none-any.whl" }, { "algorithm": "sha256", - "hash": "c97dfde1f7bd43a71c8d2a58e369e9b2bf692d1334ea9f9cae55add7d0dd0f84", - "url": "https://files.pythonhosted.org/packages/af/47/b215df9f71b4fdba1025fc05a77db2ad243fa0926755a52c5e71659f4e3c/urllib3-2.0.7.tar.gz" + "hash": "df7aa8afb0148fa78488e7899b2c59b5f4ffcfa82e6c54ccb9dd37c1d7b52d54", + "url": "https://files.pythonhosted.org/packages/36/dd/a6b232f449e1bc71802a5b7950dc3675d32c6dbc2a1bd6d71f065551adb6/urllib3-2.1.0.tar.gz" } ], "project_name": "urllib3", "requires_dists": [ "brotli>=1.0.9; platform_python_implementation == \"CPython\" and extra == \"brotli\"", "brotlicffi>=0.8.0; platform_python_implementation != \"CPython\" and extra == \"brotli\"", - "certifi; extra == \"secure\"", - "cryptography>=1.9; extra == \"secure\"", - "idna>=2.0.0; extra == \"secure\"", - "pyopenssl>=17.1.0; extra == \"secure\"", "pysocks!=1.5.7,<2.0,>=1.5.6; extra == \"socks\"", - "urllib3-secure-extra; extra == \"secure\"", "zstandard>=0.18.0; extra == \"zstd\"" ], - "requires_python": ">=3.7", - "version": "2.0.7" + "requires_python": ">=3.8", + "version": "2.1.0" }, { "artifacts": [ @@ -1999,38 +1995,38 @@ "artifacts": [ { "algorithm": "sha256", - "hash": "db1fcbad5deb9551e011ca589c5e7258b5afa78598174ac37a5f15ddcfb4ac7b", - "url": "https://files.pythonhosted.org/packages/27/92/eecef4bf7c98747b8f9051ec6fc1165c525ce5c8ec189b1d97d9259954ff/uvloop-0.18.0-cp39-cp39-musllinux_1_1_x86_64.whl" + "hash": "2df95fca285a9f5bfe730e51945ffe2fa71ccbfdde3b0da5772b4ee4f2e770d5", + "url": "https://files.pythonhosted.org/packages/12/9d/f1d263d49f1909914bcec5d5608d2f819c109b08bf06e67a2ff072e82d81/uvloop-0.19.0-cp39-cp39-musllinux_1_1_x86_64.whl" }, { "algorithm": "sha256", - "hash": "4d90858f32a852988d33987d608bcfba92a1874eb9f183995def59a34229f30d", - "url": "https://files.pythonhosted.org/packages/25/89/536d4ef50827f714048188eec1e32a2876bc19eac272be5bb3770fe3395e/uvloop-0.18.0-cp39-cp39-musllinux_1_1_aarch64.whl" + "hash": "f467a5fd23b4fc43ed86342641f3936a68ded707f4627622fa3f82a120e18256", + "url": "https://files.pythonhosted.org/packages/04/58/4d12d24220f2bf2c3125c74431035ddd7a461f9732a01cd5bd12f177370e/uvloop-0.19.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" }, { "algorithm": "sha256", - "hash": "d5d1135beffe9cd95d0350f19e2716bc38be47d5df296d7cc46e3b7557c0d1ff", - "url": "https://files.pythonhosted.org/packages/80/f9/94d2d914d351c7d5db80e102fb0d7ab3bbb798e8322ab71a9fe9f8bfa31b/uvloop-0.18.0.tar.gz" + "hash": "6e3d4e85ac060e2342ff85e90d0c04157acb210b9ce508e784a944f852a40e67", + "url": "https://files.pythonhosted.org/packages/0f/7f/6497008441376686f962a57de57897654ebd9c80f993b619ab57bc4ae61d/uvloop-0.19.0-cp39-cp39-macosx_10_9_x86_64.whl" }, { "algorithm": "sha256", - "hash": "c6d341bc109fb8ea69025b3ec281fcb155d6824a8ebf5486c989ff7748351a37", - "url": "https://files.pythonhosted.org/packages/b7/4f/335fa1a3bba326a9f8a6462701d8b21ec2103b0fc29c9d0b63be4f7bcb56/uvloop-0.18.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl" + "hash": "13dfdf492af0aa0a0edf66807d2b465607d11c4fa48f4a1fd41cbea5b18e8e8b", + "url": "https://files.pythonhosted.org/packages/1e/f9/8de4d58175c7a5cf3731fcfc43e7fb93e0972a098bffdc926507f02cd655/uvloop-0.19.0-cp39-cp39-macosx_10_9_universal2.whl" }, { "algorithm": "sha256", - "hash": "895a1e3aca2504638a802d0bec2759acc2f43a0291a1dff886d69f8b7baff399", - "url": "https://files.pythonhosted.org/packages/ca/f1/071710bbd5cd1b727b3b34dd4bf6ad9477e5cbd52be344bcd29f899abfe0/uvloop-0.18.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" + "hash": "492e2c32c2af3f971473bc22f086513cedfc66a130756145a931a90c3958cb17", + "url": "https://files.pythonhosted.org/packages/84/8d/3e23cb85cc2c12918a6d7585fdf50a05c69191a4969881e22e0eebcbf686/uvloop-0.19.0-cp39-cp39-musllinux_1_1_aarch64.whl" }, { "algorithm": "sha256", - "hash": "f3b18663efe0012bc4c315f1b64020e44596f5fabc281f5b0d9bc9465288559c", - "url": "https://files.pythonhosted.org/packages/ce/03/f112c3898e3afdbd7b734dee680540605f6e0082a12f33b07c9769e10346/uvloop-0.18.0-cp39-cp39-macosx_10_9_x86_64.whl" + "hash": "0246f4fd1bf2bf702e06b0d45ee91677ee5c31242f39aab4ea6fe0c51aedd0fd", + "url": "https://files.pythonhosted.org/packages/9c/16/728cc5dde368e6eddb299c5aec4d10eaf25335a5af04e8c0abd68e2e9d32/uvloop-0.19.0.tar.gz" }, { "algorithm": "sha256", - "hash": "e14de8800765b9916d051707f62e18a304cde661fa2b98a58816ca38d2b94029", - "url": "https://files.pythonhosted.org/packages/e5/04/35da196ec6750b64b20c647639c0c5af1577a06c4218733898999f49f515/uvloop-0.18.0-cp39-cp39-macosx_10_9_universal2.whl" + "hash": "8ca4956c9ab567d87d59d49fa3704cf29e37109ad348f2d5223c9bf761a332e7", + "url": "https://files.pythonhosted.org/packages/fe/4d/199e8c6e4a810b60cc012f9dc34fcf4df0e93d927de75ce4ed54a4d36274/uvloop-0.19.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl" } ], "project_name": "uvloop", @@ -2047,8 +2043,8 @@ "sphinx-rtd-theme~=0.5.2; extra == \"docs\"", "sphinxcontrib-asyncio~=0.3.0; extra == \"docs\"" ], - "requires_python": ">=3.7.0", - "version": "0.18.0" + "requires_python": ">=3.8.0", + "version": "0.19.0" }, { "artifacts": [ @@ -2074,142 +2070,142 @@ "artifacts": [ { "algorithm": "sha256", - "hash": "6681ba9e7f8f3b19440921e99efbb40fc89f26cd71bf539e45d8c8a25c976dc6", - "url": "https://files.pythonhosted.org/packages/47/96/9d5749106ff57629b54360664ae7eb9afd8302fad1680ead385383e33746/websockets-11.0.3-py3-none-any.whl" + "hash": "dc284bbc8d7c78a6c69e0c7325ab46ee5e40bb4d50e494d8131a07ef47500e9e", + "url": "https://files.pythonhosted.org/packages/79/4d/9cc401e7b07e80532ebc8c8e993f42541534da9e9249c59ee0139dcb0352/websockets-12.0-py3-none-any.whl" }, { "algorithm": "sha256", - "hash": "0ee68fe502f9031f19d495dae2c268830df2760c0524cbac5d759921ba8c8e82", - "url": "https://files.pythonhosted.org/packages/1b/3d/3dc77699fa4d003f2e810c321592f80f62b81d7b78483509de72ffe581fd/websockets-11.0.3-pp39-pypy39_pp73-macosx_10_9_x86_64.whl" + "hash": "00700340c6c7ab788f176d118775202aadea7602c5cc6be6ae127761c16d6b0b", + "url": "https://files.pythonhosted.org/packages/01/ae/d48aebf121726d2a26e48170cd7558627b09e0d47186ddfa1be017c81663/websockets-12.0-pp39-pypy39_pp73-macosx_10_9_x86_64.whl" }, { "algorithm": "sha256", - "hash": "b67c6f5e5a401fc56394f191f00f9b3811fe843ee93f4a70df3c389d1adf857d", - "url": "https://files.pythonhosted.org/packages/32/2c/ab8ea64e9a7d8bf62a7ea7a037fb8d328d8bd46dbfe083787a9d452a148e/websockets-11.0.3-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl" + "hash": "ffefa1374cd508d633646d51a8e9277763a9b78ae71324183693959cf94635a7", + "url": "https://files.pythonhosted.org/packages/03/72/e4752b208241a606625da8d8757d98c3bfc6c69c0edc47603180c208f857/websockets-12.0-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl" }, { "algorithm": "sha256", - "hash": "df41b9bc27c2c25b486bae7cf42fccdc52ff181c8c387bfd026624a491c2671b", - "url": "https://files.pythonhosted.org/packages/66/89/799f595c67b97a8a17e13d2764e088f631616bd95668aaa4c04b7cada136/websockets-11.0.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl" + "hash": "46e71dbbd12850224243f5d2aeec90f0aaa0f2dde5aeeb8fc8df21e04d99eff9", + "url": "https://files.pythonhosted.org/packages/06/dd/e8535f54b4aaded1ed44041ca8eb9de8786ce719ff148b56b4a903ef93e6/websockets-12.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl" }, { "algorithm": "sha256", - "hash": "97b52894d948d2f6ea480171a27122d77af14ced35f62e5c892ca2fae9344311", - "url": "https://files.pythonhosted.org/packages/72/89/0d150939f2e592ed78c071d69237ac1c872462cc62a750c5f592f3d4ab18/websockets-11.0.3-cp39-cp39-musllinux_1_1_x86_64.whl" + "hash": "1df2fbd2c8a98d38a66f5238484405b8d1d16f929bb7a33ed73e4801222a6f53", + "url": "https://files.pythonhosted.org/packages/0d/a4/ec1043bc6acf5bc405762ecc1327f3573441185571122ae50fc00c6d3130/websockets-12.0-cp39-cp39-macosx_11_0_arm64.whl" }, { "algorithm": "sha256", - "hash": "69269f3a0b472e91125b503d3c0b3566bda26da0a3261c49f0027eb6075086d1", - "url": "https://files.pythonhosted.org/packages/8a/77/a04d2911f6e2b9e781ce7ffc1e8516b54b85f985369eec8c853fd619d8e8/websockets-11.0.3-cp39-cp39-musllinux_1_1_i686.whl" + "hash": "a02413bc474feda2849c59ed2dfb2cddb4cd3d2f03a2fedec51d6e959d9b608b", + "url": "https://files.pythonhosted.org/packages/1b/9f/84d42c8c3e510f2a9ad09ae178c31cc89cc838b143a04bf41ff0653ca018/websockets-12.0-cp39-cp39-musllinux_1_1_i686.whl" }, { "algorithm": "sha256", - "hash": "1d5023a4b6a5b183dc838808087033ec5df77580485fc533e7dab2567851b0a4", - "url": "https://files.pythonhosted.org/packages/8b/97/34178f5f7c29e679372d597cebfeff2aa45991d741d938117d4616e81a74/websockets-11.0.3-pp39-pypy39_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl" + "hash": "23509452b3bc38e3a057382c2e941d5ac2e01e251acce7adc74011d7d8de434c", + "url": "https://files.pythonhosted.org/packages/25/a9/a3e03f9f3c4425a914e5875dd09f2c2559d61b44edd52cf1e6b73f938898/websockets-12.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl" }, { "algorithm": "sha256", - "hash": "8c82f11964f010053e13daafdc7154ce7385ecc538989a354ccc7067fd7028fd", - "url": "https://files.pythonhosted.org/packages/8f/f2/8a3eb016be19743c7eb9e67c855df0fdfa5912534ffaf83a05b62667d761/websockets-11.0.3-cp39-cp39-macosx_10_9_x86_64.whl" + "hash": "ba0cab91b3956dfa9f512147860783a1829a8d905ee218a9837c18f683239611", + "url": "https://files.pythonhosted.org/packages/2d/73/a337e1275e4c3a9752896fbe467d2c6b5f25e983a2de0992e1dfaca04dbe/websockets-12.0-pp39-pypy39_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl" }, { "algorithm": "sha256", - "hash": "3580dd9c1ad0701169e4d6fc41e878ffe05e6bdcaf3c412f9d559389d0c9e016", - "url": "https://files.pythonhosted.org/packages/a0/1a/3da73e69ebc00649d11ed836541c92c1a2df0b8a8aa641a2c8746e7c2b9c/websockets-11.0.3-cp39-cp39-macosx_11_0_arm64.whl" + "hash": "81df9cbcbb6c260de1e007e58c011bfebe2dafc8435107b0537f393dd38c8b1b", + "url": "https://files.pythonhosted.org/packages/2e/62/7a7874b7285413c954a4cca3c11fd851f11b2fe5b4ae2d9bee4f6d9bdb10/websockets-12.0.tar.gz" }, { "algorithm": "sha256", - "hash": "dcacf2c7a6c3a84e720d1bb2b543c675bf6c40e460300b628bab1b1efc7c034c", - "url": "https://files.pythonhosted.org/packages/a6/1b/5c83c40f8d3efaf0bb2fdf05af94fb920f74842b7aaf31d7598e3ee44d58/websockets-11.0.3-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl" + "hash": "b81f90dcc6c85a9b7f29873beb56c94c85d6f0dac2ea8b60d995bd18bf3e2aae", + "url": "https://files.pythonhosted.org/packages/67/cc/6fd14e45c5149e6c81c6771550ee5a4911321014e620f69baf1490001a80/websockets-12.0-cp39-cp39-musllinux_1_1_aarch64.whl" }, { "algorithm": "sha256", - "hash": "279e5de4671e79a9ac877427f4ac4ce93751b8823f276b681d04b2156713b9dd", - "url": "https://files.pythonhosted.org/packages/a6/9c/2356ecb952fd3992b73f7a897d65e57d784a69b94bb8d8fd5f97531e5c02/websockets-11.0.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl" + "hash": "ab3d732ad50a4fbd04a4490ef08acd0517b6ae6b77eb967251f4c263011a990d", + "url": "https://files.pythonhosted.org/packages/69/af/c52981023e7afcdfdb50c4697f702659b3dedca54f71e3cc99b8581f5647/websockets-12.0-cp39-cp39-macosx_10_9_universal2.whl" }, { "algorithm": "sha256", - "hash": "777354ee16f02f643a4c7f2b3eff8027a33c9861edc691a2003531f5da4f6bc8", - "url": "https://files.pythonhosted.org/packages/c0/21/cb9dfbbea8dc0ad89ced52630e7e61edb425fb9fdc6002f8d0c5dd26b94b/websockets-11.0.3-cp39-cp39-macosx_10_9_universal2.whl" + "hash": "2e5fc14ec6ea568200ea4ef46545073da81900a2b67b3e666f04adf53ad452ec", + "url": "https://files.pythonhosted.org/packages/7b/9f/f5aae5c49b0fc04ca68c723386f0d97f17363384525c6566cd382912a022/websockets-12.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl" }, { "algorithm": "sha256", - "hash": "1fdf26fa8a6a592f8f9235285b8affa72748dc12e964a5518c6c5e8f916716f7", - "url": "https://files.pythonhosted.org/packages/c4/f5/15998b164c183af0513bba744b51ecb08d396ff86c0db3b55d62624d1f15/websockets-11.0.3-cp39-cp39-musllinux_1_1_aarch64.whl" + "hash": "bbe6013f9f791944ed31ca08b077e26249309639313fff132bfbf3ba105673b9", + "url": "https://files.pythonhosted.org/packages/9c/5b/648db3556d8a441aa9705e1132b3ddae76204b57410952f85cf4a953623a/websockets-12.0-cp39-cp39-musllinux_1_1_x86_64.whl" }, { "algorithm": "sha256", - "hash": "88fc51d9a26b10fc331be344f1781224a375b78488fc343620184e95a4b27016", - "url": "https://files.pythonhosted.org/packages/d8/3b/2ed38e52eed4cf277f9df5f0463a99199a04d9e29c9e227cfafa57bd3993/websockets-11.0.3.tar.gz" + "hash": "a1d9697f3337a89691e3bd8dc56dea45a6f6d975f92e7d5f773bc715c15dde28", + "url": "https://files.pythonhosted.org/packages/c5/db/2d12649006d6686802308831f4f8a1190105ea34afb68c52f098de689ad8/websockets-12.0-cp39-cp39-macosx_10_9_x86_64.whl" }, { "algorithm": "sha256", - "hash": "6f1a3f10f836fab6ca6efa97bb952300b20ae56b409414ca85bff2ad241d2a61", - "url": "https://files.pythonhosted.org/packages/d9/36/5741e62ccf629c8e38cc20f930491f8a33ce7dba972cae93dba3d6f02552/websockets-11.0.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl" + "hash": "e469d01137942849cff40517c97a30a93ae79917752b34029f0ec72df6b46399", + "url": "https://files.pythonhosted.org/packages/c6/1a/142fa072b2292ca0897c282d12f48d5b18bdda5ac32774e3d6f9bddfd8fe/websockets-12.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl" } ], "project_name": "websockets", "requires_dists": [], - "requires_python": ">=3.7", - "version": "11.0.3" + "requires_python": ">=3.8", + "version": "12.0" }, { "artifacts": [ { "algorithm": "sha256", - "hash": "64b1df0f83706b4ef4cfb4fb0e4c2669100fd7ecacfb59e091fad300d4e04640", - "url": "https://files.pythonhosted.org/packages/f8/f8/e068dafbb844c1447c55b23c921f3d338cddaba4ea53187a7dd0058452d9/wrapt-1.15.0-py3-none-any.whl" + "hash": "6906c4100a8fcbf2fa735f6059214bb13b97f75b1a61777fcf6432121ef12ef1", + "url": "https://files.pythonhosted.org/packages/ff/21/abdedb4cdf6ff41ebf01a74087740a709e2edb146490e4d9beea054b0b7a/wrapt-1.16.0-py3-none-any.whl" }, { "algorithm": "sha256", - "hash": "76407ab327158c510f44ded207e2f76b657303e17cb7a572ffe2f5a8a48aa04d", - "url": "https://files.pythonhosted.org/packages/0f/9a/179018bb3f20071f39597cd38fc65d6285d7b89d57f6c51f502048ed28d9/wrapt-1.15.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl" + "hash": "c5cd603b575ebceca7da5a3a251e69561bec509e0b46e4993e1cac402b7247b8", + "url": "https://files.pythonhosted.org/packages/28/d3/4f079f649c515727c127c987b2ec2e0816b80d95784f2d28d1a57d2a1029/wrapt-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl" }, { "algorithm": "sha256", - "hash": "0970ddb69bba00670e58955f8019bec4a42d1785db3faa043c33d81de2bf843c", - "url": "https://files.pythonhosted.org/packages/1e/3c/cb96dbcafbf3a27413fb15e0a1997c4610283f895dc01aca955cd2fda8b9/wrapt-1.15.0-cp39-cp39-macosx_11_0_arm64.whl" + "hash": "2076fad65c6736184e77d7d4729b63a6d1ae0b70da4868adeec40989858eb3fb", + "url": "https://files.pythonhosted.org/packages/4a/cc/3402bcc897978be00fef608cd9e3e39ec8869c973feeb5e1e277670e5ad2/wrapt-1.16.0-cp39-cp39-macosx_11_0_arm64.whl" }, { "algorithm": "sha256", - "hash": "078e2a1a86544e644a68422f881c48b84fef6d18f8c7a957ffd3f2e0a74a0d4a", - "url": "https://files.pythonhosted.org/packages/78/f2/106d90140a93690eab240fae76759d62dae639fcec1bd098eccdb83aa38f/wrapt-1.15.0-cp39-cp39-musllinux_1_1_aarch64.whl" + "hash": "9b201ae332c3637a42f02d1045e1d0cccfdc41f1f2f801dafbaa7e9b4797bfc2", + "url": "https://files.pythonhosted.org/packages/70/cc/b92e1da2cad6a9f8ee481000ece07a35e3b24e041e60ff8b850c079f0ebf/wrapt-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl" }, { "algorithm": "sha256", - "hash": "7dc0713bf81287a00516ef43137273b23ee414fe41a3c14be10dd95ed98a2df9", - "url": "https://files.pythonhosted.org/packages/8a/1c/740c3ad1b7754dd7213f4df09ccdaf6b19e36da5ff3a269444ba9e103f1b/wrapt-1.15.0-cp39-cp39-musllinux_1_1_x86_64.whl" + "hash": "5f370f952971e7d17c7d1ead40e49f32345a7f7a5373571ef44d800d06b1899d", + "url": "https://files.pythonhosted.org/packages/95/4c/063a912e20bcef7124e0df97282a8af3ff3e4b603ce84c481d6d7346be0a/wrapt-1.16.0.tar.gz" }, { "algorithm": "sha256", - "hash": "2e51de54d4fb8fb50d6ee8327f9828306a959ae394d3e01a1ba8b2f937747d86", - "url": "https://files.pythonhosted.org/packages/b7/3d/9d3cd75f7fc283b6e627c9b0e904189c41ca144185fd8113a1a094dec8ca/wrapt-1.15.0-cp39-cp39-macosx_10_9_x86_64.whl" + "hash": "db2e408d983b0e61e238cf579c09ef7020560441906ca990fe8412153e3b291f", + "url": "https://files.pythonhosted.org/packages/96/e8/27ef35cf61e5147c1c3abcb89cfbb8d691b2bb8364803fcc950140bc14d8/wrapt-1.16.0-cp39-cp39-musllinux_1_1_i686.whl" }, { "algorithm": "sha256", - "hash": "cd525e0e52a5ff16653a3fc9e3dd827981917d34996600bbc34c05d048ca35cc", - "url": "https://files.pythonhosted.org/packages/c3/12/5fabf0014a0f30eb3975b7199ac2731215a40bc8273083f6a89bd6cadec6/wrapt-1.15.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl" + "hash": "b47cfad9e9bbbed2339081f4e346c93ecd7ab504299403320bf85f7f85c7d46c", + "url": "https://files.pythonhosted.org/packages/a3/1c/226c2a4932e578a2241dcb383f425995f80224b446f439c2e112eb51c3a6/wrapt-1.16.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl" }, { "algorithm": "sha256", - "hash": "9d37ac69edc5614b90516807de32d08cb8e7b12260a285ee330955604ed9dd29", - "url": "https://files.pythonhosted.org/packages/dd/eb/389f9975a6be31ddd19d29128a11f1288d07b624e464598a4b450f8d007e/wrapt-1.15.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl" + "hash": "f8212564d49c50eb4565e502814f694e240c55551a5f1bc841d4fcaabb0a9b8a", + "url": "https://files.pythonhosted.org/packages/b1/e7/459a8a4f40f2fa65eb73cb3f339e6d152957932516d18d0e996c7ae2d7ae/wrapt-1.16.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl" }, { "algorithm": "sha256", - "hash": "2cf56d0e237280baed46f0b5316661da892565ff58309d4d2ed7dba763d984b8", - "url": "https://files.pythonhosted.org/packages/f6/89/bf77b063c594795aaa056cac7b467463702f346d124d46d7f06e76e8cd97/wrapt-1.15.0-cp39-cp39-musllinux_1_1_i686.whl" + "hash": "edfad1d29c73f9b863ebe7082ae9321374ccb10879eeabc84ba3b69f2579d537", + "url": "https://files.pythonhosted.org/packages/b6/ad/7a0766341081bfd9f18a7049e4d6d45586ae5c5bb0a640f05e2f558e849c/wrapt-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl" }, { "algorithm": "sha256", - "hash": "d06730c6aed78cee4126234cf2d071e01b44b915e725a6cb439a879ec9754a3a", - "url": "https://files.pythonhosted.org/packages/f8/7d/73e4e3cdb2c780e13f9d87dc10488d7566d8fd77f8d68f0e416bfbd144c7/wrapt-1.15.0.tar.gz" + "hash": "5f15814a33e42b04e3de432e573aa557f9f0f56458745c2074952f564c50e664", + "url": "https://files.pythonhosted.org/packages/da/6f/6d0b3c4983f1fc764a422989dabc268ee87d937763246cd48aa92f1eed1e/wrapt-1.16.0-cp39-cp39-musllinux_1_1_aarch64.whl" } ], "project_name": "wrapt", "requires_dists": [], - "requires_python": "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7", - "version": "1.15.0" + "requires_python": ">=3.6", + "version": "1.16.0" } ], "platform_tag": null