Merge branch 'main' into jcannon/bind_mounts

build-support/bin/generate_docs.py

@@ -234,7 +234,7 @@ def run_pants_help_all() -> dict[str, Any]:
         "--no-verify-config",
         "help-all",
     ]
-    run = subprocess.run(argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding="utf-8")
+    run = subprocess.run(argv, capture_output=True, encoding="utf-8")
     try:
         run.check_returncode()
     except subprocess.CalledProcessError:

build-support/bin/terraform_tool_versions.py

@@ -59,7 +59,7 @@ def check_import_results(import_results: gnupg.ImportResult):
         Looks the import results for one which has an "ok" status. We can't use the number of keys
         imported because a re-import of a key results in 0 keys imported.
         """
-        has_ok = any(("ok" in r for r in import_results.results))
+        has_ok = any("ok" in r for r in import_results.results)
         return has_ok
 
 

build-support/bin/terraform_tool_versions_test.py

@@ -1,7 +1,6 @@
 #!/usr/bin/env python3
 # Copyright 2022 Pants project contributors (see CONTRIBUTORS.md).
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
-from __future__ import absolute_import, division, print_function, unicode_literals
 
 import pytest
 from terraform_tool_versions import (

docs/markdown/Getting Help/the-pants-community/team.md

@@ -24,13 +24,13 @@ Pants open source project has had many team members in over a decade. The curren
 | **Benjy Weinberger**       | "The options system — unifying our flags, config and env vars into a coherent, extensible framework." |
 | **Carina C. Zona**         | "Developing the #welcome channel on Pants community chat"                                             |
 | **Christopher Neugebauer** | Adding multi-architecture Mac support for Apple Silicon (M1s)                                         |
-| **Dan Moran**              | "Enabling batched `pytest` execution in `pants test`"                                          |
+| **Dan Moran**              | "Enabling batched `pytest` execution in `pants test`"                                                 |
 | **Daniel Wagner-Hall**     | Laid the foundation for remote execution                                                              |
 | **Danny McClanahan**       | Implemented bounded runtime polymorphism with union rules and improved plugin UX with async/await     |
 | **Eric Arellano**          | "Migrating Pants to Python 3 for my internship project"                                               |
 | **Henry Fuller**           | "Working on replacing watchman with a kernel based file watcher in the V2 engine"                     |
-| **Huon Wilson**            | "Improving the experiencing of using `mypy` with Pants"          |
-| **Jacob Floyd**            | Gave the idea for `skip_flake8` et al, which grew into a flagship feature: incremental adoption          |
+| **Huon Wilson**            | "Improving the experiencing of using `mypy` with Pants"                                               |
+| **Jacob Floyd**            | Gave the idea for `skip_flake8` et al, which grew into a flagship feature: incremental adoption       |
 | **John Sirois**            |                                                                                                       |
 | **Josh Reed**              | "Remains continually willing to bikeshed about design decisions on the Pants Slack"                   |
 | **Joshua Cannon**          | "I'm proud and thankful my voice can and is being used to shape Pants inside and out"                 |
@@ -53,6 +53,7 @@ Pants open source project has had many team members in over a decade. The curren
 | **Doron Somech**        | "Improving Scala dependency inference to support our codebase"                                           |
 | **Gautham Nair**        |                                                                                                          |
 | **Jonas Stendahl**      | "Adding formatting and linting support for Protobuf"                                                     |
+| **Krishnan Chandra**    | "Adding support for the Ruff formatter to Pants"                                                         |
 | **Marcelo Trylesinski** | "Improving onboarding experience"                                                                        |
 | **Nick Grisafi**        | "Participating in podcasts with maintainers (Eric and Josh) on developer experience and Pants!"          |
 | **Raúl Cuza**           | "The first time I was able to help someone else with pants on the Slack <https://pantsbuild.slack.com>." |

docs/markdown/Introduction/welcome-to-pants.md

@@ -42,7 +42,7 @@ Pants is designed for fast, consistent, ergonomic builds. Some noteworthy featur
 Which languages and frameworks does Pants support?
 ==================================================
 
-- Pants [ships](page:language-support) with support for [Python](doc:python), [Go](doc:go), [Java](doc:jvm-overview), [Scala](doc:jvm-overview) and [Shell](doc:shell).
+- Pants [ships](page:language-support) with support for [Python](doc:python), [Go](doc:go), [Java](doc:jvm-overview), [Scala](doc:jvm-overview) and [Shell](doc:shell-overview).
 - Pants supports a wide range of code generators (such as Thrift, Protobuf, Scrooge and Avro), linters and formatters, and it is easy to add support for new or custom ones
 - Pants can create standalone binaries, [Docker images](doc:docker), AWS Lambdas and GCP Cloud Functions
 

docs/markdown/Shell/shell.md

@@ -1,6 +1,6 @@
 ---
 title: "Shell overview"
-slug: "shell"
+slug: "shell-overview"
 excerpt: "Pants's support for Shellcheck, shfmt, and shUnit2."
 hidden: false
 createdAt: "2021-04-14T04:21:15.028Z"

docs/markdown/Using Pants/concepts/enabling-backends.md

@@ -47,12 +47,12 @@ This list is also available via `pants backends --help`, which includes any addi
 | `pants.backend.python.lint.pyupgrade`                | Enables Pyupgrade, which upgrades to new Python syntax: <https://pypi.org/project/pyupgrade/>      | [Linters and formatters](doc:python-linters-and-formatters)       |
 | `pants.backend.python.lint.yapf`                     | Enables Yapf, the Python formatter: <https://pypi.org/project/yapf/>                               | [Linters and formatters](doc:python-linters-and-formatters)       |
 | `pants.backend.python.typecheck.mypy`                | Enables MyPy, the Python type checker: <https://mypy.readthedocs.io/en/stable/>.                   | [typecheck](doc:python-check-goal)                                |
-| `pants.backend.shell`                                | Core Shell support, including shUnit2 test runner.                                                 | [Shell overview](doc:shell)                                       |
-| `pants.backend.shell.lint.shfmt`                     | Enables shfmt, a Shell autoformatter: <https://github.com/mvdan/sh>.                               | [Shell overview](doc:shell)                                       |
-| `pants.backend.shell.lint.shellcheck`                | Enables Shellcheck, a Shell linter: <https://www.shellcheck.net/>.                                 | [Shell overview](doc:shell)                                       |
+| `pants.backend.shell`                                | Core Shell support, including shUnit2 test runner.                                                 | [Shell overview](doc:shell-overview)                              |
+| `pants.backend.shell.lint.shfmt`                     | Enables shfmt, a Shell autoformatter: <https://github.com/mvdan/sh>.                               | [Shell overview](doc:shell-overview)                              |
+| `pants.backend.shell.lint.shellcheck`                | Enables Shellcheck, a Shell linter: <https://www.shellcheck.net/>.                                 | [Shell overview](doc:shell-overview)                              |
 | `pants.backend.tools.preamble`                       | Enables "preamble", a Pants fixer for copyright headers and shebang lines                          | [`preamble`](doc:reference-preamble)                              |
 | `pants.backend.tools.taplo`                          | Enables Taplo, a TOML autoformatter: <https://taplo.tamasfe.dev>                                   |                                                                   |
-| `pants.backend.url_handlers.s3`                      | Enables accessing s3 via credentials in `file(source=http_source(...))`                            |                                                              |
+| `pants.backend.url_handlers.s3`                      | Enables accessing s3 via credentials in `file(source=http_source(...))`                            |                                                                   |
 
 Available experimental backends
 -------------------------------

docs/markdown/Using Pants/concepts/options.md

@@ -81,23 +81,33 @@ Note that any dashes in the option flag name are converted to underscores: `--mu
 
 ### Config file interpolation
 
-Environment variables can be interpolated by using the syntax `%(env.ENV_VAR)s`, e.g.:
+A string value in a config file can contain placeholders of the form `%(key)s`, which will be replaced with a corresponding value. The `key` can be one of:
+
+- A string-valued option in the DEFAULT section of the same config file.
+- A string-valued option in the same section of the config file as the value containing the placeholder.
+- Any environment variable, prefixed with `env.`: `%(env.ENV_VAR)s`.
+- The following special values:
+  - `%(buildroot)s`: absolute path to the root of your repository.
+  - `%(homedir)s`: equivalent to `$HOME` or `~`.
+  - `%(user)s`: the current user's username, obtained from the system password file.
+  - `%(pants_workdir)s`: the absolute path of the global option `--pants-workdir`, which defaults
+    to `{buildroot}/.pants.d/`.
+  - `%(pants_distdir)s`: the absolute path of the global option `--pants-distdir`, which defaults
+    to `{buildroot}/dist/`.
+
+An interpolated value may itself contain placeholders, that will be recursively interpolated.
+
+For example:
 
 ```toml pants.toml
+[DEFAULT]
+domain = "my.domain"
+
 [python-repos]
-# This will substitute `%(env.PY_REPO)s` with the value of the environment
-# variable PY_REPO
-indexes.add = ["http://%(env.PY_REPO)[email protected]/index
+repo_host = "repo.%(domain)s"
+indexes.add = ["http://%(env.PY_REPO)s@%(repo_host)s/index
 ```
 
-Additionally, a few special values are pre-populated with the `%(var)s` syntax:
-
-- `%(buildroot)s`: absolute path to the root of your repository
-- `%(homedir)s`: equivalent to `$HOME` or `~`
-- `%(user)s`: equivalent to `$USER`
-- `%(pants_distdir)s`: absolute path of the global option `--pants-distdir`, which defaults 
-   to `{buildroot}/dist/`
-
 Option types
 ============
 

pants.toml

@@ -14,6 +14,7 @@ backend_packages.add = [
   "pants.backend.python.lint.flake8",
   "pants.backend.python.lint.isort",
   "pants.backend.python.typecheck.mypy",
+  "pants.backend.python.lint.pyupgrade",
   "pants.backend.python.mixed_interpreter_constraints",
   "pants.backend.shell",
   "pants.backend.shell.lint.shellcheck",
@@ -92,7 +93,6 @@ enabled = true
 repo_id = "7775F8D5-FC58-4DBC-9302-D00AE4A1505F"
 
 [cli.alias]
-run-pyupgrade = "--backend-packages=pants.backend.experimental.python.lint.pyupgrade fmt"
 --all-changed = "--changed-since=HEAD --changed-dependents=transitive"
 
 [source]
@@ -208,6 +208,10 @@ template_by_globs = "@build-support/preambles/config.yaml"
 [generate-lockfiles]
 diff = true
 
+[pyupgrade]
+args = ["--keep-runtime-typing", "--py38-plus"]
+
+
 [jvm]
 default_resolve = "jvm_testprojects"
 

src/python/pants/VERSION

@@ -1 +1 @@
-2.20.0.dev3
+2.20.0.dev4

src/python/pants/backend/adhoc/adhoc_tool.py

@@ -20,31 +20,19 @@
     AdhocToolStdoutFilenameField,
     AdhocToolWorkdirField,
 )
-from pants.build_graph.address import Address, AddressInput
-from pants.core.goals.run import RunFieldSet, RunInSandboxRequest
 from pants.core.target_types import FileSourceField
 from pants.core.util_rules.adhoc_process_support import (
     AdhocProcessRequest,
     AdhocProcessResult,
-    ExtraSandboxContents,
-    MergeExtraSandboxContents,
-    ResolvedExecutionDependencies,
-    ResolveExecutionDependenciesRequest,
+    ToolRunner,
+    ToolRunnerRequest,
 )
 from pants.core.util_rules.adhoc_process_support import rules as adhoc_process_support_rules
 from pants.core.util_rules.environments import EnvironmentNameRequest
-from pants.engine.addresses import Addresses
 from pants.engine.environment import EnvironmentName
-from pants.engine.fs import Digest, MergeDigests, Snapshot
-from pants.engine.internals.native_engine import EMPTY_DIGEST
+from pants.engine.fs import Digest, Snapshot
 from pants.engine.rules import Get, collect_rules, rule
-from pants.engine.target import (
-    FieldSetsPerTarget,
-    FieldSetsPerTargetRequest,
-    GeneratedSources,
-    GenerateSourcesRequest,
-    Targets,
-)
+from pants.engine.target import GeneratedSources, GenerateSourcesRequest
 from pants.engine.unions import UnionRule
 from pants.util.frozendict import FrozenDict
 from pants.util.logging import LogLevel
@@ -63,6 +51,7 @@ async def run_in_sandbox_request(
 ) -> GeneratedSources:
     target = request.protocol_target
     description = f"the `{target.alias}` at {target.address}"
+
     environment_name = await Get(
         EnvironmentName, EnvironmentNameRequest, EnvironmentNameRequest.from_target(target)
     )
@@ -71,104 +60,38 @@ async def run_in_sandbox_request(
     if not runnable_address_str:
         raise Exception(f"Must supply a value for `runnable` for {description}.")
 
-    runnable_address = await Get(
-        Address,
-        AddressInput,
-        AddressInput.parse(
-            runnable_address_str,
-            relative_to=target.address.spec_path,
-            description_of_origin=f"The `{AdhocToolRunnableField.alias}` field of {description}",
+    tool_runner = await Get(
+        ToolRunner,
+        ToolRunnerRequest(
+            runnable_address_str=runnable_address_str,
+            args=target.get(AdhocToolArgumentsField).value or (),
+            execution_dependencies=target.get(AdhocToolExecutionDependenciesField).value or (),
+            runnable_dependencies=target.get(AdhocToolRunnableDependenciesField).value or (),
+            target=request.protocol_target,
+            named_caches=FrozenDict(target.get(AdhocToolNamedCachesField).value or {}),
         ),
     )
 
-    addresses = Addresses((runnable_address,))
-    addresses.expect_single()
-
-    runnable_targets = await Get(Targets, Addresses, addresses)
-    field_sets = await Get(
-        FieldSetsPerTarget, FieldSetsPerTargetRequest(RunFieldSet, runnable_targets)
-    )
-    run_field_set: RunFieldSet = field_sets.field_sets[0]
-
     working_directory = target[AdhocToolWorkdirField].value or ""
     root_output_directory = target[AdhocToolOutputRootDirField].value or ""
 
-    # Must be run in target environment so that the binaries/envvars match the execution
-    # environment when we actually run the process.
-    run_request = await Get(
-        RunInSandboxRequest, {environment_name: EnvironmentName, run_field_set: RunFieldSet}
-    )
-
-    execution_environment = await Get(
-        ResolvedExecutionDependencies,
-        ResolveExecutionDependenciesRequest(
-            target.address,
-            target.get(AdhocToolExecutionDependenciesField).value,
-            target.get(AdhocToolRunnableDependenciesField).value,
-        ),
-    )
-    dependencies_digest = execution_environment.digest
-    runnable_dependencies = execution_environment.runnable_dependencies
-
-    extra_env: dict[str, str] = dict(run_request.extra_env or {})
-    extra_path = extra_env.pop("PATH", None)
-
-    extra_sandbox_contents = []
-
-    extra_sandbox_contents.append(
-        ExtraSandboxContents(
-            EMPTY_DIGEST,
-            extra_path,
-            run_request.immutable_input_digests or FrozenDict(),
-            run_request.append_only_caches or FrozenDict(),
-            run_request.extra_env or FrozenDict(),
-        )
-    )
-
-    if runnable_dependencies:
-        extra_sandbox_contents.append(
-            ExtraSandboxContents(
-                EMPTY_DIGEST,
-                f"{{chroot}}/{runnable_dependencies.path_component}",
-                runnable_dependencies.immutable_input_digests,
-                runnable_dependencies.append_only_caches,
-                runnable_dependencies.extra_env,
-            )
-        )
-
-    merged_extras = await Get(
-        ExtraSandboxContents, MergeExtraSandboxContents(tuple(extra_sandbox_contents))
-    )
-    extra_env = dict(merged_extras.extra_env)
-    if merged_extras.path:
-        extra_env["PATH"] = merged_extras.path
-
-    input_digest = await Get(Digest, MergeDigests((dependencies_digest, run_request.digest)))
-
     output_files = target.get(AdhocToolOutputFilesField).value or ()
     output_directories = target.get(AdhocToolOutputDirectoriesField).value or ()
 
-    extra_args = target.get(AdhocToolArgumentsField).value or ()
-
-    append_only_caches = {
-        **merged_extras.append_only_caches,
-        **(target.get(AdhocToolNamedCachesField).value or {}),
-    }
-
     process_request = AdhocProcessRequest(
         description=description,
         address=target.address,
         working_directory=working_directory,
         root_output_directory=root_output_directory,
-        argv=tuple(run_request.args + extra_args),
+        argv=tool_runner.args,
         timeout=None,
-        input_digest=input_digest,
-        immutable_input_digests=FrozenDict.frozen(merged_extras.immutable_input_digests),
-        append_only_caches=FrozenDict(append_only_caches),
+        input_digest=tool_runner.digest,
+        immutable_input_digests=FrozenDict.frozen(tool_runner.immutable_input_digests),
+        append_only_caches=FrozenDict(tool_runner.append_only_caches),
         output_files=output_files,
         output_directories=output_directories,
         fetch_env_vars=target.get(AdhocToolExtraEnvVarsField).value or (),
-        supplied_env_var_values=FrozenDict(extra_env),
+        supplied_env_var_values=FrozenDict(tool_runner.extra_env),
         log_on_process_errors=None,
         log_output=target[AdhocToolLogOutputField].value,
         capture_stderr_file=target[AdhocToolStderrFilenameField].value,