Merge branch 'main' into no_env_filter

CONTRIBUTORS.md

@@ -331,6 +331,7 @@ Created as part of the release process.
 + Tom Solberg
 + Tomasz Godzik
 + Tomasz Pasternak
++ Tony Sherman
 + Travis Crawford
 + Troy Howard
 + Tushar Singh

docs/notes/2.22.x.md

@@ -105,6 +105,8 @@ Setting [the `orphan_files_behaviour = "ignore"` option](https://www.pantsbuild.
 
 The `PythonToolRequirementsBase` and `PythonToolBase` classes now have a new `help_short` field. Subclasses should now use `help_short` instead of the `help` field. The `help` field will be automatically generated using `help_short`, and will include the tool's default package version and provide instructions on how to override this version using a custom lockfile.
 
+The process execution intrinsic rule in Rust now contains support for "in workspace" execution. This is local execution from within the repository itself without using an execution sandbox. `ProcessExecutionEnvironment`'s constructor has a new `execute_in_workspace` parameter which enables workspace execution.
+
 ## Full Changelog
 
 For the full changelog, see the individual GitHub Releases for this series: https://github.com/pantsbuild/pants/releases

src/python/pants/VERSION

@@ -1 +1 @@
-2.22.0.dev2
+2.22.0.dev3

src/python/pants/backend/docker/goals/package_image_test.py

@@ -149,6 +149,7 @@ def run_process_mock(process: Process) -> FallibleProcessResult:
                     docker_image=None,
                     remote_execution=False,
                     remote_execution_extra_platform_properties=[],
+                    execute_in_workspace=False,
                 ),
                 "ran_locally",
                 0,

src/python/pants/core/goals/check_test.py

@@ -312,6 +312,7 @@ def simplify(self, v: bytes | str) -> str:
                         docker_image=None,
                         remote_execution=False,
                         remote_execution_extra_platform_properties=[],
+                        execute_in_workspace=False,
                     ),
                     "ran_locally",
                     0,

src/python/pants/core/goals/test_test.py

@@ -107,6 +107,7 @@ def make_process_result_metadata(
             docker_image=docker_image,
             remote_execution=remote_execution,
             remote_execution_extra_platform_properties=[],
+            execute_in_workspace=False,
         ),
         source,
         source_run_id,

src/python/pants/core/util_rules/environments.py

@@ -12,7 +12,7 @@
 
 from pants.build_graph.address import Address, AddressInput
 from pants.engine.engine_aware import EngineAwareParameter
-from pants.engine.environment import LOCAL_ENVIRONMENT_MATCHER as LOCAL_ENVIRONMENT_MATCHER
+from pants.engine.environment import __LOCAL_WORKSPACE_ENV_NAME, LOCAL_ENVIRONMENT_MATCHER
 from pants.engine.environment import ChosenLocalEnvironmentName as ChosenLocalEnvironmentName
 from pants.engine.environment import EnvironmentName as EnvironmentName
 from pants.engine.internals.docker import DockerResolveImageRequest, DockerResolveImageResult
@@ -708,6 +708,8 @@ async def resolve_environment_name(
     if request.raw_value == LOCAL_ENVIRONMENT_MATCHER:
         local_env_name = await Get(ChosenLocalEnvironmentName)
         return local_env_name.val
+    if request.raw_value == __LOCAL_WORKSPACE_ENV_NAME:
+        return EnvironmentName(__LOCAL_WORKSPACE_ENV_NAME)
     if request.raw_value not in environments_subsystem.names:
         raise UnrecognizedEnvironmentError(
             softwrap(
@@ -814,6 +816,8 @@ async def get_target_for_environment_name(
 ) -> EnvironmentTarget:
     if env_name.val is None:
         return EnvironmentTarget(None, None)
+    if env_name.val == __LOCAL_WORKSPACE_ENV_NAME:
+        return EnvironmentTarget(__LOCAL_WORKSPACE_ENV_NAME, None)
     if env_name.val not in environments_subsystem.names:
         raise AssertionError(
             softwrap(
@@ -895,6 +899,16 @@ async def extract_process_config_from_environment(
     global_options: GlobalOptions,
     environments_subsystem: EnvironmentsSubsystem,
 ) -> ProcessExecutionEnvironment:
+    if tgt.name == __LOCAL_WORKSPACE_ENV_NAME:
+        return ProcessExecutionEnvironment(
+            environment_name=__LOCAL_WORKSPACE_ENV_NAME,
+            platform=platform.value,
+            remote_execution=False,
+            remote_execution_extra_platform_properties=[],
+            docker_image=None,
+            execute_in_workspace=True,
+        )
+
     docker_image = None
     remote_execution = False
     raw_remote_execution_extra_platform_properties: tuple[str, ...] = ()
@@ -942,6 +956,7 @@ async def extract_process_config_from_environment(
             tuple(pair.split("=", maxsplit=1))  # type: ignore[misc]
             for pair in raw_remote_execution_extra_platform_properties
         ],
+        execute_in_workspace=False,
     )
 
 

src/python/pants/core/util_rules/environments_test.py

@@ -12,7 +12,6 @@
 from pants.build_graph.address import Address, ResolveError
 from pants.core.util_rules import environments
 from pants.core.util_rules.environments import (
-    LOCAL_ENVIRONMENT_MATCHER,
     AllEnvironmentTargets,
     AmbiguousEnvironmentError,
     ChosenLocalEnvironmentName,
@@ -36,6 +35,7 @@
     extract_process_config_from_environment,
     resolve_environment_name,
 )
+from pants.engine.environment import LOCAL_ENVIRONMENT_MATCHER
 from pants.engine.internals.docker import DockerResolveImageRequest, DockerResolveImageResult
 from pants.engine.platform import Platform
 from pants.engine.process import ProcessCacheScope

src/python/pants/engine/environment.py

@@ -6,8 +6,13 @@
 
 from pants.engine.engine_aware import EngineAwareParameter
 
+# Reserved sentinel value directing Pants to find applicable local environment.
 LOCAL_ENVIRONMENT_MATCHER = "__local__"
 
+# Reserved sentinel value representing execution within the workspace and not local sandbox.
+# Note: This is temporary until support for `workspace_environment` target type lands.
+__LOCAL_WORKSPACE_ENV_NAME = "__local_workspace__"
+
 
 @dataclass(frozen=True)
 class EnvironmentName(EngineAwareParameter):

src/python/pants/engine/internals/native_engine.pyi

@@ -497,6 +497,7 @@ class ProcessExecutionEnvironment:
         docker_image: str | None,
         remote_execution: bool,
         remote_execution_extra_platform_properties: Sequence[tuple[str, str]],
+        execute_in_workspace: bool,
     ) -> None: ...
     def __eq__(self, other: ProcessExecutionEnvironment | Any) -> bool: ...
     def __hash__(self) -> int: ...

src/python/pants/engine/process_test.py

@@ -3,8 +3,12 @@
 
 from __future__ import annotations
 
+import textwrap
+from pathlib import Path
+
 import pytest
 
+from pants.engine.environment import __LOCAL_WORKSPACE_ENV_NAME, EnvironmentName
 from pants.engine.fs import (
     EMPTY_DIGEST,
     CreateDigest,
@@ -15,7 +19,9 @@
     FileContent,
     SymlinkEntry,
 )
+from pants.engine.internals.native_engine import Snapshot
 from pants.engine.internals.scheduler import ExecutionError
+from pants.engine.platform import Platform
 from pants.engine.process import (
     FallibleProcessResult,
     InteractiveProcess,
@@ -28,14 +34,16 @@
 from pants.util.contextutil import environment_as
 
 
-def new_rule_runner() -> RuleRunner:
+def new_rule_runner(**extra_kwargs) -> RuleRunner:
     return RuleRunner(
         rules=[
             QueryRule(ProcessResult, [Process]),
             QueryRule(FallibleProcessResult, [Process]),
             QueryRule(InteractiveProcessResult, [InteractiveProcess]),
             QueryRule(DigestEntries, [Digest]),
+            QueryRule(Platform, []),
         ],
+        **extra_kwargs,
     )
 
 
@@ -299,3 +307,87 @@ def test_interactive_process_inputs(rule_runner: RuleRunner, run_in_workspace: b
             "prefix1",
             "prefix2",
         }
+
+
+def test_workspace_process_basic(rule_runner) -> None:
+    rule_runner = new_rule_runner(inherent_environment=EnvironmentName(__LOCAL_WORKSPACE_ENV_NAME))
+    build_root = Path(rule_runner.build_root)
+
+    # Check that a custom exit code is returned as expected.
+    process = Process(
+        argv=["/bin/bash", "-c", "exit 143"],
+        description="a process which reports its error code",
+        cache_scope=ProcessCacheScope.PER_SESSION,  # necessary to ensure result not cached from prior test runs
+    )
+    result = rule_runner.request(FallibleProcessResult, [process])
+    assert result.exit_code == 143
+    assert result.metadata.execution_environment.environment_type == "workspace"
+
+    # Test whether there is a distinction between the workspace and chroot when a workspace
+    # process executes. Do this by putting a file in the build root which is not covered by a
+    # target, a depenency created via a digest, and have the invoked process create a file
+    # in the build root.
+    rule_runner.write_files(
+        {
+            "unmanaged.txt": "from-workspace\n",
+        }
+    )
+    input_snapshot = rule_runner.make_snapshot(
+        {
+            "dependency.txt": "from-digest\n",
+        }
+    )
+    script = textwrap.dedent(
+        """
+        cat '{chroot}/dependency.txt'
+        pwd
+        cat unmanaged.txt
+        touch created-by-invocation
+        """
+    )
+    process = Process(
+        argv=["/bin/bash", "-c", script],
+        input_digest=input_snapshot.digest,
+        description="a workspace process",
+        cache_scope=ProcessCacheScope.PER_SESSION,  # necessary to ensure result not cached from prior test runs
+    )
+    result = rule_runner.request(ProcessResult, [process])
+    lines = result.stdout.decode().splitlines()
+    assert lines == [
+        "from-digest",
+        rule_runner.build_root,
+        "from-workspace",
+    ]
+    assert (build_root / "created-by-invocation").exists()
+
+    # Test that changing the working directory works.
+    subdir = build_root / "subdir"
+    subdir.mkdir()
+    process = Process(
+        argv=["/bin/bash", "-c", "touch file-in-subdir"],
+        description="check working_directory works",
+        working_directory="subdir",
+        cache_scope=ProcessCacheScope.PER_SESSION,  # necessary to ensure result not cached from prior test runs
+    )
+    result = rule_runner.request(ProcessResult, [process])
+    assert (subdir / "file-in-subdir").exists()
+
+    # Test output capture correctly captures from the sandbox and not the workspace.
+    script = textwrap.dedent(
+        """
+        touch '{chroot}/capture-this-file' will-not-capture-this-file
+        echo this-goes-to-stdout
+        echo this-goes-to-stderr 1>&2
+        """
+    )
+    process = Process(
+        argv=["/bin/bash", "-c", script],
+        description="check output capture works",
+        output_files=["capture-this-file", "will-not-capture-this-file"],
+        cache_scope=ProcessCacheScope.PER_SESSION,  # necessary to ensure result not cached from prior test runs
+    )
+    result = rule_runner.request(ProcessResult, [process])
+    assert result.stdout.decode() == "this-goes-to-stdout\n"
+    assert result.stderr.decode() == "this-goes-to-stderr\n"
+    snapshot = rule_runner.request(Snapshot, [result.output_digest])
+    assert snapshot.files == ("capture-this-file",)

src/python/pants/testutil/rule_runner.py

@@ -615,6 +615,7 @@ def run_interactive_process(self, request: InteractiveProcess) -> InteractivePro
                     docker_image=None,
                     remote_execution=False,
                     remote_execution_extra_platform_properties=[],
+                    execute_in_workspace=False,
                 ),
             )
 

src/rust/engine/Cargo.toml

@@ -289,8 +289,8 @@ prodash = { git = "https://github.com/stuhood/prodash", rev = "stuhood/raw-messa
 prost = "0.12"
 prost-build = "0.12"
 prost-types = "0.12"
-pyo3 = "0.20"
-pyo3-build-config = "0.20"
+pyo3 = { version = "0.21", features = ["gil-refs"] }
+pyo3-build-config = "0.21"
 rand = "0.8"
 regex = "1"
 rlimit = "0.8"

src/rust/engine/process_execution/src/lib.rs

@@ -61,6 +61,10 @@ pub mod named_caches_tests;
 
 pub(crate) mod fork_exec;
 
+pub mod workspace;
+#[cfg(test)]
+pub mod workspace_tests;
+
 extern crate uname;
 
 pub use crate::children::ManagedChild;
@@ -429,11 +433,16 @@ impl Default for InputDigests {
 }
 
 #[derive(DeepSizeOf, Debug, Clone, Hash, PartialEq, Eq, Serialize)]
+/// "Where" to run a `Process`. This is the Rust-side of the environments feature.
 pub enum ProcessExecutionStrategy {
+    /// Run the Process locally in an execution sandbox.
     Local,
-    /// Stores the platform_properties.
+    /// Run the Process locally in the workspace without an execution sandbox.
+    LocalInWorkspace,
+    /// Run the Process remotely using the Remote Execution API. The vector stores the platform_properties to pass
+    /// for that execution.
     RemoteExecution(Vec<(String, String)>),
-    /// Stores the image name.
+    /// Run the Process in a Docker container. The string stores the image name.
     Docker(String),
 }
 
@@ -443,6 +452,7 @@ impl ProcessExecutionStrategy {
     pub fn cache_value(&self) -> String {
         match self {
             Self::Local => "local_execution".to_string(),
+            Self::LocalInWorkspace => "workspace_execution".to_string(),
             Self::RemoteExecution(_) => "remote_execution".to_string(),
             // NB: this image will include the container ID, thanks to
             // https://github.com/pantsbuild/pants/pull/17101.
@@ -453,6 +463,7 @@ impl ProcessExecutionStrategy {
     pub fn strategy_type(&self) -> &'static str {
         match self {
             Self::Local => "local",
+            Self::LocalInWorkspace => "workspace",
             Self::RemoteExecution(_) => "remote",
             Self::Docker(_) => "docker",
         }