From be656945be0a874bb506bcd763d4b6529ab2bd95 Mon Sep 17 00:00:00 2001 From: Joseph Tilahun Date: Wed, 23 Aug 2023 19:43:55 -0700 Subject: [PATCH] user api change: change the Pip version option to a string (#19643) Currently, Pants defines a limited set of Pip versions that it supports in an enum. For its part, Pants really does nothing with the Pip version except pass it along to PEX. Only PEX actually cares about the Pip version. Having Pants middle-man the Pip version doesn't make sense. I did the following: 1. Change the Pip version option `pip_version` from an enum to a string so that Pants itself can't care about what the chosen version of Pip is. 2. Delete the enum that defines the limited set of supported Pip versions. 3. Update the documentation of `pip_version` to clarify that the list of available Pip versions is determined by PEX, not Pants. --- .../python-goals/python-package-goal.md | 2 +- .../python-integrations/awslambda-python.md | 2 +- .../google-cloud-function-python.md | 2 +- .../Python/python-integrations/pyoxidizer.md | 2 +- docs/markdown/Python/python.md | 2 +- docs/markdown/Python/python/pex-files.md | 19 -------- docs/markdown/Python/python/pex.md | 44 +++++++++++++++++++ .../pants/backend/python/goals/lockfile.py | 2 +- .../pants/backend/python/subsystems/setup.py | 25 +++-------- .../pants/backend/python/target_types.py | 2 +- .../backend/python/util_rules/pex_cli.py | 4 +- 11 files changed, 59 insertions(+), 47 deletions(-) delete mode 100644 docs/markdown/Python/python/pex-files.md create mode 100644 docs/markdown/Python/python/pex.md diff --git a/docs/markdown/Python/python-goals/python-package-goal.md b/docs/markdown/Python/python-goals/python-package-goal.md index 74253b4bdb9..3ce9628297c 100644 --- a/docs/markdown/Python/python-goals/python-package-goal.md +++ b/docs/markdown/Python/python-goals/python-package-goal.md @@ -30,7 +30,7 @@ You can run `pants package ::` to build all artifacts in your project. Pants wil Creating a PEX file from a `pex_binary` target ---------------------------------------------- -Running `package` on a `pex_binary` target will create an executable [PEX file](doc:pex-files). +Running `package` on a `pex_binary` target will create an executable [PEX file](doc:pex). The PEX file will contain all the code needed to run the binary, namely: diff --git a/docs/markdown/Python/python-integrations/awslambda-python.md b/docs/markdown/Python/python-integrations/awslambda-python.md index fcf4aa3c7b7..4b175bb4c98 100644 --- a/docs/markdown/Python/python-integrations/awslambda-python.md +++ b/docs/markdown/Python/python-integrations/awslambda-python.md @@ -203,7 +203,7 @@ Migrating from Pants 2.16 and earlier Pants has implemented a new way to package Lambda functions in 2.17, which is now the default in 2.18, resulting in smaller packages and faster cold starts. This involves some changes: -- In Pants 2.16 and earlier, Pants used the [Lambdex](https://github.com/pantsbuild/lambdex) project. First, Pants would convert your code into a [Pex file](doc:pex-files) and then use Lambdex to adapt this to be better understood by AWS by adding a shim handler at the path `lambdex_handler.handler`. This shim handler first triggers the Pex initialization to choose and unzip dependencies, during the "INIT" phase. +- In Pants 2.16 and earlier, Pants used the [Lambdex](https://github.com/pantsbuild/lambdex) project. First, Pants would convert your code into a [Pex file](doc:pex) and then use Lambdex to adapt this to be better understood by AWS by adding a shim handler at the path `lambdex_handler.handler`. This shim handler first triggers the Pex initialization to choose and unzip dependencies, during the "INIT" phase. - In Pants 2.17, the use of Lambdex was deprecated, in favour of choosing the appropriate dependencies ahead of time, as described above, without needing to do this on each cold start. This results in a zip file laid out in the format recommended by AWS, and includes a re-export of the handler at the path `lambda_function.handler`. - In Pants 2.18, the new behaviour is now the default behaviour. Layers can now be built using Pants, and this addition includes renaming the `python_awslambda` target to `python_aws_lambda_function`. - In Pants 2.19, the old Lambdex behaviour will be entirely removed. diff --git a/docs/markdown/Python/python-integrations/google-cloud-function-python.md b/docs/markdown/Python/python-integrations/google-cloud-function-python.md index 3baf22ee4e2..fa4452833d0 100644 --- a/docs/markdown/Python/python-integrations/google-cloud-function-python.md +++ b/docs/markdown/Python/python-integrations/google-cloud-function-python.md @@ -120,7 +120,7 @@ Migrating from Pants 2.16 and earlier Pants has implemented a new way to package Google Cloud Functions in 2.17, which is now the default in 2.18, resulting in smaller packages and faster cold starts. This involves some changes: -- In Pants 2.16 and earlier, Pants used the [Lambdex](https://github.com/pantsbuild/lambdex) project. First, Pants would convert your code into a [Pex file](doc:pex-files) and then use Lambdex to adapt this to be better understood by GCF by adding a shim handler. This shim handler first triggers the Pex initialization to choose and unzip dependencies, during initialization. +- In Pants 2.16 and earlier, Pants used the [Lambdex](https://github.com/pantsbuild/lambdex) project. First, Pants would convert your code into a [Pex file](doc:pex) and then use Lambdex to adapt this to be better understood by GCF by adding a shim handler. This shim handler first triggers the Pex initialization to choose and unzip dependencies, during initialization. - In Pants 2.17, the use of Lambdex was deprecated, in favour of choosing the appropriate dependencies ahead of time, as described above, without needing to do this on each cold start. This results in a zip file laid out in the format recommended by GCF, and includes a re-export of the handler. - In Pants 2.18, the new behaviour is now the default behaviour. - In Pants 2.19, the old Lambdex behaviour will be entirely removed. diff --git a/docs/markdown/Python/python-integrations/pyoxidizer.md b/docs/markdown/Python/python-integrations/pyoxidizer.md index ad5c784503f..22422b55f86 100644 --- a/docs/markdown/Python/python-integrations/pyoxidizer.md +++ b/docs/markdown/Python/python-integrations/pyoxidizer.md @@ -5,7 +5,7 @@ excerpt: "Creating Python binaries through PyOxidizer." hidden: false createdAt: "2022-02-04T18:41:48.950Z" --- -PyOxidizer allows you to distribute your code as a single binary file, similar to [Pex files](doc:pex-files). Unlike Pex, these binaries include a Python interpreter, often greatly simplifying distribution. +PyOxidizer allows you to distribute your code as a single binary file, similar to [Pex files](doc:pex). Unlike Pex, these binaries include a Python interpreter, often greatly simplifying distribution. See our blog post on [Packaging Python with the Pants PyOxidizer Plugin](https://blog.pantsbuild.org/packaging-python-with-the-pyoxidizer-pants-plugin/) for more discussion of the benefits of PyOxidizer. diff --git a/docs/markdown/Python/python.md b/docs/markdown/Python/python.md index 75153980840..63f0719a513 100644 --- a/docs/markdown/Python/python.md +++ b/docs/markdown/Python/python.md @@ -24,5 +24,5 @@ There are also [goals](doc:project-introspection) for querying and understanding - [Third-party dependencies](doc:python-third-party-dependencies) - [Interpreter compatibility](doc:python-interpreter-compatibility) - [Linters and formatters](doc:python-linters-and-formatters) -- [Pex files](doc:pex-files) +- [Pex files](doc:pex) - [Building distributions](doc:python-distributions) diff --git a/docs/markdown/Python/python/pex-files.md b/docs/markdown/Python/python/pex-files.md deleted file mode 100644 index 4f9b3a6a9b3..00000000000 --- a/docs/markdown/Python/python/pex-files.md +++ /dev/null @@ -1,19 +0,0 @@ ---- -title: "Pex files" -slug: "pex-files" -hidden: false -createdAt: "2020-03-21T20:47:00.042Z" ---- -When working with Python code, Pants makes frequent use of the [Pex](https://github.com/pantsbuild/pex) (Python EXecutable) format. So, you'll see Pex referenced frequently in this documentation. - -A Pex is a self-contained Python environment, similar in spirit to a virtualenv. A Pex can contain combinations of Python source files, 3rd-party requirements (sdists or wheels), resource files, and metadata describing the contents. - -Importantly, this metadata can include: - -- Python interpreter constraints. -- Python platforms, like `macosx_11_0_arm64-cp-39-cp39`. -- An entry point or console script. - -A Pex can be bundled into a single `.pex` file. This file, when executed, knows how to unpack itself, find an interpreter that matches its constraints, and run itself on that interpreter. Therefore deploying code packaged in a Pex file is as simple as copying the file to an environment that has a suitable Python interpreter. - -Check out [blog.pantsbuild.org/pants-pex-and-docker](https://blog.pantsbuild.org/pants-pex-and-docker/) for how this workflow gets even better when combined with Pants's Docker support! diff --git a/docs/markdown/Python/python/pex.md b/docs/markdown/Python/python/pex.md new file mode 100644 index 00000000000..aeafbc73cb8 --- /dev/null +++ b/docs/markdown/Python/python/pex.md @@ -0,0 +1,44 @@ +--- +title: "Pex" +slug: "pex" +hidden: false +createdAt: "2020-03-21T20:47:00.042Z" +--- +Pex files +--------- +When working with Python code, Pants makes frequent use of the [Pex](https://github.com/pantsbuild/pex) (Python EXecutable) format. So, you'll see Pex referenced frequently in this documentation. + +A Pex is a self-contained Python environment, similar in spirit to a virtualenv. A Pex can contain combinations of Python source files, 3rd-party requirements (sdists or wheels), resource files, and metadata describing the contents. + +Importantly, this metadata can include: + +- Python interpreter constraints. +- Python platforms, like `macosx_11_0_arm64-cp-39-cp39`. +- An entry point or console script. + +A Pex can be bundled into a single `.pex` file. This file, when executed, knows how to unpack itself, find an interpreter that matches its constraints, and run itself on that interpreter. Therefore deploying code packaged in a Pex file is as simple as copying the file to an environment that has a suitable Python interpreter. + +Check out [blog.pantsbuild.org/pants-pex-and-docker](https://blog.pantsbuild.org/pants-pex-and-docker/) for how this workflow gets even better when combined with Pants's Docker support! + +Setting Pex and Pip versions +---------------------------- + +Pants makes use of the [Pex](https://github.com/pantsbuild/pex) command-line tool internally for Pex building. The Pex version that Pants uses is specified by the `version` option under the `pex-cli` subsystem. The known Pex versions are specified by the `known_versions` option under the `pex-cli` subsystem. You can see all Pex tool options and their current values by running `pants help-advanced pex-cli`. To upgrade the Pex version, update these option values accordingly. For instance, in `pants.toml`, to upgrade to Pex 2.1.143: + +```[pex-cli] +version = "v2.1.143" +known_versions = [ + "v2.1.143|macos_arm64|7dba8776000b4f75bc9af850cb65b2dc7720ea211733e8cb5243c0b210ef3c19|4194291", + "v2.1.143|macos_x86_64|7dba8776000b4f75bc9af850cb65b2dc7720ea211733e8cb5243c0b210ef3c19|4194291", + "v2.1.143|linux_x86_64|7dba8776000b4f75bc9af850cb65b2dc7720ea211733e8cb5243c0b210ef3c19|4194291", + "v2.1.143|linux_arm64|7dba8776000b4f75bc9af850cb65b2dc7720ea211733e8cb5243c0b210ef3c19|4194291" +] +``` + +The Pex version determines which Pip versions are supported. To see the lists of Pip versions a certain version of Pex supports you can either install that version of Pex as a standalone CLI and run `pex --help`, or examine [pex/pip/version.py](https://github.com/pantsbuild/pex/blob/main/pex/pip/version.py) in the sources of the relevant Pex version. + +The Pip version that Pex uses is determined by the `pip_version` option in Pants. To upgrade the Pip version, update this option value accordingly. For instance, in `pants.toml`, to set the Pip version to be the latest supported by Pex: + +```[python] +pip_version = "latest" +``` diff --git a/src/python/pants/backend/python/goals/lockfile.py b/src/python/pants/backend/python/goals/lockfile.py index 599452bb163..b926a815f91 100644 --- a/src/python/pants/backend/python/goals/lockfile.py +++ b/src/python/pants/backend/python/goals/lockfile.py @@ -119,7 +119,7 @@ async def generate_lockfile( # want to let users change this, as `style=strict` is safer. "--style=universal", "--pip-version", - python_setup.pip_version.value, + python_setup.pip_version, "--resolver-version", "pip-2020-resolver", # PEX files currently only run on Linux and Mac machines; so we hard code this diff --git a/src/python/pants/backend/python/subsystems/setup.py b/src/python/pants/backend/python/subsystems/setup.py index 14670a0e91e..daee3e34c72 100644 --- a/src/python/pants/backend/python/subsystems/setup.py +++ b/src/python/pants/backend/python/subsystems/setup.py @@ -28,20 +28,6 @@ logger = logging.getLogger(__name__) -@enum.unique -class PipVersion(enum.Enum): - V20_3_4 = "20.3.4-patched" - V22_2_2 = "22.2.2" - V22_3 = "22.3" - V22_3_1 = "22.3.1" - V23_0 = "23.0" - V23_0_1 = "23.0.1" - V23_1 = "23.1" - V23_1_1 = "23.1.1" - V23_1_2 = "23.1.2" - LATEST = "latest" - - @enum.unique class InvalidLockfileBehavior(enum.Enum): error = "error" @@ -238,13 +224,16 @@ def interpreter_constraints(self) -> tuple[str, ...]: """ ), ) - pip_version = EnumOption( - default=PipVersion.V23_1_2, + pip_version = StrOption( + default="23.1.2", help=softwrap( - """ + f""" Use this version of Pip for resolving requirements and generating lockfiles. - N.B.: The `latest` value selects the latest of the listed choices which is not + The value used here must be one of the Pip versions supported by the underlying PEX + version. See {doc_url("pex")} for details. + + N.B.: The `latest` value selects the latest of the choices listed by PEX which is not necessarily the latest Pip version released on PyPI. """ ), diff --git a/src/python/pants/backend/python/target_types.py b/src/python/pants/backend/python/target_types.py index 21f4175a149..c8b7421f923 100644 --- a/src/python/pants/backend/python/target_types.py +++ b/src/python/pants/backend/python/target_types.py @@ -710,7 +710,7 @@ class PexBinary(Target): A Python target that can be converted into an executable PEX file. PEX files are self-contained executable files that contain a complete Python environment - capable of running the target. For more information, see {doc_url('pex-files')}. + capable of running the target. For more information, see {doc_url('pex')}. """ ) diff --git a/src/python/pants/backend/python/util_rules/pex_cli.py b/src/python/pants/backend/python/util_rules/pex_cli.py index cafece0e980..1535c44c5a8 100644 --- a/src/python/pants/backend/python/util_rules/pex_cli.py +++ b/src/python/pants/backend/python/util_rules/pex_cli.py @@ -166,9 +166,7 @@ async def setup_pex_cli_process( # All old-style pex runs take the --pip-version flag, but only certain subcommands of the # `pex3` console script do. So if invoked with a subcommand, the caller must selectively # set --pip-version only on subcommands that take it. - pip_version_args = ( - [] if request.subcommand else ["--pip-version", python_setup.pip_version.value] - ) + pip_version_args = [] if request.subcommand else ["--pip-version", python_setup.pip_version] args = [ *request.subcommand, *global_args,