Improve how we manage which PDM to use

Move our PDM version pin to a separate file. This both makes it easier
to manage updates to the same and also allows the pinned version to be
usable outside of the venv_sync script. Teach venv_upgrade to use PDM
itself to generate this file, which accomplishes the outstanding goal of
being able to pin both PDM and its dependencies.  Also, modify venv_sync
to tell PDM to not complain about newer versions.

setup/BUILD.bazel

@@ -13,6 +13,7 @@ exports_files([
     "mac/source_distribution/Brewfile-maintainer-only",
     "python/pdm.lock",
     "python/pyproject.toml",
+    "python/requirements.txt",
     "ubuntu/binary_distribution/packages-jammy.txt",
     "ubuntu/source_distribution/packages-jammy.txt",
     "ubuntu/source_distribution/packages-jammy-clang.txt",

setup/python/pyproject.toml

@@ -20,6 +20,9 @@ dependencies = [
 ]
 
 [dependency-groups]
+pdm = [
+    "pdm",
+]
 test = [
     "flask",
     "six",

setup/python/requirements.txt

@@ -0,0 +1,12 @@
+# This file does NOT describe the requirements for Drake itself, but for
+# setting up the Python virtual environment that Drake will manage. It is used
+# by Drake's @python, and also when preparing a virtual environment to support
+# a Drake installation.
+#
+# This file will be generated by tools/workspace/python/venv_upgrade.
+#
+# The present contents are adequate to bootstrap the PDM installation, after
+# which (the first subsequent time venv_upgrade is run) PDM itself will
+# generate a complete list of its dependencies with pinned versions.
+
+pdm==2.22.0

setup/python/requirements.txt.in

@@ -0,0 +1,13 @@
+# This file does NOT describe the requirements for Drake itself, but for
+# setting up the Python virtual environment that Drake will manage. It is used
+# by Drake's @python, and also when preparing a virtual environment to support
+# a Drake installation.
+#
+# This file will be generated by tools/workspace/python/venv_upgrade.
+#
+# This version serves two purposes. First, it can be used to bootstrap the PDM
+# installation (the initial versions of PDM and dependencies will not be
+# pinned). Second, the above text is used as the template for the generated
+# version.
+
+pdm

tools/workspace/python/repository.bzl

@@ -114,9 +114,13 @@ def _prepare_venv(repo_ctx, python):
     if os_name != "mac os x":
         return python
 
-    # Locate the lock file and mark it to be monitored for changes.
-    pylock = repo_ctx.path(Label("@drake//setup:python/pdm.lock")).realpath
-    repo_ctx.watch(pylock)
+    # Locate lock files and mark them to be monitored for changes.
+    requirements = repo_ctx.path(
+        Label("@drake//setup:python/requirements.txt"),
+    ).realpath
+    pdmlock = repo_ctx.path(Label("@drake//setup:python/pdm.lock")).realpath
+    repo_ctx.watch(requirements)
+    repo_ctx.watch(pdmlock)
 
     # Choose which dependencies to install.
     if repo_ctx.attr.requirements_flavor == "test":

tools/workspace/python/venv_sync

@@ -53,32 +53,34 @@ fi
 # Place the venv(s) in a sibling directory to the output base. That should be a
 # suitable disk location for build artifacts, but without polluting the actual
 # output base that Bazel owns.
+readonly drake_root="$(cd "$(dirname $0)/../../.." && pwd)"
 readonly bazel_output_base="$(cd "${repository}/../.." && pwd)"
 readonly drake_python="${bazel_output_base}.drake_python"
 mkdir -p "${drake_python}"
 
 # Install PDM into a virtual environment. We segregate PDM from the environment
 # it is managing, so that changes to the managed environment cannot break PDM.
+readonly setup="${drake_root}/setup/python"
 readonly venv_pdm="${drake_python}/venv.pdm"
 mkvenv "${venv_pdm}"
-# TODO(jeremy.nimmer) Ideally, we also would pin all of the dependencies of
-# PDM here, but it's not obvious to me how to do that in a way which is easy to
-# upgrade/re-pin over time.
-"${venv_pdm}/bin/pip" install -U pdm==2.22.0
+"${venv_pdm}/bin/pip" install -U -r "${setup}/requirements.txt"
 
 # Don't nag about new versions of PDM; we'll update the above pin when we want
 # to use something newer, and otherwise want to use the pinned version, so
 # being informed about newer versions is just noise.
 export PDM_CHECK_UPDATE=0
 
 # Prepare the PDM "project directory".
-readonly drake_root="$(cd "$(dirname $0)/../../.." && pwd)"
-readonly setup="${drake_root}/setup/python"
 readonly project="${drake_python}/project"
 mkdir -p "${project}"
 ln -nsf "${setup}/pyproject.toml" "${project}/pyproject.toml"
 ln -nsf "${setup}/pdm.lock" "${project}/pdm.lock"
 
+# Don't nag about new versions of PDM; venv_upgrade will check for those.
+# Aside from that, we want to use the pinned version, so being informed about
+# newer versions is just noise.
+export PDM_CHECK_UPDATE=0
+
 # Prepare the venv that will hold Drake's requirements.
 readonly venv_drake="${drake_python}/venv.drake"
 mkvenv "${venv_drake}"

tools/workspace/python/venv_upgrade

@@ -3,13 +3,23 @@
 # Drake script to upgrade our requirements lockfile.
 # - Users should make edits to pyproject.toml, and then
 # - run this script to compile pyproject.toml to pdm.lock.
+#
+# The --reuse option may be given to apply changes to pyproject.toml while
+# attempting to preserve existing version pins.
+#
+# This script should also be run periodically to update pinned Python packages
+# to the latest available versions.
 
 set -eux -o pipefail
 
 commit=
 files_to_commit=()
+lock_args=()
 while [ "${1:-}" != "" ]; do
     case "$1" in
+        --reuse)
+            lock_args+=( --update-reuse )
+            ;;
         --commit)
             commit=1
             ;;
@@ -34,11 +44,19 @@ check_working_tree() {
     fi
 }
 
+update_requirements() {
+    sed $'/will be generated/{s/will be/is/\nq\n}' < "$1.in" > "$1"
+    "${venv_pdm}/bin/pdm" export -p "${project}" -G pdm --no-default |
+        grep -vE '^#' >> "$1"
+}
+
 # Chdir to the Drake root.
 cd "$(dirname $0)/../../.."
 readonly drake_python="$(bazel info output_base).drake_python"
 readonly project="${drake_python}/project"
 readonly venv_pdm="${drake_python}/venv.pdm"
+readonly python="${venv_pdm}/bin/python"
+readonly workspace="$(pwd)/tools/workspace/python"
 
 # If committing, check that the working tree is clean (enough). Note that the
 # lock file is ignored.
@@ -47,11 +65,27 @@ readonly venv_pdm="${drake_python}/venv.pdm"
 # Ensure venv exists.
 bazel fetch @python --force
 
-# Remove and recreate the lock file.
+# Update the lock file.
 readonly lockfile="./setup/python/pdm.lock"
-rm -f "${lockfile}"
-"${venv_pdm}/bin/pdm" lock -p "${project}" -L "${lockfile}"
-files_to_commit+=($(git diff --name-only HEAD -- "${lockfile}"))
+lock_args+=( -d -p "${project}" -L "${lockfile}" )
+"${venv_pdm}/bin/pdm" lock ${lock_args[@]}
+
+# Update pins for PDM itself
+readonly requirements="./setup/python/requirements.txt"
+update_requirements "${requirements}"
+files_to_commit+=( "$(git diff --name-only HEAD -- "${requirements}")" )
+
+if [ ${#files_to_commit[@]} -gt 0 ]; then
+    # PDM needs to be updated.
+    bazel fetch @python
+
+    # Update the lock files (again).
+    update_requirements "${requirements}"
+    "${venv_pdm}/bin/pdm" lock ${lock_args[@]}
+fi
+
+# Check for changes to the lock file.
+files_to_commit+=( "$(git diff --name-only HEAD -- "${lockfile}")" )
 
 # If committing, do the commit.
 if [ -n "${commit}" ]; then