From 3a05c50807eaaca16e953a6694b5290ae2cf9d52 Mon Sep 17 00:00:00 2001 From: Matthew Woehlke Date: Fri, 3 Jan 2025 13:39:32 -0500 Subject: [PATCH] Improve how we manage which PDM to use Move our PDM version pin to a separate file. This both makes it easier to manage updates to the same and also allows the pinned version to be usable outside of the venv_sync script. Add a simple script to update version pins in this file. Teach venv_upgrade to use this to update the PDM version pin in addition to updating the PDM lock file. Also, modify venv_sync to tell PDM to not complain about newer versions. --- setup/python/requirements.txt | 12 ++++++ tools/workspace/python/update_requirements.py | 43 +++++++++++++++++++ tools/workspace/python/venv_sync | 14 +++--- tools/workspace/python/venv_upgrade | 18 +++++++- 4 files changed, 80 insertions(+), 7 deletions(-) create mode 100644 setup/python/requirements.txt create mode 100755 tools/workspace/python/update_requirements.py diff --git a/setup/python/requirements.txt b/setup/python/requirements.txt new file mode 100644 index 000000000000..7674ef2f5d5a --- /dev/null +++ b/setup/python/requirements.txt @@ -0,0 +1,12 @@ +# This file describes does NOT describe the requirements for Drake itself, but +# for setting up the Python virtual environment that Drake will manage. It is +# used by Drake's @python, and also when preparing a virtual environment to +# support a Drake installation. +# +# Specific version pins in this file are updated automatically by +# tools/workspace/python/venv_upgrade. + +# TODO(jeremy.nimmer) Ideally, we also would pin all of the dependencies of +# PDM here, but it's not obvious to me how to do that in a way which is easy to +# upgrade/re-pin over time. +pdm==2.22.0 diff --git a/tools/workspace/python/update_requirements.py b/tools/workspace/python/update_requirements.py new file mode 100755 index 000000000000..67d7521fb085 --- /dev/null +++ b/tools/workspace/python/update_requirements.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 + +import json +import sys +import urllib.request + + +def get_latest_version(name): + r = urllib.request.urlopen(f'https://pypi.org/pypi/{name}/json') + d = json.loads(r.read()) + return d['info']['version'] + + +def update_requirements(requirements_path): + updated = False + + lines = [] + with open(requirements_path, 'r') as f: + for line in f: + line = line.strip() + + if not line.startswith('#') and '==' in line: + name, current = line.split('==') + latest = get_latest_version(name) + if latest != current: + line = f'{name}=={latest}' + updated = True + + lines.append(line) + + if updated: + with open(requirements_path, 'w') as f: + f.write('\n'.join(lines)) + print(requirements_path) + + +def main(args): + for p in args: + update_requirements(p) + + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/tools/workspace/python/venv_sync b/tools/workspace/python/venv_sync index b04f964fb0b2..89da3b0e8822 100755 --- a/tools/workspace/python/venv_sync +++ b/tools/workspace/python/venv_sync @@ -53,18 +53,17 @@ fi # Place the venv(s) in a sibling directory to the output base. That should be a # suitable disk location for build artifacts, but without polluting the actual # output base that Bazel owns. +readonly drake_root="$(cd "$(dirname $0)/../../.." && pwd)" readonly bazel_output_base="$(cd "${repository}/../.." && pwd)" readonly drake_python="${bazel_output_base}.drake_python" mkdir -p "${drake_python}" # Install PDM into a virtual environment. We segregate PDM from the environment # it is managing, so that changes to the managed environment cannot break PDM. +readonly setup="${drake_root}/setup/python" readonly venv_pdm="${drake_python}/venv.pdm" mkvenv "${venv_pdm}" -# TODO(jeremy.nimmer) Ideally, we also would pin all of the dependencies of -# PDM here, but it's not obvious to me how to do that in a way which is easy to -# upgrade/re-pin over time. -"${venv_pdm}/bin/pip" install -U pdm==2.22.0 +"${venv_pdm}/bin/pip" install -U -r "${setup}/requirements.txt" # Don't nag about new versions of PDM; we'll update the above pin when we want # to use something newer, and otherwise want to use the pinned version, so @@ -72,13 +71,16 @@ mkvenv "${venv_pdm}" export PDM_CHECK_UPDATE=0 # Prepare the PDM "project directory". -readonly drake_root="$(cd "$(dirname $0)/../../.." && pwd)" -readonly setup="${drake_root}/setup/python" readonly project="${drake_python}/project" mkdir -p "${project}" ln -nsf "${setup}/pyproject.toml" "${project}/pyproject.toml" ln -nsf "${setup}/pdm.lock" "${project}/pdm.lock" +# Don't nag about new versions of PDM; venv_upgrade will check for those. +# Aside from that, we want to use the pinned version, so being informed about +# newer versions is just noise. +export PDM_CHECK_UPDATE=0 + # Prepare the venv that will hold Drake's requirements. readonly venv_drake="${drake_python}/venv.drake" mkvenv "${venv_drake}" diff --git a/tools/workspace/python/venv_upgrade b/tools/workspace/python/venv_upgrade index cd13ec8c4836..6770907041ef 100755 --- a/tools/workspace/python/venv_upgrade +++ b/tools/workspace/python/venv_upgrade @@ -34,11 +34,21 @@ check_working_tree() { fi } +# Ensure GNU sed is available. +if [ "$(uname)" == "Darwin" ]; then + readonly sed=gsed +else + readonly sed=sed +fi +${sed} --version + # Chdir to the Drake root. cd "$(dirname $0)/../../.." readonly drake_python="$(bazel info output_base).drake_python" readonly project="${drake_python}/project" readonly venv_pdm="${drake_python}/venv.pdm" +readonly python="${venv_pdm}/bin/python" +readonly workspace="$(pwd)/tools/workspace/python" # If committing, check that the working tree is clean (enough). Note that the # lock file is ignored. @@ -47,11 +57,17 @@ readonly venv_pdm="${drake_python}/venv.pdm" # Ensure venv exists. bazel fetch @python --force +# Determine if PDM itself is up to date; if not, update its pin. +readonly venv_requirements="./setup/python/requirements.txt" +files_to_commit+=( "$( + "${python}" "${workspace}/update_requirements.py" "${venv_requirements}" +)" ) + # Remove and recreate the lock file. readonly lockfile="./setup/python/pdm.lock" rm -f "${lockfile}" "${venv_pdm}/bin/pdm" lock -p "${project}" -L "${lockfile}" -files_to_commit+=($(git diff --name-only HEAD -- "${lockfile}")) +files_to_commit+=( "$(git diff --name-only HEAD -- "${lockfile}")" ) # If committing, do the commit. if [ -n "${commit}" ]; then