From 682328c555e68033e248b9ea461f5ec92536650d Mon Sep 17 00:00:00 2001 From: David Zbarsky Date: Thu, 5 Oct 2023 15:32:40 -0400 Subject: [PATCH] Use coreutils toolchain for venv creation --- MODULE.bazel | 3 +++ py/private/py_binary.bzl | 12 ++++++++---- py/private/utils.bzl | 1 + py/private/venv/venv.bzl | 8 ++++++-- py/private/venv/venv.tmpl.sh | 24 +++++++++++++----------- 5 files changed, 31 insertions(+), 17 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index 8c319445..946bf37f 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -9,5 +9,8 @@ module( # Lower-bound versions of direct dependencies. # When bumping, add a comment explaining what's required from the newer release. bazel_dep(name = "aspect_bazel_lib", version = "1.33.0") +aspect_bazel_lib_ext = use_extension("@aspect_bazel_lib//lib:extensions.bzl", "ext") +use_repo(aspect_bazel_lib_ext, "coreutils_toolchains") + bazel_dep(name = "bazel_skylib", version = "1.4.2") bazel_dep(name = "rules_python", version = "0.19.0") diff --git a/py/private/py_binary.bzl b/py/private/py_binary.bzl index 676497a9..36f513d9 100644 --- a/py/private/py_binary.bzl +++ b/py/private/py_binary.bzl @@ -5,10 +5,10 @@ load("@aspect_bazel_lib//lib:expand_make_vars.bzl", "expand_locations", "expand_ load("//py/private:py_library.bzl", _py_library = "py_library_utils") load("//py/private:providers.bzl", "PyWheelInfo") load("//py/private:py_wheel.bzl", py_wheel = "py_wheel_lib") -load("//py/private:utils.bzl", "PY_TOOLCHAIN", "SH_TOOLCHAIN", "dict_to_exports", "resolve_toolchain") +load("//py/private:utils.bzl", "COREUTILS_TOOLCHAIN", "PY_TOOLCHAIN", "SH_TOOLCHAIN", "dict_to_exports", "resolve_toolchain") load("//py/private/venv:venv.bzl", _py_venv = "py_venv_utils") -def _py_binary_rule_imp(ctx): +def _py_binary_rule_impl(ctx): bash_bin = ctx.toolchains[SH_TOOLCHAIN].path interpreter = resolve_toolchain(ctx) main = ctx.file.main @@ -115,17 +115,21 @@ _attrs = dict({ "_runfiles_lib": attr.label( default = "@bazel_tools//tools/bash/runfiles", ), + "_coreutils_toolchain": attr.label( + default = "@coreutils_toolchains//:resolved_toolchain", + ), }) _attrs.update(**_py_venv.attrs) _attrs.update(**_py_library.attrs) py_base = struct( - implementation = _py_binary_rule_imp, + implementation = _py_binary_rule_impl, attrs = _attrs, toolchains = [ - SH_TOOLCHAIN, + COREUTILS_TOOLCHAIN, PY_TOOLCHAIN, + SH_TOOLCHAIN, ], ) diff --git a/py/private/utils.bzl b/py/private/utils.bzl index 935822f5..f8a22585 100644 --- a/py/private/utils.bzl +++ b/py/private/utils.bzl @@ -1,4 +1,5 @@ "General utilities for building python rules." +COREUTILS_TOOLCHAIN = "@aspect_bazel_lib//lib:coreutils_toolchain_type" PY_TOOLCHAIN = "@bazel_tools//tools/python:toolchain_type" SH_TOOLCHAIN = "@bazel_tools//tools/sh:toolchain_type" diff --git a/py/private/venv/venv.bzl b/py/private/venv/venv.bzl index c6fbc4dd..788c9636 100644 --- a/py/private/venv/venv.bzl +++ b/py/private/venv/venv.bzl @@ -3,7 +3,7 @@ load("@aspect_bazel_lib//lib:paths.bzl", "BASH_RLOCATION_FUNCTION", "to_manifest_path") load("//py/private:providers.bzl", "PyWheelInfo") load("//py/private:py_library.bzl", _py_library = "py_library_utils") -load("//py/private:utils.bzl", "PY_TOOLCHAIN", "SH_TOOLCHAIN", "resolve_toolchain") +load("//py/private:utils.bzl", "COREUTILS_TOOLCHAIN", "PY_TOOLCHAIN", "SH_TOOLCHAIN", "resolve_toolchain") def _wheel_path_map(file): return file.path @@ -100,6 +100,7 @@ def _make_venv(ctx, name = None, strip_pth_workspace_root = None): content = pth_lines, ) + coreutils = ctx.toolchains[COREUTILS_TOOLCHAIN] venv_directory = ctx.actions.declare_directory("%s.source" % name) common_substitutions = { @@ -113,6 +114,7 @@ def _make_venv(ctx, name = None, strip_pth_workspace_root = None): "{{PYTHON_INTERPRETER_PATH}}": interpreter.python.path, "{{VENV_LOCATION}}": venv_directory.path, "{{USE_MANIFEST_PATH}}": "false", + "{{COREUTILS_BIN}}": coreutils.coreutils_info.bin.path, } make_venv_for_action_sh = ctx.actions.declare_file(name + "_venv.sh") @@ -150,6 +152,7 @@ def _make_venv(ctx, name = None, strip_pth_workspace_root = None): inputs = venv_creation_depset, command = make_venv_for_action_sh.path, tools = [ + ctx.attr._coreutils_toolchain.files_to_run, interpreter.files, ], progress_message = "Creating virtual environment for %{label}", @@ -202,8 +205,9 @@ _common_attrs = dict({ }) _toolchains = [ - SH_TOOLCHAIN, + COREUTILS_TOOLCHAIN, PY_TOOLCHAIN, + SH_TOOLCHAIN, ] _attrs = dict({ diff --git a/py/private/venv/venv.tmpl.sh b/py/private/venv/venv.tmpl.sh index 3df042e0..fd6d65a9 100644 --- a/py/private/venv/venv.tmpl.sh +++ b/py/private/venv/venv.tmpl.sh @@ -62,14 +62,16 @@ export VIRTUAL_ENV_DISABLE_PROMPT=1 . "${VBIN_LOCATION}/activate" unset VIRTUAL_ENV_DISABLE_PROMPT +COREUTILS_BIN="{{COREUTILS_BIN}}" + # Now symlink in pip from the toolchain # Python venv will also link `pip3.x`, but this seems unnecessary for this use -ln -snf "${PIP_LOCATION}" "${VPIP_LOCATION}" +${COREUTILS_BIN} ln -snf "${PIP_LOCATION}" "${VPIP_LOCATION}" # Need to symlink in the pip site-packages folder not just the binary. # Ask Python where the site-packages folder is and symlink the pip package in from the toolchain VENV_SITE_PACKAGES=$(${VPYTHON} -c 'import site; print(site.getsitepackages()[0])') -ln -snf "${PYTHON_SITE_PACKAGES}/pip" "${VENV_SITE_PACKAGES}/pip" +${COREUTILS_BIN} ln -snf "${PYTHON_SITE_PACKAGES}/pip" "${VENV_SITE_PACKAGES}/pip" # If the incoming requirements file has setuptools the skip creating a symlink to our own as they will cause # error when installing. @@ -79,9 +81,9 @@ HAS_SETUPTOOLS=$? set -o errexit if [ ${HAS_SETUPTOOLS} -gt 0 ]; then - ln -snf "${PYTHON_SITE_PACKAGES}/_distutils_hack" "${VENV_SITE_PACKAGES}/_distutils_hack" + ${COREUTILS_BIN} ln -snf "${PYTHON_SITE_PACKAGES}/_distutils_hack" "${VENV_SITE_PACKAGES}/_distutils_hack" - ln -snf "${PYTHON_SITE_PACKAGES}/setuptools" "${VENV_SITE_PACKAGES}/setuptools" + ${COREUTILS_BIN} ln -snf "${PYTHON_SITE_PACKAGES}/setuptools" "${VENV_SITE_PACKAGES}/setuptools" fi INSTALL_WHEELS={{INSTALL_WHEELS}} @@ -115,28 +117,28 @@ fi # The .pth file adds to the interpreters sys.path, without having to set `PYTHONPATH`. This allows us to still # run with the interpreter with the `-I` flag. This stops some import mechanisms breaking out the sandbox by using # relative imports. -cat "${PTH_FILE}" > "${VENV_SITE_PACKAGES}/first_party.pth" +${COREUTILS_BIN} cat "${PTH_FILE}" > "${VENV_SITE_PACKAGES}/first_party.pth" # Remove the cfg file as it contains absolute paths. # The entrypoint script for py_binary and py_test will create a new one. # For local venvs, we'll create a new one below. PYVENV_CFG="${VENV_LOCATION}/pyvenv.cfg" -rm "${PYVENV_CFG}" +${COREUTILS_BIN} rm "${PYVENV_CFG}" if [ "$USE_MANIFEST_PATH" = false ]; then # Tear down the symlinks created above as these won't be able to be resolved by bazel when validating the TreeArtifact. - find "${VENV_LOCATION}" -type l -exec rm {} + + ${COREUTILS_BIN} find "${VENV_LOCATION}" -type l -exec ${COREUTILS_BIN} rm {} + fi if [ "$USE_MANIFEST_PATH" = true ]; then # If we are in a 'bazel run' then remove the symlinks to the execroot Python and replace them with a link to external - rm ${VBIN_LOCATION}/python* + ${COREUTILS_BIN} rm ${VBIN_LOCATION}/python* - ln -snf "${REAL_PYTHON_LOCATION}" "${VBIN_LOCATION}/python" - ln -snf "${VBIN_LOCATION}/python" "${VBIN_LOCATION}/python3" + ${COREUTILS_BIN} ln -snf "${REAL_PYTHON_LOCATION}" "${VBIN_LOCATION}/python" + ${COREUTILS_BIN} ln -snf "${VBIN_LOCATION}/python" "${VBIN_LOCATION}/python3" PYTHON_SYMLINK_VERSION_SUFFIX=$(${PYTHON} -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")') - ln -snf "${VBIN_LOCATION}/python" "${VBIN_LOCATION}/python${PYTHON_SYMLINK_VERSION_SUFFIX}" + ${COREUTILS_BIN} ln -snf "${VBIN_LOCATION}/python" "${VBIN_LOCATION}/python${PYTHON_SYMLINK_VERSION_SUFFIX}" PYTHON_VERSION=$(${PYTHON} -c 'import platform; print(platform.python_version())') echo "home = ${VBIN_LOCATION}" > "${PYVENV_CFG}"