From acf6b871e6380626ed60c9f227bfec149617aa64 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Sat, 9 Nov 2024 07:13:07 +0900 Subject: [PATCH] Enable VCPKG_BINARY_SOURCES explicitly --- ci/docker/python-wheel-manylinux.dockerfile | 4 +- ci/scripts/install_vcpkg.sh | 53 +++++++++------------ dev/tasks/java-jars/github.yml | 6 +++ dev/tasks/python-wheels/github.linux.yml | 6 +++ docker-compose.yml | 1 + 5 files changed, 39 insertions(+), 31 deletions(-) diff --git a/ci/docker/python-wheel-manylinux.dockerfile b/ci/docker/python-wheel-manylinux.dockerfile index cdaf34ac0b2ba..8e5989d6c712c 100644 --- a/ci/docker/python-wheel-manylinux.dockerfile +++ b/ci/docker/python-wheel-manylinux.dockerfile @@ -68,6 +68,7 @@ RUN /arrow/ci/scripts/install_ccache.sh ${ccache} /usr/local # Install vcpkg ARG GITHUB_REPOSITORY_OWNER ARG GITHUB_TOKEN +ARG VCPKG_BINARY_SOURCES ARG vcpkg COPY ci/vcpkg/*.patch \ ci/vcpkg/*linux*.cmake \ @@ -79,12 +80,13 @@ ARG build_type=release ENV CMAKE_BUILD_TYPE=${build_type} \ GITHUB_REPOSITORY_OWNER="${GITHUB_REPOSITORY_OWNER}" \ GITHUB_TOKEN="${GITHUB_TOKEN}" \ - VCPKG_BINARY_SOURCES="clear;nuget,GitHub,readwrite" \ + VCPKG_BINARY_SOURCES="${VCPKG_BINARY_SOURCES}" \ VCPKG_DEFAULT_TRIPLET=${arch_short}-linux-static-${build_type} \ VCPKG_FEATURE_FLAGS="manifests" \ VCPKG_FORCE_SYSTEM_BINARIES=1 \ VCPKG_OVERLAY_TRIPLETS=/arrow/ci/vcpkg +# TODO: Use --mount=type=secret for GITHUB_TOKEN RUN arrow/ci/scripts/install_vcpkg.sh ${VCPKG_ROOT} ${vcpkg} ENV PATH="${PATH}:${VCPKG_ROOT}" diff --git a/ci/scripts/install_vcpkg.sh b/ci/scripts/install_vcpkg.sh index 8826a22a1eb91..a1bc602403976 100755 --- a/ci/scripts/install_vcpkg.sh +++ b/ci/scripts/install_vcpkg.sh @@ -53,40 +53,33 @@ if [ -f "${vcpkg_ports_patch}" ]; then echo "Patch successfully applied to the VCPKG port files!" fi -if [ -n "${GITHUB_TOKEN:-}" ] && [ -n "${GITHUB_REPOSITORY_OWNER:-}" ]; then - can_use_nuget=yes +if [ -n "${GITHUB_TOKEN:-}" ] && \ + [ -n "${GITHUB_REPOSITORY_OWNER:-}" ] && \ + [ -n "${VCPKG_BINARY_SOURCES:-}" ] ; then if type dnf 2>/dev/null; then dnf install -y epel-release - if dnf info mono-complete 2>/dev/null; then - dnf install -y mono-complete - curl \ - --location \ - --output "${vcpkg_destination}/nuget" \ - https://dist.nuget.org/win-x86-commandline/latest/nuget.exe - else - # manylinux2014_aarch64 image doesn't have mono-complete in - # EPEL. It has Mono but it's old. - can_use_nuget=no - fi + dnf install -y mono-complete + curl \ + --location \ + --output "${vcpkg_destination}/nuget" \ + https://dist.nuget.org/win-x86-commandline/latest/nuget.exe fi - if [ "${can_use_nuget}" = "yes" ]; then - PATH="${vcpkg_destination}:${PATH}" - nuget_url="https://nuget.pkg.github.com/${GITHUB_REPOSITORY_OWNER}/index.json" - nuget="$(vcpkg fetch nuget | tail -n 1)" - if type mono 2>/dev/null; then - nuget="mono ${nuget}" - fi - ${nuget} \ - sources add \ - -source "${nuget_url}" \ - -storepasswordincleartext \ - -name "GitHub" \ - -username "${GITHUB_REPOSITORY_OWNER}" \ - -password "${GITHUB_TOKEN}" - ${nuget} \ - setapikey "${GITHUB_TOKEN}" \ - -source "${nuget_url}" + PATH="${vcpkg_destination}:${PATH}" + nuget_url="https://nuget.pkg.github.com/${GITHUB_REPOSITORY_OWNER}/index.json" + nuget="$(vcpkg fetch nuget | tail -n 1)" + if type mono 2>/dev/null; then + nuget="mono ${nuget}" fi + ${nuget} \ + sources add \ + -source "${nuget_url}" \ + -storepasswordincleartext \ + -name "GitHub" \ + -username "${GITHUB_REPOSITORY_OWNER}" \ + -password "${GITHUB_TOKEN}" + ${nuget} \ + setapikey "${GITHUB_TOKEN}" \ + -source "${nuget_url}" fi popd diff --git a/dev/tasks/java-jars/github.yml b/dev/tasks/java-jars/github.yml index 13faa2f936cc6..755fe7762d887 100644 --- a/dev/tasks/java-jars/github.yml +++ b/dev/tasks/java-jars/github.yml @@ -56,6 +56,12 @@ jobs: {{ macros.github_set_sccache_envvars()|indent(8) }} GITHUB_TOKEN: {{ '${{ secrets.GITHUB_TOKEN }}' }} run: | + if [ "${ARCH}" = "aarch_64" ]; then + # We can't use NuGet on manylinux2014_aarch64 because Mono is old. + : + else + export VCPKG_BINARY_SOURCES="clear;nuget,GitHub,readwrite" + fi archery docker run \ -e ARROW_JAVA_BUILD=OFF \ -e ARROW_JAVA_TEST=OFF \ diff --git a/dev/tasks/python-wheels/github.linux.yml b/dev/tasks/python-wheels/github.linux.yml index c1c856c3c465b..dccdddfb0e43d 100644 --- a/dev/tasks/python-wheels/github.linux.yml +++ b/dev/tasks/python-wheels/github.linux.yml @@ -55,6 +55,12 @@ jobs: env: GITHUB_TOKEN: {{ '${{ secrets.GITHUB_TOKEN }}' }} run: | + if [ "{{ manylinux_version }}" = "2014" ] && [ "{{ arch }}" = "arm64" ]; then + # We can't use NuGet on manylinux2014_aarch64 because Mono is old. + : + else + export VCPKG_BINARY_SOURCES="clear;nuget,GitHub,readwrite" + fi archery docker run \ -e SETUPTOOLS_SCM_PRETEND_VERSION={{ arrow.no_rc_version }} \ python-wheel-manylinux-{{ manylinux_version }} diff --git a/docker-compose.yml b/docker-compose.yml index dc0e38bf369ca..b9271c7d76795 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -78,6 +78,7 @@ x-vcpkg-build-args: &vcpkg-build-args vcpkg: ${VCPKG} GITHUB_REPOSITORY_OWNER: GITHUB_TOKEN: + VCPKG_BINARY_SOURCES: # CPU/memory limit presets to pass to Docker. #