GH-43951: [CI][Python] Use GitHub Packages for vcpkg cache

apache · Nov 5, 2024 · c931193 · c931193
1 parent 4274db8
commit c931193
Show file tree

Hide file tree

Showing 4 changed files with 46 additions and 13 deletions.
diff --git a/ci/docker/python-wheel-manylinux.dockerfile b/ci/docker/python-wheel-manylinux.dockerfile
@@ -66,6 +66,8 @@ COPY ci/scripts/install_ccache.sh arrow/ci/scripts/
 RUN /arrow/ci/scripts/install_ccache.sh ${ccache} /usr/local
 
 # Install vcpkg
+ARG GITHUB_ACTOR
+ARG GITHUB_TOKEN
 ARG vcpkg
 COPY ci/vcpkg/*.patch \
      ci/vcpkg/*linux*.cmake \
@@ -75,10 +77,13 @@ COPY ci/scripts/install_vcpkg.sh \
 ENV VCPKG_ROOT=/opt/vcpkg
 ARG build_type=release
 ENV CMAKE_BUILD_TYPE=${build_type} \
-    VCPKG_FORCE_SYSTEM_BINARIES=1 \
-    VCPKG_OVERLAY_TRIPLETS=/arrow/ci/vcpkg \
+    GITHUB_ACTOR="${GITHUB_ACTOR}" \
+    GITHUB_TOKEN="${GITHUB_TOKEN}" \
+    VCPKG_BINARY_SOURCES="clear;nuget,https://nuget.pkg.github.com/${GITHUB_ACTOR}/index.json,readwrite" \
     VCPKG_DEFAULT_TRIPLET=${arch_short}-linux-static-${build_type} \
-    VCPKG_FEATURE_FLAGS="manifests"
+    VCPKG_FEATURE_FLAGS="manifests" \
+    VCPKG_FORCE_SYSTEM_BINARIES=1 \
+    VCPKG_OVERLAY_TRIPLETS=/arrow/ci/vcpkg
 
 RUN arrow/ci/scripts/install_vcpkg.sh ${VCPKG_ROOT} ${vcpkg}
 ENV PATH="${PATH}:${VCPKG_ROOT}"
@@ -93,7 +98,7 @@ RUN vcpkg install \
         --clean-after-build \
         --x-install-root=${VCPKG_ROOT}/installed \
         --x-manifest-root=/arrow/ci/vcpkg \
-        --x-feature=azure \ 
+        --x-feature=azure \
         --x-feature=flight \
         --x-feature=gcs \
         --x-feature=json \

diff --git a/ci/scripts/install_vcpkg.sh b/ci/scripts/install_vcpkg.sh
@@ -17,7 +17,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-set -e
+set -eu
 
 if [ "$#" -lt 1 ]; then
   echo "Usage: $0 ``<target-directory> [<vcpkg-version> [<vcpkg-ports-patch>]]"
@@ -42,7 +42,7 @@ pushd ${vcpkg_destination}
 
 git checkout "${vcpkg_version}"
 
-if [[ "$OSTYPE" == "msys" ]]; then
+if [[ "${OSTYPE:-}" == "msys" ]]; then
   ./bootstrap-vcpkg.bat -disableMetrics
 else
   ./bootstrap-vcpkg.sh -disableMetrics
@@ -53,4 +53,19 @@ if [ -f "${vcpkg_ports_patch}" ]; then
   echo "Patch successfully applied to the VCPKG port files!"
 fi
 
+if [ -n "${GITHUB_TOKEN:-}" ] && [ -n "${GITHUB_ACTOR:-}" ]; then
+  PATH="${PATH}:${VCPKG_ROOT}"
+  nuget_url="https://nuget.pkg.github.com/${GITHUB_ACTOR}/index.json"
+  "$(vcpkg fetch nuget)" \
+    sources add \
+    -Source "${nuget_url}" \
+    -StorePasswordInClearText \
+    -Name GitHubPackages \
+    -UserName "${GITHUB_ACTOR}" \
+    -Password "${GITHUB_TOKEN}"
+  "$(vcpkg fetch nuget)" \
+    setapikey "${GITHUB_TOKEN}" \
+    -Source "${nuget_url}"
+fi
+
 popd
diff --git a/dev/tasks/python-wheels/github.linux.yml b/dev/tasks/python-wheels/github.linux.yml
@@ -19,6 +19,9 @@
 
 {{ macros.github_header() }}
 
+permissions:
+  packages: write
+
 jobs:
   build:
     name: "Build wheel for manylinux {{ manylinux_version }}"
@@ -49,7 +52,12 @@ jobs:
 
       - name: Build wheel
         shell: bash
-        run: archery docker run -e SETUPTOOLS_SCM_PRETEND_VERSION={{ arrow.no_rc_version }} python-wheel-manylinux-{{ manylinux_version }}
+        run: |
+          archery docker run \
+            -e SETUPTOOLS_SCM_PRETEND_VERSION={{ arrow.no_rc_version }} \
+            python-wheel-manylinux-{{ manylinux_version }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
       - uses: actions/upload-artifact@v4
         with:

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -53,26 +53,30 @@
 #
 # See more in cpp/build-support/run-test.sh::print_coredumps
 
-x-common: &common
-  GITHUB_ACTIONS:
-
 x-ccache: &ccache
   CCACHE_COMPILERCHECK: content
   CCACHE_COMPRESS: 1
   CCACHE_COMPRESSLEVEL: 6
   CCACHE_MAXSIZE: 1G
   CCACHE_DIR: /ccache
 
+x-common: &common
+  GITHUB_ACTIONS:
+
+x-cpp: &cpp
+  ARROW_RUNTIME_SIMD_LEVEL:
+  ARROW_SIMD_LEVEL:
+
 x-sccache: &sccache
   AWS_ACCESS_KEY_ID:
   AWS_SECRET_ACCESS_KEY:
   SCCACHE_BUCKET:
   SCCACHE_REGION:
   SCCACHE_S3_KEY_PREFIX: ${SCCACHE_S3_KEY_PREFIX:-sccache}
 
-x-cpp: &cpp
-  ARROW_RUNTIME_SIMD_LEVEL:
-  ARROW_SIMD_LEVEL:
+x-vcpkg: &vcpkg
+  GITHUB_ACTOR:
+  GITHUB_TOKEN:
 
 # CPU/memory limit presets to pass to Docker.
 #
@@ -1144,6 +1148,7 @@ services:
     image: ${REPO}:${ARCH}-python-${PYTHON}-wheel-manylinux-2-28-vcpkg-${VCPKG}
     build:
       args:
+        <<: [*vcpkg]
         arch: ${ARCH}
         arch_short: ${ARCH_SHORT}
         base: quay.io/pypa/manylinux_2_28_${ARCH_ALIAS}:2024-08-03-32dfa47