Cache generic Linux binaries CI, make architecture name handling cons…

…istent
phusion · Oct 13, 2024 · 23da975 · 23da975
1 parent 73698ea
commit 23da975
Show file tree

Hide file tree

Showing 5 changed files with 77 additions and 26 deletions.
diff --git a/.github/workflows/binaries.yml b/.github/workflows/binaries.yml
@@ -18,26 +18,72 @@ jobs:
         include:
           - runner: ubuntu-24.04
             arch: x86_64
+            docker_arch: amd64
           - runner: passenger-ubuntu-24.04-arm64-4cpu
-            arch: arm64
+            arch: aarch64
+            docker_arch: arm64
     runs-on: ${{ matrix.runner }}
-    env:
-      WORKSPACE: ${{ github.workspace }}
-      OUTPUT_DIR: ${{ github.workspace }}/output-linux-${{ matrix.arch }}
-      ARCHITECTURE: ${{ matrix.arch }}
-      CACHE_DIR: ${{ github.workspace }}/cache/linux-${{ matrix.arch }}/executor-${{ github.run_id }}
-      RUNTIME_DIR: ${{ github.workspace }}/cache/linux-${{ matrix.arch }}/executor-${{ github.run_id }}/runtime
     steps:
-    - uses: actions/checkout@v4
-      with:
-        submodules: true
-    - uses: docker/setup-buildx-action@v3
-    - run: ./dev/ci/tests/binaries/build-linux
-    - uses: actions/upload-artifact@v4
-      with:
-        name: binaries-linux-${{ matrix.arch }}
-        path: 'output-linux-${{ matrix.arch }}/**/*'
-    - run: ./dev/ci/tests/binaries/test-linux
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+
+      - run: mkdir cache output
+        working-directory: packaging/binaries/linux
+
+      - name: Fetch cache
+        run: ./dev/ci/fetch-cache-az-blob-storage
+        env:
+          AZURE_STORAGE_CONNECTION_STRING: ${{ secrets.AZURE_CI_STORAGE_CONNECTION_STRING }}
+          CONTAINER_NAME: ${{ vars.SCCACHE_AZURE_BLOB_CONTAINER }}
+          BLOB_NAME: "binaries/linux-binaries-cache-${{ hashFiles('packaging/binaries/linux/docker_image/Dockerfile') }}-${{ matrix.runner }}-${{ matrix.arch }}.tar.zstd"
+          CACHE_PATH: packaging/binaries/linux/cache
+
+      - name: Build
+        run: >
+          ./build
+          -p "$WORKSPACE"
+          -c cache
+          -o output
+          -A "$DOCKER_ARCH"
+          -j 2
+          passenger nginx
+        env:
+          WORKSPACE: ${{ github.workspace }}
+          DOCKER_ARCH: ${{ matrix.docker_arch }}
+        working-directory: packaging/binaries/linux
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: binaries-linux-${{ matrix.arch }}
+          path: packages/binaries/linux/output/**/*
+
+      - name: Update cache
+        run: ./dev/ci/update-cache-az-blob-storage
+        env:
+          AZURE_STORAGE_CONNECTION_STRING: ${{ secrets.AZURE_CI_STORAGE_CONNECTION_STRING }}
+          CONTAINER_NAME: ${{ vars.SCCACHE_AZURE_BLOB_CONTAINER }}
+          BLOB_NAME: "binaries/linux-binaries-cache-${{ hashFiles('packaging/binaries/linux/docker_image/Dockerfile') }}-${{ matrix.runner }}-${{ matrix.arch }}.tar.zstd"
+          CACHE_PATH: packaging/binaries/linux/cache
+        if: always()
+
+      - name: Package
+        run: ./package -i output -o output -a "$ARCHITECTURE"
+        env:
+          ARCHITECTURE: ${{ matrix.arch }}
+        working-directory: packaging/binaries/linux
+
+      - name: Test
+        run: >
+          ./test
+          -p "$WORKSPACE"
+          -i output
+          -I output
+          -A "$DOCKER_ARCH"
+        env:
+          WORKSPACE: ${{ github.workspace }}
+          DOCKER_ARCH: ${{ matrix.docker_arch }}
+        working-directory: packaging/binaries/linux
 
   build_macos:
     name: Binaries macOS ${{ matrix.arch }}

diff --git a/dev/ci/tests/binaries/Jenkinsfile b/dev/ci/tests/binaries/Jenkinsfile
@@ -1,12 +1,13 @@
 JOB_NAME_AS_ID = null
 
-def setupLinuxTest(enablerFlag, architecture, block) {
+def setupLinuxTest(enablerFlag, architecture, dockerArch, block) {
   if (enablerFlag) {
     node("linux && ${architecture}") {
       withEnv([
         "OUTPUT_DIR=${env.WORKSPACE}/output-linux-${architecture}",
         "CACHE_DIR=${env.JENKINS_HOME}/cache/${env.JOB_NAME_AS_ID}/linux-${architecture}/executor-${env.EXECUTOR_NUMBER}",
-        "ARCHITECTURE=${architecture}"
+        "ARCHITECTURE=${architecture}",
+        "DOCKER_ARCH=${dockerArch}"
       ], block)
     }
   } else {
@@ -41,7 +42,7 @@ pipeline {
 
   parameters {
     booleanParam(name: 'LINUX_X86_64', defaultValue: true, description: 'Linux x86_64 binaries')
-    booleanParam(name: 'LINUX_ARM64', defaultValue: true, description: 'Linux arm64 binaries')
+    booleanParam(name: 'LINUX_AARCH64', defaultValue: true, description: 'Linux aarch64 binaries')
     booleanParam(name: 'MACOS', defaultValue: true, description: 'macOS binaries')
   }
 
@@ -73,15 +74,15 @@ pipeline {
         script {
           parallel(
             'Linux x86_64': {
-              setupLinuxTest(params.LINUX_X86_64, 'x86_64') {
+              setupLinuxTest(params.LINUX_X86_64, 'x86_64', 'amd64') {
                 checkout scm
                 sh './dev/ci/tests/binaries/build-linux'
                 archiveArtifacts artifacts: 'output-linux-x86_64/**/*'
                 sh './dev/ci/tests/binaries/test-linux'
               }
             },
-            'Linux arm64': {
-              setupLinuxTest(params.LINUX_ARM64, 'aarch64') {
+            'Linux aarch64': {
+              setupLinuxTest(params.LINUX_AARCH64, 'aarch64', 'arm64') {
                 checkout scm
                 sh './dev/ci/tests/binaries/build-linux'
                 archiveArtifacts artifacts: 'output-linux-aarch64/**/*'

diff --git a/dev/ci/tests/binaries/build-linux b/dev/ci/tests/binaries/build-linux
@@ -6,6 +6,7 @@
 #
 #   WORKSPACE
 #   ARCHITECTURE
+#   DOCKER_ARCH
 #
 # Optional environment variables:
 #
@@ -21,6 +22,7 @@ source "./shared/lib/library.sh"
 
 require_envvar WORKSPACE "$WORKSPACE"
 require_envvar ARCHITECTURE "$ARCHITECTURE"
+require_envvar DOCKER_ARCH "$DOCKER_ARCH"
 
 PASSENGER_ROOT="${PASSENGER_ROOT:-$WORKSPACE}"
 OUTPUT_DIR="${OUTPUT_DIR:-$WORKSPACE/output}"
@@ -33,6 +35,6 @@ run ./linux/build \
 	-p "$PASSENGER_ROOT" \
 	-c "$CACHE_DIR" \
 	-o "$OUTPUT_DIR" \
-	-a "$ARCHITECTURE" \
+	-A "$DOCKER_ARCH" \
 	-j 1 \
 	passenger nginx
diff --git a/dev/ci/tests/binaries/test-linux b/dev/ci/tests/binaries/test-linux
@@ -6,6 +6,7 @@
 #
 #   WORKSPACE
 #   ARCHITECTURE
+#   DOCKER_ARCH
 #
 # Optional environment variables:
 #
@@ -21,6 +22,7 @@ source "./shared/lib/library.sh"
 
 require_envvar WORKSPACE "$WORKSPACE"
 require_envvar ARCHITECTURE "$ARCHITECTURE"
+require_envvar DOCKER_ARCH "$DOCKER_ARCH"
 
 PASSENGER_ROOT="${PASSENGER_ROOT:-$WORKSPACE}"
 OUTPUT_DIR="${OUTPUT_DIR:-$WORKSPACE/output}"
@@ -41,5 +43,5 @@ run ./linux/test \
 	-p "$PASSENGER_ROOT" \
 	-i "$OUTPUT_DIR" \
 	-I "$OUTPUT_DIR" \
-	-a "$ARCHITECTURE" \
+	-A "$DOCKER_ARCH" \
 	"${EXTRA_TEST_PARAMS[@]}"
diff --git a/packaging/binaries b/packaging/binaries