Cache toolchain build to speed up testing development

Windows-on-ARM-Experiments · Dec 8, 2023 · 80eecf1 · 80eecf1
1 parent 6a956a7
commit 80eecf1
Show file tree

Hide file tree

Showing 5 changed files with 64 additions and 28 deletions.
diff --git a/.github/scripts/config.sh b/.github/scripts/config.sh
@@ -8,12 +8,14 @@ BINUTILS_VERSION=${BINUTILS_VERSION:-binutils-master}
 
 TARGET=${TARGET:-aarch64-w64-mingw32}
 CRT=${CRT:-msvcrt}
+TOOLCHAIN_NAME=${TOOLCHAIN_NAME:-$TARGET-$CRT}
+
 SOURCE_PATH=${SOURCE_PATH:-$PWD/code}
-BUILD_PATH=${BUILD_PATH:-$PWD/build-$TARGET}
+BUILD_PATH=${BUILD_PATH:-$PWD/build-$TOOLCHAIN_NAME}
 BUILD_MAKE_OPTIONS=${BUILD_MAKE_OPTIONS:-V=1 -j$(nproc)}
-TOOLCHAIN_PATH=${TOOLCHAIN_PATH:-~/cross}
+TOOLCHAIN_PATH=${TOOLCHAIN_PATH:-~/cross-$TOOLCHAIN_NAME}
 TOOLCHAIN_FILE=${TOOLCHAIN_FILE:-$PWD/.github/cmake/$TARGET.cmake}
-TOOLCHAIN_PACKAGE_NAME=${TOOLCHAIN_PACKAGE_NAME:-$TARGET-$CRT-toolchain.tar.gz}
+TOOLCHAIN_PACKAGE_NAME=${TOOLCHAIN_PACKAGE_NAME:-$TOOLCHAIN_NAME-toolchain.tar.gz}
 
 RUN_CONFIG=${RUN_CONFIG:-1}
 RUN_INSTALL=${RUN_INSTALL:-1}

diff --git a/.github/scripts/get-sha.sh b/.github/scripts/get-sha.sh
@@ -0,0 +1,2 @@
+remote=$( git ls-remote -h https://github.com/$1 $2 | tee /dev/fd/2 )
+echo sha=$(echo $remote | cut -d ' ' -f1 | tr -d '[:blank:]' ) >> "$GITHUB_OUTPUT"
diff --git a/.github/scripts/pack-openblas-tests.sh b/.github/scripts/pack-openblas-tests.sh
@@ -4,7 +4,7 @@ source `dirname $0`/config.sh
 
 echo "::group::Pack OpenBLAS"
     cd $SOURCE_PATH/OpenBLAS
-    zip aarch64-msvcrt-openblas-tests.zip ctest/x* ctest/cin* ctest/din* ctest/sin* ctest/zin*
+    zip $TOOLCHAIN_NAME-openblas-tests.zip ctest/x* ctest/cin* ctest/din* ctest/sin* ctest/zin*
 echo "::endgroup::"
 
 echo 'Success!'
diff --git a/.github/scripts/unpack-openblas-tests.ps1 b/.github/scripts/unpack-openblas-tests.ps1
@@ -1,2 +1,2 @@
 Remove-Item $env:OPENBLAS_TESTS_PATH -Recurse -Force -ErrorAction Ignore
-Expand-Archive -LiteralPath aarch64-msvcrt-openblas-tests.zip -DestinationPath $env:OPENBLAS_TESTS_PATH
+Expand-Archive -LiteralPath $env:TOOLCHAIN_NAME-openblas-tests.zip -DestinationPath $env:OPENBLAS_TESTS_PATH
diff --git a/.github/workflows/advanced.yml b/.github/workflows/advanced.yml
@@ -22,6 +22,8 @@ on:
 
 env:
   TOOLCHAIN_PATH: ${{ github.workspace }}/cross
+  TOOLCHAIN_NAME: aarch64-w64-mingw32-msvcrt
+  TOOLCHAIN_PACKAGE_NAME: aarch64-w64-mingw32-msvcrt-toolchain.tar.gz
 
 jobs:
   build-toolchain:
@@ -34,82 +36,118 @@ jobs:
 
     env:
       BINUTILS_REPO: Windows-on-ARM-Experiments/binutils-woarm64
-      BINUTILS_BRANCH: ${{ github.event.inputs.binutils_branch }}
+      BINUTILS_BRANCH: ${{ github.event.inputs.binutils_branch || 'woarm64' }}
       BINUTILS_VERSION: binutils-master
       GCC_REPO: Windows-on-ARM-Experiments/gcc-woarm64
-      GCC_BRANCH: ${{ github.event.inputs.gcc_branch }}
+      GCC_BRANCH: ${{ github.event.inputs.gcc_branch || 'woarm64' }}
       GCC_VERSION: gcc-master
       MINGW_REPO: Windows-on-ARM-Experiments/mingw-woarm64
-      MINGW_BRANCH: ${{ github.event.inputs.mingw_branch }}
+      MINGW_BRANCH: ${{ github.event.inputs.mingw_branch || 'woarm64' }}
       MINGW_VERSION: mingw-w64-master
       TARGET: ${{ matrix.target }}
       CRT: ${{ matrix.crt }}
+      TOOLCHAIN_NAME: ${{ matrix.target }}-${{ matrix.crt }}
       TOOLCHAIN_PACKAGE_NAME: ${{ matrix.target }}-${{ matrix.crt }}-toolchain.tar.gz
 
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
         with:
           path: ${{ github.workspace }}
-
+
+      - name: Get binutils SHA
+        id: binutils-sha
+        run: |
+          .github/scripts/get-sha.sh ${{ env.BINUTILS_REPO }} ${{ env.BINUTILS_BRANCH }}
+
+      - name: Get GCC SHA
+        id: gcc-sha
+        run: |
+          .github/scripts/get-sha.sh ${{ env.GCC_REPO }} ${{ env.GCC_BRANCH }}
+
+      - name: Get MinGW SHA
+        id: mingw-sha
+        run: |
+          .github/scripts/get-sha.sh ${{ env.MINGW_REPO }} ${{ env.MINGW_BRANCH }}
+
+      - name: Cache toolchain
+        id: cache-toolchain
+        uses: actions/cache@v3
+        with:
+          path: ${{ github.workspace }}/${{ env.TOOLCHAIN_PACKAGE_NAME }}
+          key: ${{ env.TOOLCHAIN_NAME }}-${{ steps.binutils-sha.outputs.sha }}-${{ steps.gcc-sha.outputs.sha }}-${{ steps.mingw-sha.outputs.sha }}
+
       - name: Checkout binutils
+        if: steps.cache-toolchain.outputs.cache-hit != 'true'
         uses: actions/checkout@v4
         with:
           repository: ${{ env.BINUTILS_REPO }}
           ref: ${{ env.BINUTILS_BRANCH }}
           path: ${{ github.workspace }}/code/${{ env.BINUTILS_VERSION }}
 
       - name: Checkout GCC
+        if: steps.cache-toolchain.outputs.cache-hit != 'true'
         uses: actions/checkout@v4
         with:
           repository: ${{ env.GCC_REPO }}
           ref: ${{ env.GCC_BRANCH }}
           path: ${{ github.workspace }}/code/${{ env.GCC_VERSION }}
 
       - name: Checkout MinGW
+        if: steps.cache-toolchain.outputs.cache-hit != 'true'
         uses: actions/checkout@v4
         with:
           repository: ${{ env.MINGW_REPO }}
           ref: ${{ env.MINGW_BRANCH }}
           path: ${{ github.workspace }}/code/${{ env.MINGW_VERSION }}
 
       - name: Install dependencies
+        if: steps.cache-toolchain.outputs.cache-hit != 'true'
         run: |
           .github/scripts/install-dependencies.sh
 
       - name: Install libraries
+        if: steps.cache-toolchain.outputs.cache-hit != 'true'
         run: |
           .github/scripts/install-libraries.sh
   
       - name: Build binutils
+        if: steps.cache-toolchain.outputs.cache-hit != 'true'
         run: |
           .github/scripts/build-binutils.sh
 
       - name: Build MinGW headers
+        if: steps.cache-toolchain.outputs.cache-hit != 'true'
         run: |
           .github/scripts/build-mingw-headers.sh
 
       - name: Build GCC
+        if: steps.cache-toolchain.outputs.cache-hit != 'true'
         run: |
           .github/scripts/build-gcc.sh
 
       - name: Build MinGW CRT
+        if: steps.cache-toolchain.outputs.cache-hit != 'true'
         run: |
           .github/scripts/build-mingw-crt.sh
 
       - name: Build libgcc
+        if: steps.cache-toolchain.outputs.cache-hit != 'true'
         run: |
           .github/scripts/build-libgcc.sh
 
       - name: Build MinGW
+        if: steps.cache-toolchain.outputs.cache-hit != 'true'
         run: |
           .github/scripts/build-mingw.sh
 
       - name: Build GCC libs
+        if: steps.cache-toolchain.outputs.cache-hit != 'true'
         run: |
           .github/scripts/build-gcc-libs.sh
 
       - name: Pack toolchain
+        if: steps.cache-toolchain.outputs.cache-hit != 'true'
         run: |
           .github/scripts/pack-toolchain.sh
 
@@ -124,10 +162,6 @@ jobs:
     needs: [build-toolchain]
     runs-on: ubuntu-latest
 
-    env:
-      TARGET: aarch64-w64-mingw32
-      TOOLCHAIN_PACKAGE_NAME: aarch64-w64-mingw32-msvcrt-toolchain.tar.gz
-
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
@@ -148,10 +182,10 @@ jobs:
         run: |
           .github/scripts/build-aarch64-tests.sh
 
-      - name: Upload aarch64-w64-mingw32-msvcrt-tests
+      - name: Upload tests artifact
         uses: actions/upload-artifact@v3
         with:
-          name: aarch64-w64-mingw32-msvcrt-tests
+          name: ${{ env.TOOLCHAIN_NAME }} tests
           path: tests/build/bin/aarch64-mingw-tests.exe
           retention-days: 3
 
@@ -160,12 +194,12 @@ jobs:
     runs-on: [Windows, GCC, ARM64]
 
     steps:
-      - name: Download aarch64-w64-mingw32-msvcrt-tests
+      - name: Download ${{ env.TOOLCHAIN_NAME }} tests
         uses: actions/download-artifact@v3
         with:
-          name: aarch64-w64-mingw32-msvcrt-tests
+          name: ${{ env.TOOLCHAIN_NAME }} tests
 
-      - name: Execute aarch64-mingw-tests
+      - name: Execute ${{ env.TOOLCHAIN_NAME }} tests
         run:
           ${{ github.workspace }}\aarch64-mingw-tests.exe
 
@@ -175,8 +209,6 @@ jobs:
 
     env:
       OPENBLAS_REPO: OpenMathLib/OpenBLAS.git
-      TARGET: aarch64-w64-mingw32
-      TOOLCHAIN_PACKAGE_NAME: aarch64-w64-mingw32-msvcrt-toolchain.tar.gz
 
     steps:
       - name: Checkout repository
@@ -205,15 +237,15 @@ jobs:
         run: |
           .github/scripts/build-openblas.sh
 
-      - name: Pack OpenBLAS
+      - name: Pack OpenBLAS tests
         run: |
           .github/scripts/pack-openblas-tests.sh
   
-      - name: Upload aarch64-msvcrt-openblas-tests
+      - name: Upload tests artifact
         uses: actions/upload-artifact@v3
         with:
-          name: aarch64-msvcrt-openblas-tests
-          path: ${{ github.workspace }}/code/OpenBLAS/aarch64-msvcrt-openblas-tests.zip
+          name: ${{ env.TOOLCHAIN_NAME }}-openblas-tests
+          path: ${{ github.workspace }}/code/OpenBLAS/${{ env.TOOLCHAIN_NAME }}-openblas-tests.zip
           retention-days: 3
 
   execute-openblas-tests:
@@ -229,15 +261,15 @@ jobs:
         with:
           path: ${{ github.workspace }}
 
-      - name: Download aarch64-msvcrt-openblas-tests
+      - name: Download OpenBLAS tests
         uses: actions/download-artifact@v3
         with:
-          name: aarch64-msvcrt-openblas-tests
+          name: ${{ env.TOOLCHAIN_NAME }}-openblas-tests
 
-      - name: Unpack aarch64-msvcrt-openblas-tests.zip
+      - name: Unpack OpenBLAS tests
         run: |
           .github/scripts/unpack-openblas-tests.ps1
   
-      - name: Execute aarch64-mingw-tests
+      - name: Execute OpenBLAS tests
         run: |
           C:\Program` Files\Git\bin\bash.exe .github/scripts/test-openblas.sh