Optimize submodule downloads in CI (#618)

kuznia-rdzeni · Mar 14, 2024 · f53c30e · f53c30e
1 parent 78cb3f5
commit f53c30e
Showing 1 changed file with 59 additions and 27 deletions.
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -61,8 +61,14 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v3
-        with:
-          submodules: recursive
+
+      - name: Get submodules HEAD hash
+        working-directory: .
+        run: |
+          # ownership workaround
+          git config --global --add safe.directory /__w/coreblocks/coreblocks
+          # paths in command are relative!
+          git submodule > .gitmodules-hash
 
       - name: Cache compiled and reference riscv-arch-test
         id: cache-riscv-arch-test
@@ -79,11 +85,17 @@ jobs:
               '**/test/external/riscof/coreblocks/**',
               '**/test/external/riscof/spike_simple/**',
               '**/test/external/riscof/config.ini',
-              '**/.git/modules/test/external/riscof/riscv-arch-test/HEAD',
-              '**/docker/riscv-toolchain.Dockerfile'
+              '**/.gitmodules-hash',
+              '**/docker/riscv-toolchain.Dockerfile',
+              '**/.github/workflows/main.yml'
             ) }}
-          restore-keys: |
-            ${{ env.cache-name }}-${{ runner.os }}-
+          lookup-only: true
+
+      - if: ${{ steps.cache-riscv-arch-test.outputs.cache-hit != 'true' }}
+        name: Checkout with submodules
+        uses: actions/checkout@v3
+        with:
+          submodules: recursive
 
       - if: ${{ steps.cache-riscv-arch-test.outputs.cache-hit != 'true' }}
         name: Setup PATH
@@ -112,7 +124,9 @@ jobs:
         name: Build tests for Coreblocks
         run: |
           MAKEFILE_PATH=riscof_work/Makefile.build-DUT-coreblocks ../../../ci/riscof_run_makefile.sh
+
       - if: ${{ steps.cache-riscv-arch-test.outputs.cache-hit != 'true' }}
+        name: Upload compiled and reference tests artifact
         uses: actions/upload-artifact@v3
         with:
           name: "riscof-tests"
@@ -130,8 +144,6 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v3
-        with:
-          submodules: recursive
 
       - name: Set up Python
         uses: actions/setup-python@v4
@@ -146,11 +158,18 @@ jobs:
           python3 -m pip install -r requirements-dev.txt
 
       - uses: actions/download-artifact@v3
+        name: Download full verilog core
         with:
           name: "verilog-full-core"
           path: .
 
+      - name: Get submodules HEAD hash
+        run: |
+          git config --global --add safe.directory /__w/coreblocks/coreblocks
+          git submodule > .gitmodules-hash
+
       - uses: actions/cache@v3
+        name: Download tests from cache
         env:
           cache-name: cache-riscv-arch-test
         with:
@@ -163,8 +182,9 @@ jobs:
               '**/test/external/riscof/coreblocks/**',
               '**/test/external/riscof/spike_simple/**',
               '**/test/external/riscof/config.ini',
-              '**/.git/modules/test/external/riscof/riscv-arch-test/HEAD',
-              '**/docker/riscv-toolchain.Dockerfile'
+              '**/.gitmodules-hash',
+              '**/docker/riscv-toolchain.Dockerfile',
+              '**/.github/workflows/main.yml'
             ) }}
           fail-on-cache-miss: true
 
@@ -182,13 +202,14 @@ jobs:
     runs-on: ubuntu-latest
     container: ghcr.io/kuznia-rdzeni/riscv-toolchain:2024.03.12
     timeout-minutes: 10
-    outputs:
-        cache_hit: ${{ steps.cache-regression.outputs.cache-hit }}
     steps:
       - name: Checkout
         uses: actions/checkout@v3
-        with:
-          submodules: recursive
+
+      - name: Get submodules HEAD hash
+        run: |
+          git config --global --add safe.directory /__w/coreblocks/coreblocks
+          git submodule > .gitmodules-hash
 
       - name: Cache regression-tests
         id: cache-regression
@@ -197,15 +218,20 @@ jobs:
           cache-name: cache-regression-tests
         with:
           path: test/external/riscv-tests/test-*
-
           key: ${{ env.cache-name }}-${{ runner.os }}-${{ hashFiles(
-              '**/test/external/riscv-tests/environment/**',
+              '**/test/external/riscv-tests/environment/custom/**',
               '**/test/external/riscv-tests/Makefile',
-              '**/.git/modules/test/external/riscv-tests/riscv-tests/HEAD',
-              '**/docker/riscv-toolchain.Dockerfile'
+              '**/.gitmodules-hash',
+              '**/docker/riscv-toolchain.Dockerfile',
+              '**/.github/workflows/main.yml'
             ) }}
-          restore-keys: |
-            ${{ env.cache-name }}-${{ runner.os }}-
+          lookup-only: true
+
+      - if: ${{ steps.cache-regression.outputs.cache-hit != 'true' }}
+        name: Checkout with submodules
+        uses: actions/checkout@v3
+        with:
+          submodules: recursive
 
       - if: ${{ steps.cache-regression.outputs.cache-hit != 'true' }}
         run: cd test/external/riscv-tests && make
@@ -225,8 +251,6 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v3
-        with:
-          submodules: recursive
 
       - name: Set up Python
         uses: actions/setup-python@v4
@@ -241,21 +265,29 @@ jobs:
           python3 -m pip install -r requirements-dev.txt
 
       - uses: actions/download-artifact@v3
+        name: Download full verilog core
         with:
           name: "verilog-full-core"
           path: .
 
+      - name: Get submodules HEAD hash
+        run: |
+          git config --global --add safe.directory /__w/coreblocks/coreblocks
+          git submodule > .gitmodules-hash
+
       - uses: actions/cache@v3
+        name: Download tests from cache
         env:
           cache-name: cache-regression-tests
         with:
           path: test/external/riscv-tests/test-*
           key: ${{ env.cache-name }}-${{ runner.os }}-${{ hashFiles(
-              '**/test/external/riscv-tests/environment/**',
-              '**/test/external/riscv-tests/Makefile',
-              '**/.git/modules/test/external/riscv-tests/riscv-tests/HEAD',
-              '**/docker/riscv-toolchain.Dockerfile'
-              ) }}
+              '**/test/external/riscv-tests/environment/custom/**',
+              '**/test/external/riscv-tests/Makefile', 
+              '**/.gitmodules-hash',
+              '**/docker/riscv-toolchain.Dockerfile',
+              '**/.github/workflows/main.yml'
+            ) }}
           fail-on-cache-miss: true
 
       - name: Run tests