diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index d2ee69b22..78fb8b7e2 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -32,7 +32,7 @@ jobs:
         with:
           path: test/external/riscv-tests/test-*
           key: ${{ env.cache-name }}-${{ runner.os }}-${{
-            job.container.id }}-${{ hashFiles('**/test/external/riscv-tests') }}
+            job.container.id }}-${{ hashFiles('**/test/external/riscv-tests/{environment/**,Makefile}')' }}
           restore-keys: |
             ${{ env.cache-name }}-${{ runner.os }}-${{ job.container.id }}-
 
@@ -75,7 +75,7 @@ jobs:
         with:
           path: test/external/riscv-tests/test-*
           key: ${{ env.cache-name }}-${{ runner.os }}-${{
-            needs.build-regression-tests.outputs.container_id }}-${{ hashFiles('**/test/external/riscv-tests') }}
+            needs.build-regression-tests.outputs.container_id }}-${{ hashFiles('**/test/external/riscv-tests/{environment/**,Makefile}') }}
           fail-on-cache-miss: true
 
       - name: Run tests