diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml
index 9089fd980c..a21353f8d9 100644
--- a/.github/workflows/build-test.yml
+++ b/.github/workflows/build-test.yml
@@ -20,25 +20,27 @@ jobs:
                 build_type: [ 'Debug', 'Release' ]
                 cloud_provider: [ 'AWS', 'AZURE', 'GCP' ]
         steps:
+            - uses: actions/checkout@v1
             - name: Restore cached deps
               id: cache-restore-deps
               uses: actions/cache/restore@v4
               with:
                 path: |
                   dep-cache/
-                key: ${{ matrix.build_type }}-dep-cache
-            - uses: actions/checkout@v1
+                key: ${{ matrix.build_type }}-${{ github.event.pull_request.base.sha }}-dep-cache
+              if: github.event_name == 'pull_request'
             - name: Build
               shell: bash
               env:
                   BUILD_TYPE: ${{ matrix.build_type }}
               run: ci/build_linux.sh
-            - name: Save deps
+            - name: Cache deps
               id: cache-save-deps
               uses: actions/cache/save@v4
               with:
                 path: dep-cache
-                key: ${{ steps.cache-restore-deps.outputs.cache-primary-key }}
+                key: ${{ matrix.build_type }}-${{ github.sha }}-dep-cache
+              if: github.ref_name == github.event.repository.default_branch
             - uses: actions/setup-python@v1
               with:
                 python-version: '3.7'