diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
new file mode 100644
index 0000000..96bc8c5
--- /dev/null
+++ b/.github/workflows/benchmark.yml
@@ -0,0 +1,24 @@
+name: Benchmarks
+
+on:
+  # Only run benchmarks for PRs that modify code, dependencies or test inventory
+  pull_request:
+    paths:
+      - 'src/**'
+      - 'benches/**'
+      - 'Cargo.toml'
+      - 'tests/inventory/**'
+
+jobs:
+  benchmarks:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+          profile: minimal
+      - uses: boa-dev/criterion-compare-action@v3
+        with:
+          features: bench
+          branchName: ${{ github.base_ref }}
diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
index 03e6fd5..7a93522 100644
--- a/.github/workflows/rust.yml
+++ b/.github/workflows/rust.yml
@@ -45,17 +45,3 @@ jobs:
       - uses: actions-rs/cargo@v1
         with:
           command: test
-
-  benchmarks:
-    runs-on: ubuntu-latest
-    if: github.base_ref != ''
-    steps:
-      - uses: actions/checkout@v4
-      - uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-          profile: minimal
-      - uses: boa-dev/criterion-compare-action@v3
-        with:
-          features: bench
-          branchName: ${{ github.base_ref }}