diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml
index c0d07fef80..2e6a6bcfcf 100644
--- a/.github/workflows/backport.yml
+++ b/.github/workflows/backport.yml
@@ -1,16 +1,26 @@
 name: Backport
 on:
-  pull_request:
+  pull_request_target:
     types:
       - closed
       - labeled
 
 jobs:
   backport:
-    runs-on: ubuntu-latest
     name: Backport
+    runs-on: ubuntu-latest
+    # Only react to merged PRs for security reasons.
+    # See https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request_target.
+    if: >
+      github.event.pull_request.merged
+      && (
+        github.event.action == 'closed'
+        || (
+          github.event.action == 'labeled'
+          && contains(github.event.label.name, 'backport')
+        )
+      )
     steps:
-      - name: Backport
-        uses: tibdex/backport@v1
+      - uses: tibdex/backport@9565281eda0731b1d20c4025c43339fb0a23812e # v2.0.4
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}