diff --git a/.github/workflows/check-compatibility.yml b/.github/workflows/check-compatibility.yml
index dab8afd4ec1f2..1c86b5d79e610 100644
--- a/.github/workflows/check-compatibility.yml
+++ b/.github/workflows/check-compatibility.yml
@@ -6,7 +6,6 @@ on:
 
 jobs:
   check-compatibility:
-    if: github.repository == 'opensearch-project/OpenSearch'
     permissions:
       contents: read
     runs-on: ubuntu-latest
@@ -52,8 +51,18 @@ jobs:
         with:
           name: results.txt
 
+      - name: Find Comment
+        uses: peter-evans/find-comment@v2
+        id: fc
+        with:
+          issue-number: ${{ github.event.number }}
+          comment-author: 'github-actions[bot]'
+          body-includes: 'Compatibility status:'
+
       - name: Add comment on the PR
         uses: peter-evans/create-or-update-comment@v3
         with:
+          comment-id: ${{ steps.fc.outputs.comment-id }}
           issue-number: ${{ github.event.number }}
           body-path: results.txt
+          edit-mode: replace