Split automerge_plugin-only_prs into two workflows: `check_if_pr_is…

…_automergeable` and `automerge_plugin-only_prs` (#488) * generalize trigger for all workflows * split automergeable check and approval/merge into two workflows * nit (lowercase -> capital constants) --------- Co-authored-by: Katherine Fairchild <[email protected]>
brain-score · Jan 10, 2024 · cc481da · cc481da
1 parent 21c6b41
commit cc481da
Show file tree

Hide file tree

Showing 7 changed files with 162 additions and 77 deletions.
diff --git a/.github/workflows/automerge_plugin-only_prs.yml b/.github/workflows/automerge_plugin-only_prs.yml
@@ -1,22 +1,15 @@
 name: Automatically merge plugin-only PRs
 
 
-# Triggered on all PRs either by
-# - completion of CI checks, OR
-# - tagging with "automerge" or "automerge-web" labels
-# Checks if Travis and Jenkins tests pass and PR is automergeable. 
-# A PR is automergeable iff labeled "automerge" or originates from 
-# web submission (labeled "automerge-web") AND only makes changes to plugins
-# (subdirs of /benchmarks, /data, /models, or /metrics). 
-# If conditions are met, the PR is automatically approved and merged.
+# Triggered on all PRs labeled "automerge-approved"
+# (Label is set by the "check_if_pr_is_automergeable" workflow)
+# Confirms PR is automergeable (see check_if_pr_is_automergeable.yml for details)
+# If PR is automergeable, approves and merges the PR
 
 
 on:
   pull_request:
     types: [labeled]
-  check_run:
-    types: [completed]
-  status:
 
 permissions: write-all
 
@@ -25,6 +18,7 @@ jobs:
   check_test_results:
     name: Check if all tests have passed and PR meets automerge conditions
     runs-on: ubuntu-latest
+    if: ${{ github.event.label.name == 'automerge-approved' }}
     outputs:
       ALL_TESTS_PASS: ${{ steps.gettestresults.outputs.TEST_RESULTS }}
     steps:
@@ -35,11 +29,8 @@ jobs:
       - name: Get test results and ensure automergeable
         id: gettestresults
         run: |
-          echo "pr_head_sha=$( python brainscore_vision/submission/check_test_status.py get_sha )"          
-          echo "Checking test results for SHA $pr_head_sha"
-          echo "test_results=$( python brainscore_vision/submission/check_test_status.py )"
-          echo "::set-output name=TEST_RESULTS::$test_results"
-
+          echo "Checking test results for PR head pr_head=$( python brainscore_vision/submission/actions_helpers.py get_pr_head )"          
+          echo "{TEST_RESULTS}={$( python brainscore_vision/submission/actions_helpers.py )}" >> $GITHUB_OUTPUT
 
   automerge:
     name: If tests pass and PR is automergeable, approve and merge

diff --git a/.github/workflows/check_if_pr_is_automergeable.yml b/.github/workflows/check_if_pr_is_automergeable.yml
@@ -0,0 +1,60 @@
+name: Check if PR is automergeable
+
+
+# Triggered on all PRs either by
+# - completion of CI checks (status or check_run events), OR
+# - tagging with "automerge" or "automerge-web" labels
+# This workflow checks if the PR that invoked the trigger is automergeable. 
+# A PR is automergeable iff it:
+# 1) is labeled "automerge" OR "automerge-web" (originates from web submission) 
+# 2) only changes plugins (subdirs of /benchmarks, /data, /models, /metrics)
+# 3) passes all tests (Travis and Jenkins). 
+# If all 3 conditions are met, the "automerge-approved" label is applied to the PR
+# (This label triggers the `automerge_plugin-only_prs` workflow to merge the PR.)
+
+
+on:
+  pull_request:
+    types: [labeled]
+  check_run:
+    types: [completed]
+  status:
+
+permissions: write-all
+
+jobs:
+
+  check_test_results:
+    name: Check if all tests have passed and PR meets automerge conditions
+    runs-on: ubuntu-latest
+    outputs:
+      ALL_TESTS_PASS: ${{ steps.gettestresults.outputs.TEST_RESULTS }}
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Get test results and ensure automergeable
+        id: gettestresults
+        run: |
+          echo "Checking test results for PR head pr_head=$( python brainscore_vision/submission/actions_helpers.py get_pr_head )"          
+          echo "{TEST_RESULTS}={$( python brainscore_vision/submission/actions_helpers.py )}" >> $GITHUB_OUTPUT
+
+  approve_automerge:
+    name: If tests pass and PR is automergeable, apply "approve_automerge" label to PR
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+    needs: check_test_results
+    if: ${{ needs.check_test_results.outputs.ALL_TESTS_PASS == 'True' }}
+    steps:
+      - name: Get PR number from workflow context
+        run: |
+          echo "{PR_NUMBER}={$( python brainscore_vision/submission/actions_helpers.py get_pr_num )}" >> $GITHUB_ENV
+      - name: Add automerge-approved label to PR
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GH_REPO: ${{ github.repository }}
+          NUMBER: ${{ env.PR_NUMBER }}
+          LABELS: automerge-approved
+        run: gh issue edit "$NUMBER" --add-label "$LABELS"
diff --git a/.github/workflows/travis_trigger.sh b/.github/workflows/travis_trigger.sh
diff --git a/.github/workflows/workflow_trigger.sh b/.github/workflows/workflow_trigger.sh
@@ -0,0 +1,12 @@
+#!/bin/bash
+
+GH_WORKFLOW_TRIGGER=$1
+PULL_REQUEST_SHA=$2
+STATUS_DESCRIPTION=$3
+CONTEXT=$4
+
+curl -L -X POST \
+-H "Authorization: token $GH_WORKFLOW_TRIGGER" \
+-d $'{"state": "success", "description": "'"$STATUS_DESCRIPTION"'", 
+  "context": "'"$CONTEXT"'"}' \
+  "https://api.github.com/repos/brain-score/brain-score/statuses/$PULL_REQUEST_SHA"
diff --git a/.travis.yml b/.travis.yml
@@ -48,7 +48,7 @@ jobs:
           CHANGED_FILES=$( git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*" && git fetch && echo $(git diff --name-only origin/$TRAVIS_PULL_REQUEST_BRANCH origin/$TRAVIS_BRANCH -C $TRAVIS_BUILD_DIR) | tr '\n' ' ' ) &&
           PLUGIN_ONLY=$( python -c "from brainscore_core.plugin_management.parse_plugin_changes import is_plugin_only; is_plugin_only(\"${CHANGED_FILES}\", \"brainscore_${DOMAIN}\")" )
           if [ "$PLUGIN_ONLY" = "True" ]; then 
-            bash ${TRAVIS_BUILD_DIR}/.github/workflows/travis_trigger.sh $GH_WORKFLOW_TRIGGER $TRAVIS_PULL_REQUEST_SHA; 
+            bash ${TRAVIS_BUILD_DIR}/.github/workflows/workflow_trigger.sh $GH_WORKFLOW_TRIGGER $TRAVIS_PULL_REQUEST_SHA "Successful Travis PR build for plugin-only PR" "continuous-integration/travis"; 
           fi
 
 notifications:

diff --git a/...re_vision/submission/check_test_status.py → ...core_vision/submission/actions_helpers.py b/...re_vision/submission/check_test_status.py → ...core_vision/submission/actions_helpers.py
@@ -1,4 +1,13 @@
-# Uses GitHub API to check test suite status (Travis, Jenkins)
+"""
+Supports two GitHub Actions automerge workflows:
+1. automerge_plugin-only_prs
+2. check_if_pr_is_automergeable
+Uses GitHub API to support the following functions
+- Retrieve a PR Head (SHA or branch name) from the workflow trigger event
+- Retrieve a PR number from a PR Head (SHA or branch name)
+- Check test suite status (Travis, Jenkins)
+Final outputs are accessed by the action via print()
+"""
 
 import json
 import os
@@ -9,44 +18,55 @@
 BASE_URL = "https://api.github.com/repos/brain-score/vision"
 
 
+def get_data(url: str) -> dict:
+    r = requests.get(url)
+    assert r.status_code == 200, f'{r.status_code}: {r.reason}'
+    return r.json()
+
+def get_pr_num_from_head(pr_head) -> int:
+    """
+    Given either an SHA (check_run or status event) or a branch name (pull request event),
+    returns the number of the pull request with that head SHA or branch name.
+    """
+    event_type = os.environ["GITHUB_EVENT_NAME"]
+
+    if event_type == "pull_request":
+        query = f"repo:brain-score/vision type:pr head:{pr_head}"
+    else:
+        query = f"repo:brain-score/vision type:pr sha:{pr_head}"
+    url = f"https://api.github.com/search/issues?q={query}"
+    pull_requests = get_data(url)
+    assert pull_requests["total_count"] == 1, f'Expected one PR associated with this SHA but found none or more than one, cannot automerge'
+    pr_num = pull_requests["items"][0]["number"]
+
+    return pr_num
+
 def _load_event_file() -> dict:
     with open(os.environ["GITHUB_EVENT_PATH"]) as f:
         return json.load(f)
 
-def _get_pr_head_sha_from_github_event(pr_head_sha):
+def get_pr_head_from_github_event() -> str:
+    """
+    Based on the event that triggered the workflow (check_run, status, or pull_request),
+    returns either an SHA (check_run or status) or branch name (pull_request)
+    """
+    pr_head = None
     event_type = os.environ["GITHUB_EVENT_NAME"]
 
     if event_type == "status":
         f = _load_event_file()
         candidate_branches = [branch for branch in f["branches"] if branch["name"] != "master"]
         if len(candidate_branches) == 1:
-            pr_head_sha = candidate_branches[0]["commit"]["sha"]
+            pr_head = candidate_branches[0]["commit"]["sha"]
 
     elif event_type == "check_run":
         f = _load_event_file()
-        pr_head_sha = f["check_run"]["head_sha"]
+        pr_head = f["check_run"]["head_sha"]
 
     elif event_type == "pull_request":
-        pr_head_sha = os.environ["GITHUB_HEAD_REF"]
+        pr_head = os.environ["GITHUB_HEAD_REF"]
 
-    return pr_head_sha
-
-def get_pr_head_sha() -> Union[str, None]:
-    pr_head_sha = None
-
-    # Running in a GitHub Action
-    if "GITHUB_EVENT_NAME" in os.environ:
-        pr_head_sha = _get_pr_head_sha_from_github_event(pr_head_sha)
-    # If not running in GitHub Action, assume first arg is SHA
-    elif "GITHUB_EVENT_NAME" not in os.environ and len(sys.argv) > 1:
-        pr_head_sha = sys.argv[1]
-
-    return pr_head_sha
-
-def get_data(url: str) -> dict:
-    r = requests.get(url)
-    assert r.status_code == 200, f'{r.status_code}: {r.reason}'
-    return r.json()
+    return pr_head
 
 def _get_end_time(d: dict) -> str:
     return d['end_time']
@@ -86,16 +106,21 @@ def is_labeled_automerge(check_runs_json: dict) -> bool:
 
 if __name__ == "__main__":
 
-    pr_head_sha = get_pr_head_sha()
-    if not pr_head_sha:
-        print("No PR Head SHA found. Exiting."); sys.exit()
-    # if file called with get_sha, print SHA and quit (GitHub Actions logging)
+    pr_head = get_pr_head_from_github_event()
+    if not pr_head:
+        print("No PR head found. Exiting."); sys.exit()
+
+    # GitHub Actions helpers
     if len(sys.argv) > 1:
-        if sys.argv[1] == "get_sha":
-            print(pr_head_sha); sys.exit()
-
-    check_runs_json = get_data(f"{BASE_URL}/commits/{pr_head_sha}/check-runs")
-    statuses_json = get_data(f"{BASE_URL}/statuses/{pr_head_sha}")
+        if sys.argv[1] == "get_pr_head":
+            print(pr_head)
+        elif sys.argv[1] == "get_pr_num":
+            print(get_pr_num_from_head(pr_head))
+        sys.exit()
+
+    # Check test results and ensure PR is automergeable
+    check_runs_json = get_data(f"{BASE_URL}/commits/{pr_head}/check-runs")
+    statuses_json = get_data(f"{BASE_URL}/statuses/{pr_head}")
 
     results_dict = {'travis_branch_result': get_check_runs_result('Travis CI - Branch', check_runs_json),
                     'travis_pr_result': get_statuses_result('continuous-integration/travis', statuses_json),

diff --git a/...test_submission/test_check_test_status.py → ...s/test_submission/test_actions_helpers.py b/...test_submission/test_check_test_status.py → ...s/test_submission/test_actions_helpers.py
@@ -1,37 +1,44 @@
 import pytest
 from subprocess import call
 
-from brainscore_vision.submission.check_test_status import BASE_URL, get_data, get_check_runs_result, get_statuses_result, are_all_tests_passing, is_labeled_automerge, get_pr_head_sha
+from brainscore_vision.submission.actions_helpers import BASE_URL, get_pr_num_from_head, get_data, get_check_runs_result, get_statuses_result, are_all_tests_passing, is_labeled_automerge, get_pr_head_from_github_event
 
 PR_HEAD_SHA = '209e6c81d39179fd161a1bd3a5845682170abfd2'
+PR_BRANCH_NAME = 'web_submission_11/add_plugins'
 
 
-def test_get_pr_head_sha_local(monkeypatch):
-    monkeypatch.setattr('sys.argv', ['create_test_status.py', PR_HEAD_SHA])
-    assert get_pr_head_sha() == PR_HEAD_SHA
+def test_get_pr_num_from_head_pull_request(monkeypatch):
+    monkeypatch.setenv('GITHUB_EVENT_NAME', 'pull_request')
+    pr_num = get_pr_num_from_head(PR_BRANCH_NAME)
+    assert pr_num == 442
+
+def test_get_pr_num_from_head_non_pull_request(monkeypatch):
+    monkeypatch.setenv('GITHUB_EVENT_NAME', 'status')
+    pr_num = get_pr_num_from_head(PR_HEAD_SHA)
+    assert pr_num == 442
 
-def test_get_pr_head_sha_github_status(monkeypatch, mocker):
+def test_get_pr_head_status_event(monkeypatch, mocker):
     monkeypatch.setenv('GITHUB_EVENT_NAME', 'status')
     mock_status_json = {'branches': [{'name': 'master', 'commit': {'sha': 123}}, {'name': 'pr_branch', 'commit': {'sha': PR_HEAD_SHA}}]}
-    mocker.patch('brainscore_vision.submission.check_test_status._load_event_file', return_value=mock_status_json)
-    assert get_pr_head_sha() == PR_HEAD_SHA
+    mocker.patch('brainscore_vision.submission.actions_helpers._load_event_file', return_value=mock_status_json)
+    assert get_pr_head_from_github_event() == PR_HEAD_SHA
 
-def test_get_pr_head_sha_github_status_master_only(monkeypatch, mocker):
+def test_get_pr_head_status_event_master_only(monkeypatch, mocker):
     monkeypatch.setenv('GITHUB_EVENT_NAME', 'status')
     mock_status_json = {'branches': [{'name': 'master', 'commit': {'sha': 123}}]}
-    mocker.patch('brainscore_vision.submission.check_test_status._load_event_file', return_value=mock_status_json)
-    assert not get_pr_head_sha()
+    mocker.patch('brainscore_vision.submission.actions_helpers._load_event_file', return_value=mock_status_json)
+    assert not get_pr_head_from_github_event()
 
-def test_get_pr_head_sha_github_check_run(monkeypatch, mocker):
+def test_get_pr_head_check_run_event(monkeypatch, mocker):
     monkeypatch.setenv('GITHUB_EVENT_NAME', 'check_run')
     mock_check_run_json = {'check_run': {'head_sha': PR_HEAD_SHA}}
-    mocker.patch('brainscore_vision.submission.check_test_status._load_event_file', return_value=mock_check_run_json)
-    assert get_pr_head_sha() == PR_HEAD_SHA
+    mocker.patch('brainscore_vision.submission.actions_helpers._load_event_file', return_value=mock_check_run_json)
+    assert get_pr_head_from_github_event() == PR_HEAD_SHA
 
-def test_get_pr_head_sha_github_pull_request(monkeypatch):
+def test_get_pr_head_pull_request_event(monkeypatch):
     monkeypatch.setenv('GITHUB_EVENT_NAME', 'pull_request')
-    monkeypatch.setenv('GITHUB_HEAD_REF', PR_HEAD_SHA)
-    assert get_pr_head_sha() == PR_HEAD_SHA
+    monkeypatch.setenv('GITHUB_HEAD_REF', PR_BRANCH_NAME)
+    assert get_pr_head_from_github_event() == PR_BRANCH_NAME
 
 def test_get_check_runs_data():
     data = get_data(f"{BASE_URL}/commits/{PR_HEAD_SHA}/check-runs")
@@ -70,13 +77,13 @@ def test_one_test_failing():
 def test_is_labeled_automerge(mocker):
     dummy_check_runs_json = {"check_runs": [{"pull_requests": [{"url": "https://api.github.com/repos/brain-score/vision/pulls/453"}]}]}
     dummy_pull_request_data  = {"labels": [{"name": "automerge-web"}]}
-    mocker.patch('brainscore_vision.submission.check_test_status.get_data', return_value=dummy_pull_request_data) 
+    mocker.patch('brainscore_vision.submission.actions_helpers.get_data', return_value=dummy_pull_request_data) 
     assert is_labeled_automerge(dummy_check_runs_json) == True
 
 def test_is_not_labeled_automerge(mocker):
     dummy_check_runs_json = {"check_runs": [{"pull_requests": [{"url": "https://api.github.com/repos/brain-score/vision/pulls/453"}]}]}
     dummy_pull_request_data  = {'labels': []}
-    mocker.patch('brainscore_vision.submission.check_test_status.get_data', return_value=dummy_pull_request_data) 
+    mocker.patch('brainscore_vision.submission.actions_helpers.get_data', return_value=dummy_pull_request_data) 
     assert is_labeled_automerge(dummy_check_runs_json) == False
 
 def test_sha_associated_with_more_than_one_pr():