Move automerge label check logic from workflow step into `check_test_…

…status.py` (#454)

* move automergable check into check_test_status.py

* move automerge label check logic into check_test_status.py

---------

Co-authored-by: Katherine Fairchild <[email protected]>

.github/workflows/automerge_plugin-only_prs.yml

@@ -3,15 +3,12 @@ name: Automatically merge plugin-only PRs
 
 # Triggered on all PRs either by
 # - completion of CI checks, OR
-# - tagging with label
-# If PR is labeled "automerge" or "automerge-web" 
-# (all website submissions are tagged "automerge-web"),
-# checks if Travis and Jenkins tests pass. 
-# A successful status update from Travis also confirms that
-# the PR is plugin-only. Therefore, if all tests pass, this
-# indicates that no changes are made beyond new or revised plugins
-# (subdirs of /benchmarks, /data, /models, or /metrics) 
-# and the PR is automatically approved and merged.
+# - tagging with "automerge" or "automerge-web" labels
+# Checks if Travis and Jenkins tests pass and PR is automergeable. 
+# A PR is automergeable iff labeled "automerge" or originates from 
+# web submission (labeled "automerge-web") AND only makes changes to plugins
+# (subdirs of /benchmarks, /data, /models, or /metrics). 
+# If conditions are met, the PR is automatically approved and merged.
 
 
 on:
@@ -25,26 +22,9 @@ permissions: write-all
 
 jobs:
 
-  isautomerge:
-    name: Set as 'automerge' if PR is labeled with 'automerge' or 'automerge-web'
-    runs-on: ubuntu-latest
-    if: |
-      contains( github.event.pull_request.labels.*.name, 'automerge') ||
-      contains( github.event.pull_request.labels.*.name, 'automerge-web')
-    outputs:
-      AUTOMERGE: ${{ steps.setautomerge.outputs.AUTOMERGE }}
-    steps:
-      - name: Set 'automerge' to 'True' # job only runs if True
-        id: setautomerge
-        run: |
-          echo "::set-output name=AUTOMERGE::True"
-
-
   check_test_results:
-    name: Check if all tests (Travis, Jenkins) have passed (confirms PR is plugin-only)
+    name: Check if all tests have passed and PR meets automerge conditions
     runs-on: ubuntu-latest
-    needs: [isautomerge]
-    if: ${{ needs.isautomerge.outputs.AUTOMERGE == 'True' }}
     outputs:
       ALL_TESTS_PASS: ${{ steps.gettestresults.outputs.TEST_RESULTS }}
     steps:
@@ -60,7 +40,7 @@ jobs:
 
 
   automerge:
-    name: If plugin-only, approve and merge
+    name: If tests pass and PR is automergeable, approve and merge
     runs-on: ubuntu-latest
     needs: check_test_results
     if: ${{ needs.check_test_results.outputs.ALL_TESTS_PASS == 'True' }}

brainscore_vision/submission/check_test_status.py

@@ -34,12 +34,18 @@ def get_statuses_result(context: str, statuses_json: dict) -> str:
     last_status_result = _return_last_result(statuses)
     return last_status_result
 
-def all_tests_passing(test_results: list):
+def are_all_tests_passing(test_results: list):
     if any(result != "success" for result in test_results):
         return False
     else:
         return True
-
+
+def is_labeled_automerge(check_runs_json: dict) -> bool:
+    pull_requests = [check_run['pull_requests'] for check_run in check_runs_json['check_runs']]
+    assert all(len(pull_request) == 1 for pull_request in pull_requests), f'Expected one PR associated with this SHA but found none or more than one, cannot automerge'
+    pull_request_data = get_data(pull_requests[0][0]["url"])
+    labeled_automerge = any(label['name'] in ('automerge', 'automerge-web') for label in pull_request_data['labels'])
+    return labeled_automerge
 
 if __name__ == "__main__":
 
@@ -53,4 +59,9 @@ def all_tests_passing(test_results: list):
     jenkins_plugintests_result = get_statuses_result('Brain-Score Jenkins CI - plugin tests', statuses_json)
     jenkins_unittests_result = get_statuses_result('Brain-Score Jenkins CI', statuses_json)
 
-    print(all_tests_passing([travis_branch_result, travis_pr_result, jenkins_plugintests_result, jenkins_unittests_result]))
+    tests_pass = are_all_tests_passing([travis_branch_result, travis_pr_result, jenkins_plugintests_result, jenkins_unittests_result])
+
+    if tests_pass:
+        print(is_labeled_automerge(check_runs_json))
+    else:
+        print(False)

tests/test_submission/test_check_test_status.py

@@ -1,4 +1,6 @@
-from brainscore_vision.submission.check_test_status import BASE_URL, get_data, get_check_runs_result, get_statuses_result, all_tests_passing
+import pytest
+
+from brainscore_vision.submission.check_test_status import BASE_URL, get_data, get_check_runs_result, get_statuses_result, are_all_tests_passing, is_labeled_automerge
 
 PR_HEAD_SHA = '209e6c81d39179fd161a1bd3a5845682170abfd2'
 
@@ -21,9 +23,29 @@ def test_get_statuses_result():
     assert jenkins_plugintests_result == 'failure'
 
 def test_one_test_failing():
-    success = all_tests_passing(['success', 'success', 'success', 'failure'])
+    success = are_all_tests_passing(['success', 'success', 'success', 'failure'])
     assert success == False
 
-def test_all_tests_passing():
-    success = all_tests_passing(['success', 'success', 'success', 'success'])
+def test_are_all_tests_passing():
+    success = are_all_tests_passing(['success', 'success', 'success', 'success'])
     assert success == True
+
+def test_is_labeled_automerge(mocker):
+    dummy_check_runs_json = {"check_runs": [{"pull_requests": [{"url": "https://api.github.com/repos/brain-score/vision/pulls/453"}]}]}
+    dummy_pull_request_data  = {"labels": [{"name": "automerge-web"}]}
+    mocker.patch('brainscore_vision.submission.check_test_status.get_data', return_value=dummy_pull_request_data) 
+    assert is_labeled_automerge(dummy_check_runs_json) == True
+
+def test_is_not_labeled_automerge(mocker):
+    dummy_check_runs_json = {"check_runs": [{"pull_requests": [{"url": "https://api.github.com/repos/brain-score/vision/pulls/453"}]}]}
+    dummy_pull_request_data  = {'labels': []}
+    mocker.patch('brainscore_vision.submission.check_test_status.get_data', return_value=dummy_pull_request_data) 
+    assert is_labeled_automerge(dummy_check_runs_json) == False
+
+def test_sha_associated_with_more_than_one_pr(mocker):
+    dummy_check_runs_json = {"check_runs": [{"pull_requests": [{"url": "https://api.github.com/repos/brain-score/vision/pulls/453"}, {"url": "https://api.github.com/repos/brain-score/vision/pulls/452"}]}]}
+    with pytest.raises(AssertionError):
+        is_labeled_automerge(dummy_check_runs_json)
+
+
+