From b61ffe1acfebde5d2df0cfb2f9a7c83c220e32b5 Mon Sep 17 00:00:00 2001 From: Katherine Fairchild Date: Tue, 2 Jan 2024 20:30:50 -0500 Subject: [PATCH 1/9] import script from core --- .travis.yml | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/.travis.yml b/.travis.yml index af0d4232c..1a9dfe141 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,9 +1,12 @@ +version: ~> 1.0 language: python env: global: - PYTEST_SETTINGS="not requires_gpu and not memory_intense and not slow and not travis_slow" + - DOMAIN="vision" - MODIFIES_PLUGIN="False" - PLUGIN_ONLY="False" + - WEB_SUBMISSION="False" matrix: include: - name: 3.7 public @@ -32,18 +35,9 @@ install: - conda update -q conda - conda info -a - pip list # list installed package versions -script: - # if ONLY plugin files changed, ONLY run tests for those plugins; otherwise, run full test suite - - | - if [ ! -z "$TRAVIS_PULL_REQUEST_BRANCH" ]; then - CHANGED_FILES=$( git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*" && git fetch && echo $(git diff --name-only origin/$TRAVIS_PULL_REQUEST_BRANCH origin/$TRAVIS_BRANCH -C $TRAVIS_BUILD_DIR) | tr '\n' ' ' ) && - TESTING_NEEDED=$( python -c "from brainscore_core.plugin_management.parse_plugin_changes import get_testing_info; get_testing_info(\"${CHANGED_FILES}\", 'brainscore_vision')" ) && - read MODIFIES_PLUGIN PLUGIN_ONLY <<< $TESTING_NEEDED && echo MODIFIES_PLUGIN: $MODIFIES_PLUGIN && echo PLUGIN_ONLY: $PLUGIN_ONLY; - fi - - if [ "$MODIFIES_PLUGIN" = "True" ]; then python -c "from brainscore_core.plugin_management.parse_plugin_changes import run_changed_plugin_tests; run_changed_plugin_tests(\"${CHANGED_FILES}\", 'brainscore_vision')"; fi - - if [ "$PRIVATE_ACCESS" = 1 ] && [ -n "${GITHUB_TOKEN}" ] && [ "$PLUGIN_ONLY" = "False" ]; then pytest -m "private_access and $PYTEST_SETTINGS"; fi - - if [ "$PRIVATE_ACCESS" != 1 ] && [ "$PLUGIN_ONLY" = "False" ]; then pytest -m "not private_access and $PYTEST_SETTINGS" --ignore "tests/test_submission"; fi - +import: + - brain-score/core:brainscore_core/travis/script.yml@main + notifications: slack: if: | From 3bffb8acc1d2c4f0f5ccc80e1a60281066240cd5 Mon Sep 17 00:00:00 2001 From: Katherine Fairchild Date: Tue, 2 Jan 2024 22:26:02 -0500 Subject: [PATCH 2/9] add action workflows --- .../workflows/automerge_plugin-only_prs.yml | 131 ++++++++++++++++++ .github/workflows/score_new_plugins.yml | 129 +++++++++++++++++ 2 files changed, 260 insertions(+) create mode 100644 .github/workflows/automerge_plugin-only_prs.yml create mode 100644 .github/workflows/score_new_plugins.yml diff --git a/.github/workflows/automerge_plugin-only_prs.yml b/.github/workflows/automerge_plugin-only_prs.yml new file mode 100644 index 000000000..9d621dde9 --- /dev/null +++ b/.github/workflows/automerge_plugin-only_prs.yml @@ -0,0 +1,131 @@ +name: Automatically merge plugin-only PRs + + +# Triggered on all PRs either by +# - completion of CI checks, OR +# - tagging with label +# 1) If PR is labeled "automerge" or "automerge-web" +# (all website submissions are tagged "automerge-web"), +# checks if Travis tests pass. If yes, THEN +# 2) Checks if PR modifies any code outside plugin dirs. +# If no changes are made beyond new or revised plugins +# (subdirs of /benchmarks, /data, /models, or /metrics) +# the PR is automatically approved and merged. + + +on: + pull_request: + types: [labeled] + status: + +permissions: write-all + +jobs: + setup: + name: setup + runs-on: ubuntu-latest + outputs: + BSC_DATABASESECRET: ${{ steps.setenvvars.outputs.BSC_DATABASESECRET }} + steps: + - name: Check out repository code + uses: actions/checkout@v2 + - name: Check if main & set dev/prod vars + id: setenvvars + run: | + if [ ${{ github.ref }} != 'refs/heads/main' ]; then + echo "::set-output name=BSC_DATABASESECRET:: ${{ secrets.BSC_DATABASESECRET_DEV }}" + else + echo "::set-output name=BSC_DATABASESECRET:: ${{ secrets.BSC_DATABASESECRET_PROD }}" + fi + + + isautomerge: + name: Set as 'automerge' if PR is labeled with 'automerge' or 'automerge-web' + runs-on: ubuntu-latest + if: | + contains( github.event.pull_request.labels.*.name, 'automerge') || + contains( github.event.pull_request.labels.*.name, 'automerge-web') + outputs: + AUTOMERGE: ${{ steps.setautomerge.outputs.AUTOMERGE }} + steps: + - name: Set 'automerge' to 'True' # job only runs if True + id: setautomerge + run: | + echo "::set-output name=AUTOMERGE::True" + + + travis_success: + name: Check if Travis build is successful + runs-on: ubuntu-latest + needs: [isautomerge] + if: ${{ needs.isautomerge.outputs.AUTOMERGE == 'True' }} + outputs: + TRAVIS_OK: ${{ steps.istravisok.outputs.TRAVIS_OK }} + steps: + - name: Get Travis build status + id: gettravisstatus + run: | + echo ${{ github.event.pull_request.head.sha }} + echo "TRAVIS_CONCLUSION=$(python -c "import requests; r = requests.get(\"https://api.github.com/repos/brain-score/brain-score/commits/${{ github.event.pull_request.head.sha }}/check-runs\"); print(next(run['conclusion'] for run in r.json()['check_runs'] if run['name'] == 'Travis CI - Pull Request'))")" >> $GITHUB_ENV + - name: Check if Travis was successful + id: istravisok + run: | + if [ "$TRAVIS_CONCLUSION" == "success" ] + then + travisok=True + elif [ "$TRAVIS_CONCLUSION" == "None" ] + then + travisok=Wait + else + travisok=False + fi + echo "::set-output name=TRAVIS_OK::$travisok" + + + plugin_only: + name: Ensure PR ONLY changes plugin files + runs-on: ubuntu-latest + needs: travis_success + if: ${{ needs.travis_success.outputs.TRAVIS_OK == 'True' }} + outputs: + PLUGIN_ONLY: ${{ steps.ispluginonly.outputs.PLUGIN_ONLY }} + steps: + - name: Parse plugin_only confirmation from Travis status update + id: getpluginonlyvalue + run: echo "PLUGIN_ONLY=$(python -c "import requests; r = requests.get(\"https://api.github.com/repos/brain-score/brain-score/statuses/$github.event.pull_request.head.sha\"); print(next(status['description'].split('- ')[1] for status in r.json() if status['description'].startswith('Run automerge workflow')))")" >> $GITHUB_ENV + - name: Check if PR is plugin only + id: ispluginonly + run: | + if [ "$PLUGIN_ONLY" == "True" ] + then + pluginonly=True + else + pluginonly=False + fi + echo "::set-output name=PLUGIN_ONLY::$pluginonly" + + + automerge: + name: If plugin-only, approve and merge + runs-on: ubuntu-latest + needs: plugin_only + if: ${{ needs.plugin_only.outputs.PLUGIN_ONLY == 'True' }} + steps: + - name: Auto Approve + uses: hmarr/auto-approve-action@v3.1.0 + + - name: Auto Merge (GitHub submissions) + uses: plm9606/automerge_actions@1.2.2 + with: + github-token: ${{ secrets.WORKFLOW_TOKEN }} + label-name: "automerge" + merge-method: "squash" + auto-delete: "true" + + - name: Auto Merge (brain-score.org submissions) + uses: plm9606/automerge_actions@1.2.2 + with: + github-token: ${{ secrets.WORKFLOW_TOKEN }} + label-name: "automerge-web" + merge-method: "squash" + auto-delete: "true" diff --git a/.github/workflows/score_new_plugins.yml b/.github/workflows/score_new_plugins.yml new file mode 100644 index 000000000..dcc7fb1db --- /dev/null +++ b/.github/workflows/score_new_plugins.yml @@ -0,0 +1,129 @@ +name: Trigger scoring run + + +# Triggered on all PRs on merge to main +# If changes are made to a subdir of /benchmarks or /models, +# a Jenkins scoring run is triggered for the corresponding plugin + + +on: + pull_request: + branches: + - main + types: + - closed + +permissions: write-all + +jobs: + setup: + name: setup + runs-on: ubuntu-latest + outputs: + BSC_DATABASESECRET: ${{ steps.setenvvars.outputs.BSC_DATABASESECRET }} + steps: + - name: Check out repository code + uses: actions/checkout@v4 + - name: Set up Python 3.7 + uses: actions/setup-python@v4 + with: + python-version: 3.7 + - name: Check if main & set dev/prod vars + id: setenvvars + run: | + if [ ${{ github.ref }} != 'refs/heads/main' ]; then + echo "::set-output name=BSC_DATABASESECRET:: ${{ secrets.BSC_DATABASESECRET_DEV }}" + else + echo "::set-output name=BSC_DATABASESECRET:: ${{ secrets.BSC_DATABASESECRET_PROD }}" + fi + + + changes_models_or_benchmarks: + name: Check if PR makes changes to /models or /benchmarks + runs-on: ubuntu-latest + outputs: + PLUGIN_INFO: ${{ steps.getpluginfo.outputs.PLUGIN_INFO }} + RUN_SCORE: ${{ steps.runscore.outputs.RUN_SCORE }} + steps: + - name: Check out repository code + uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Save changed files to env var + run: echo "CHANGED_FILES=$(git diff --name-only origin/main~1 origin/$GITHUB_HEAD_REF | tr '\n' ' ')" >> $GITHUB_ENV + + - name: Installing package dependencies + run: | + python -m pip install --upgrade pip setuptools + python -m pip install ".[test]" + + - name: Get plugin info + id: getpluginfo + run: | + echo "PLUGIN_INFO='$(python -c 'from brainscore_core.plugin_management.parse_plugin_changes import get_scoring_info; get_scoring_info("${{ env.CHANGED_FILES }}", "brainscore_vision")')'" >> $GITHUB_OUTPUT + + - name: Run scoring + id: runscore + run: | + echo "RUN_SCORE=$(jq -r '.run_score' <<< ${{ steps.getpluginfo.outputs.PLUGIN_INFO }})" >> $GITHUB_OUTPUT + + get_submitter_info: + name: Get PR author email and (if web submission) Brain-Score user ID + runs-on: ubuntu-latest + needs: [setup, changes_models_or_benchmarks] + if: ${{ needs.changes_models_or_benchmarks.outputs.RUN_SCORE == 'True' }} + env: + BSC_DATABASESECRET: ${{ needs.setup.outputs.BSC_DATABASESECRET }} + PLUGIN_INFO: ${{ needs.changes_models_or_benchmarks.outputs.PLUGIN_INFO }} + outputs: + PLUGIN_INFO: ${{ steps.add_email_to_pluginfo.outputs.PLUGIN_INFO }} + steps: + - name: Parse user ID from PR title (WEB ONLY where we don't have access to the GitHub user) + id: getuid + if: ${{ github.event.pull_request.labels.*.name == 'automerge-web' }} + run: | + echo "BS_UID="$(<<<${{ github.event.pull_request.title }} | sed -E 's/.*\(user:([^)]+)\).*/\1/'"" >> $GITHUB_ENV + - name: Add user ID to PLUGIN_INFO (WEB ONLY) + id: add_uid_to_pluginfo + if: ${{ github.event.pull_request.labels.*.name == 'automerge-web' }} + run: | + echo "The Brain-Score user ID is ${{ steps.getuid.outputs.BS_UID }}" + echo "PLUGIN_INFO="$(<<<$PLUGIN_INFO jq '. + {user_id: ${{ steps.getuid.outputs.UID }} }')"" >> $GITHUB_ENV + + - name: Get PR author email from GitHub username + id: getemail + uses: evvanErb/get-github-email-by-username-action@v1.25 + with: + github-username: ${{github.event.pull_request.user.login}} # PR author's username + token: ${{ secrets.GITHUB_TOKEN }} # Including token enables most reliable way to get a user's email + - name: Add PR author email to PLUGIN_INFO + id: add_email_to_pluginfo + run: | + echo "The PR author email is ${{ steps.getemail.outputs.email }}" + echo "PLUGIN_INFO=$(<<<$PLUGIN_INFO tr -d "'" | jq -c '. + {author_email: "${{ steps.getemail.outputs.email }}"}')" >> $GITHUB_OUTPUT + + + runscore: + name: Score plugins + runs-on: ubuntu-latest + needs: [changes_models_or_benchmarks, get_submitter_info] + if: ${{ needs.changes_models_or_benchmarks.outputs.RUN_SCORE == 'True' }} + env: + PLUGIN_INFO: ${{ needs.get_submitter_info.outputs.PLUGIN_INFO }} + JENKINS_USER: ${{ secrets.JENKINS_USR }} + JENKINS_TOKEN: ${{ secrets.JENKINS_TOKEN }} + JENKINS_TRIGGER: ${{ secrets.JENKINS_TRIGGER }} + steps: + - name: Add domain, public, competition, and model_type to PLUGIN_INFO + run: | + echo "PLUGIN_INFO=$(<<<$PLUGIN_INFO tr -d "'" | jq -c '. + {domain: "vision", public: true, competition: "None", model_type: "Base_Model"}')" >> $GITHUB_ENV + + - name: Check out repository code + uses: actions/checkout@v2 + + - name: Build project and run scoring + run: | + python -m pip install --upgrade pip setuptools + python -m pip install ".[test]" + python -c 'from brainscore_core.submission.endpoints import call_jenkins; call_jenkins('\''${{ env.PLUGIN_INFO }}'\'')' From a8d14f1f26cf8565c2e37fdf16187b73eba28260 Mon Sep 17 00:00:00 2001 From: Katherine Fairchild Date: Tue, 2 Jan 2024 22:31:16 -0500 Subject: [PATCH 3/9] trigger automerge for plugin-only web submissions --- .github/workflows/travis_trigger.sh | 10 ++++++++ .travis.yml | 40 ++++++++++++++++++++--------- 2 files changed, 38 insertions(+), 12 deletions(-) create mode 100755 .github/workflows/travis_trigger.sh diff --git a/.github/workflows/travis_trigger.sh b/.github/workflows/travis_trigger.sh new file mode 100755 index 000000000..1aa336267 --- /dev/null +++ b/.github/workflows/travis_trigger.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +GH_WORKFLOW_TRIGGER=$1 +TRAVIS_PULL_REQUEST_SHA=$2 + +curl -L -X POST \ +-H "Authorization: token $GH_WORKFLOW_TRIGGER" \ +-d '{"state": "success", "description": "Run automerge workflow for plugin-only PR", + "context": "continuous-integration/travis"}' \ + "https://api.github.com/repos/brain-score/brain-score/statuses/$TRAVIS_PULL_REQUEST_SHA" diff --git a/.travis.yml b/.travis.yml index 1a9dfe141..05ba0f955 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,17 +7,6 @@ env: - MODIFIES_PLUGIN="False" - PLUGIN_ONLY="False" - WEB_SUBMISSION="False" -matrix: - include: - - name: 3.7 public - python: '3.7' - - name: 3.7 private - python: '3.7' - env: - - PRIVATE_ACCESS=1 - - secure: f1rWEwrslh7qa2g/QlKs001sGC3uaOxZNQSfNOPj+TMCqEo2c6OzImC4hyz+WqCyc6N/lFT4yYo2RhvaqStHMRmu/+9aZmuH05Bb0KQpfzNFA+yGa/U5WR3/4u6KRvDAeNEi9drT2LuacTyGbldmQsquujK0jrPpFWpe7zUUKv0zb0lJf0zcjeSrZlDXLlgD6DCqow7OqHRvW04dPZVy1OArRwtPV6DJ6Rqo1MqFQGHJ806VPlXhSoydb7a58dhGajqPjomdmZjhd3wS6Lv6uetTE/VVb4EP4e7n0qfZIx/TpnWG0SR44pcP7OCNARWYANsAivzxnQ0shyXnIzOo8ZcPYiPpt/5D53i5idTBxXyuDaHGQvgwuY5XLZzznEedBgZa4OvjxAXlLEQjdVDfSsZeYaV9gyFkeTlLnK1zvWi0US38eF2Qtm3Sx3D/5TtBKK2n38tyK5gg/XvJNycaXvIl7iVcnI2ifpqD1mUWI6C9j9Tk19/XEpWkwaFi91+0LZF1GhjBu8o3G5Np4RIOKXi3TIHkpbMM5mf11T6Bm9LvEMq1h8bgRQigEbeJF8CbUOSVFv+AaXsggGjQhuwdyvy2JZo+tO1nfhi+kW3XrDGPsz1R7Wfqduyn7UUh5OiFymeZwKseYKnwU47KyCqDwrq5Mnx1MlSidnVmPriadR4= - - secure: WE7FPwy07VzJTKAd2xwZdBhtmh8jk7ojwk4B2rIcBQu0vwUXc1MgO8tBLD7s08lBedBjqZiLZEW31uPMEyWNysouDt16a5gm2d149LR7flI3MOifBtxINfJuC3eOEG65bPgN/bYEsIpLKnu3469d5nxZkK7xsjbWTxHGoUpLvVPsmHY2ZM5/jftybs7fI0do4NMG2XffKfZbiFb447Ao3xeQeEfW6IkJllzgGnlG9FJATFidrbwDNdmzAnvPEnDoKAf7ZvhPV0x9yR5V6P4Ck5hxl8mlPdBa1cRMO8s/1ag1c7YJ3AF9ZlwcwqTiGsT8DHTVRxSz4nFHJTMlrm9j84u7WzLZJBhPgF0UeLN3AQgiAZ3c2TFDvjQWeHVuSPkV5GrKlfhSvR82s9yPEdHQxxwYymBbAr6rJR4NtXTyZX0vg8NRKHssZKLSafs/D/pt9xXspqu8HAHc+mS0lCips79XptSr5BEsioil3D2io3tbzrGugpTeJ7oEA787vKn2Cm4XmhyQ0UBhvwsPZ351l27wZYuNV07o9Ik83hN/w4o2v899QQ/zbX42Iy8ZUCWOPX7MV7+TA7SMxru3qx7HL5hDM8kTetxbLB6Ckr+JOdX8L2Fb5L3TVDpsvfv0ebXgwaQR/ez8/7bcXmBqcERApHDz73HaMXUap+iDR4FLdXE= - - AWS_DEFAULT_REGION=us-east-1 before_install: - pip install --upgrade pip - pip install setuptools==60.5.0 @@ -37,7 +26,34 @@ install: - pip list # list installed package versions import: - brain-score/core:brainscore_core/travis/script.yml@main - + - brain-score/core:brainscore_core/travis/submission_failure.yml@main + +jobs: + include: + - name: 3.7 public + python: '3.7' + - name: 3.7 private + python: '3.7' + env: + - PRIVATE_ACCESS=1 + - secure: f1rWEwrslh7qa2g/QlKs001sGC3uaOxZNQSfNOPj+TMCqEo2c6OzImC4hyz+WqCyc6N/lFT4yYo2RhvaqStHMRmu/+9aZmuH05Bb0KQpfzNFA+yGa/U5WR3/4u6KRvDAeNEi9drT2LuacTyGbldmQsquujK0jrPpFWpe7zUUKv0zb0lJf0zcjeSrZlDXLlgD6DCqow7OqHRvW04dPZVy1OArRwtPV6DJ6Rqo1MqFQGHJ806VPlXhSoydb7a58dhGajqPjomdmZjhd3wS6Lv6uetTE/VVb4EP4e7n0qfZIx/TpnWG0SR44pcP7OCNARWYANsAivzxnQ0shyXnIzOo8ZcPYiPpt/5D53i5idTBxXyuDaHGQvgwuY5XLZzznEedBgZa4OvjxAXlLEQjdVDfSsZeYaV9gyFkeTlLnK1zvWi0US38eF2Qtm3Sx3D/5TtBKK2n38tyK5gg/XvJNycaXvIl7iVcnI2ifpqD1mUWI6C9j9Tk19/XEpWkwaFi91+0LZF1GhjBu8o3G5Np4RIOKXi3TIHkpbMM5mf11T6Bm9LvEMq1h8bgRQigEbeJF8CbUOSVFv+AaXsggGjQhuwdyvy2JZo+tO1nfhi+kW3XrDGPsz1R7Wfqduyn7UUh5OiFymeZwKseYKnwU47KyCqDwrq5Mnx1MlSidnVmPriadR4= + - secure: WE7FPwy07VzJTKAd2xwZdBhtmh8jk7ojwk4B2rIcBQu0vwUXc1MgO8tBLD7s08lBedBjqZiLZEW31uPMEyWNysouDt16a5gm2d149LR7flI3MOifBtxINfJuC3eOEG65bPgN/bYEsIpLKnu3469d5nxZkK7xsjbWTxHGoUpLvVPsmHY2ZM5/jftybs7fI0do4NMG2XffKfZbiFb447Ao3xeQeEfW6IkJllzgGnlG9FJATFidrbwDNdmzAnvPEnDoKAf7ZvhPV0x9yR5V6P4Ck5hxl8mlPdBa1cRMO8s/1ag1c7YJ3AF9ZlwcwqTiGsT8DHTVRxSz4nFHJTMlrm9j84u7WzLZJBhPgF0UeLN3AQgiAZ3c2TFDvjQWeHVuSPkV5GrKlfhSvR82s9yPEdHQxxwYymBbAr6rJR4NtXTyZX0vg8NRKHssZKLSafs/D/pt9xXspqu8HAHc+mS0lCips79XptSr5BEsioil3D2io3tbzrGugpTeJ7oEA787vKn2Cm4XmhyQ0UBhvwsPZ351l27wZYuNV07o9Ik83hN/w4o2v899QQ/zbX42Iy8ZUCWOPX7MV7+TA7SMxru3qx7HL5hDM8kTetxbLB6Ckr+JOdX8L2Fb5L3TVDpsvfv0ebXgwaQR/ez8/7bcXmBqcERApHDz73HaMXUap+iDR4FLdXE= + - AWS_DEFAULT_REGION=us-east-1 + - stage: "Automerge check" + python: '3.9' + install: python -m pip install -e ".[test]" + if: type = pull_request + script: + - | + if [ ! -z "$TRAVIS_PULL_REQUEST_BRANCH" ]; then + CHANGED_FILES=$( git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*" && git fetch && echo $(git diff --name-only origin/$TRAVIS_PULL_REQUEST_BRANCH origin/$TRAVIS_BRANCH -C $TRAVIS_BUILD_DIR) | tr '\n' ' ' ) && + PLUGIN_ONLY=$( python -c "from brainscore_core.plugin_management.parse_plugin_changes import is_plugin_only; is_plugin_only(\"${CHANGED_FILES}\", \"brainscore_${DOMAIN}\")" ) + fi + - | + if [ "$PLUGIN_ONLY" = "True" ]; then + bash ${TRAVIS_BUILD_DIR}/.github/workflows/travis_trigger.sh $GH_WORKFLOW_TRIGGER $TRAVIS_PULL_REQUEST_SHA; + fi + notifications: slack: if: | From b3baebc2ee7bf4492a363b04131dc965a287e7d2 Mon Sep 17 00:00:00 2001 From: Katherine Fairchild Date: Wed, 3 Jan 2024 12:58:51 -0500 Subject: [PATCH 4/9] model_type=Brain_Model --- .github/workflows/score_new_plugins.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/score_new_plugins.yml b/.github/workflows/score_new_plugins.yml index dcc7fb1db..2568bdb60 100644 --- a/.github/workflows/score_new_plugins.yml +++ b/.github/workflows/score_new_plugins.yml @@ -117,7 +117,7 @@ jobs: steps: - name: Add domain, public, competition, and model_type to PLUGIN_INFO run: | - echo "PLUGIN_INFO=$(<<<$PLUGIN_INFO tr -d "'" | jq -c '. + {domain: "vision", public: true, competition: "None", model_type: "Base_Model"}')" >> $GITHUB_ENV + echo "PLUGIN_INFO=$(<<<$PLUGIN_INFO tr -d "'" | jq -c '. + {domain: "vision", public: true, competition: "None", model_type: "Brain_Model"}')" >> $GITHUB_ENV - name: Check out repository code uses: actions/checkout@v2 From 268a031c310d45cd84cc5b7ac37846e3b90a56bd Mon Sep 17 00:00:00 2001 From: Katherine Fairchild Date: Wed, 3 Jan 2024 13:15:49 -0500 Subject: [PATCH 5/9] python -> 3.7 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 05ba0f955..fd1545e71 100644 --- a/.travis.yml +++ b/.travis.yml @@ -40,7 +40,7 @@ jobs: - secure: WE7FPwy07VzJTKAd2xwZdBhtmh8jk7ojwk4B2rIcBQu0vwUXc1MgO8tBLD7s08lBedBjqZiLZEW31uPMEyWNysouDt16a5gm2d149LR7flI3MOifBtxINfJuC3eOEG65bPgN/bYEsIpLKnu3469d5nxZkK7xsjbWTxHGoUpLvVPsmHY2ZM5/jftybs7fI0do4NMG2XffKfZbiFb447Ao3xeQeEfW6IkJllzgGnlG9FJATFidrbwDNdmzAnvPEnDoKAf7ZvhPV0x9yR5V6P4Ck5hxl8mlPdBa1cRMO8s/1ag1c7YJ3AF9ZlwcwqTiGsT8DHTVRxSz4nFHJTMlrm9j84u7WzLZJBhPgF0UeLN3AQgiAZ3c2TFDvjQWeHVuSPkV5GrKlfhSvR82s9yPEdHQxxwYymBbAr6rJR4NtXTyZX0vg8NRKHssZKLSafs/D/pt9xXspqu8HAHc+mS0lCips79XptSr5BEsioil3D2io3tbzrGugpTeJ7oEA787vKn2Cm4XmhyQ0UBhvwsPZ351l27wZYuNV07o9Ik83hN/w4o2v899QQ/zbX42Iy8ZUCWOPX7MV7+TA7SMxru3qx7HL5hDM8kTetxbLB6Ckr+JOdX8L2Fb5L3TVDpsvfv0ebXgwaQR/ez8/7bcXmBqcERApHDz73HaMXUap+iDR4FLdXE= - AWS_DEFAULT_REGION=us-east-1 - stage: "Automerge check" - python: '3.9' + python: '3.7' install: python -m pip install -e ".[test]" if: type = pull_request script: From 0cf0c055ace4d6a941f0eb3318f5126f38470ad3 Mon Sep 17 00:00:00 2001 From: Katherine Fairchild Date: Wed, 3 Jan 2024 14:12:13 -0500 Subject: [PATCH 6/9] update repo to vision --- .../workflows/automerge_plugin-only_prs.yml | 21 ++----------------- 1 file changed, 2 insertions(+), 19 deletions(-) diff --git a/.github/workflows/automerge_plugin-only_prs.yml b/.github/workflows/automerge_plugin-only_prs.yml index 9d621dde9..989326573 100644 --- a/.github/workflows/automerge_plugin-only_prs.yml +++ b/.github/workflows/automerge_plugin-only_prs.yml @@ -21,23 +21,6 @@ on: permissions: write-all jobs: - setup: - name: setup - runs-on: ubuntu-latest - outputs: - BSC_DATABASESECRET: ${{ steps.setenvvars.outputs.BSC_DATABASESECRET }} - steps: - - name: Check out repository code - uses: actions/checkout@v2 - - name: Check if main & set dev/prod vars - id: setenvvars - run: | - if [ ${{ github.ref }} != 'refs/heads/main' ]; then - echo "::set-output name=BSC_DATABASESECRET:: ${{ secrets.BSC_DATABASESECRET_DEV }}" - else - echo "::set-output name=BSC_DATABASESECRET:: ${{ secrets.BSC_DATABASESECRET_PROD }}" - fi - isautomerge: name: Set as 'automerge' if PR is labeled with 'automerge' or 'automerge-web' @@ -66,7 +49,7 @@ jobs: id: gettravisstatus run: | echo ${{ github.event.pull_request.head.sha }} - echo "TRAVIS_CONCLUSION=$(python -c "import requests; r = requests.get(\"https://api.github.com/repos/brain-score/brain-score/commits/${{ github.event.pull_request.head.sha }}/check-runs\"); print(next(run['conclusion'] for run in r.json()['check_runs'] if run['name'] == 'Travis CI - Pull Request'))")" >> $GITHUB_ENV + echo "TRAVIS_CONCLUSION=$(python -c "import requests; r = requests.get(\"https://api.github.com/repos/brain-score/vision/commits/${{ github.event.pull_request.head.sha }}/check-runs\"); print(next(run['conclusion'] for run in r.json()['check_runs'] if run['name'] == 'Travis CI - Pull Request'))")" >> $GITHUB_ENV - name: Check if Travis was successful id: istravisok run: | @@ -92,7 +75,7 @@ jobs: steps: - name: Parse plugin_only confirmation from Travis status update id: getpluginonlyvalue - run: echo "PLUGIN_ONLY=$(python -c "import requests; r = requests.get(\"https://api.github.com/repos/brain-score/brain-score/statuses/$github.event.pull_request.head.sha\"); print(next(status['description'].split('- ')[1] for status in r.json() if status['description'].startswith('Run automerge workflow')))")" >> $GITHUB_ENV + run: echo "PLUGIN_ONLY=$(python -c "import requests; r = requests.get(\"https://api.github.com/repos/brain-score/vision/statuses/$github.event.pull_request.head.sha\"); print(next(status['description'].split('- ')[1] for status in r.json() if status['description'].startswith('Run automerge workflow')))")" >> $GITHUB_ENV - name: Check if PR is plugin only id: ispluginonly run: | From 2355a2b3c0cbd042fd648d3eb6ff23c79d28b862 Mon Sep 17 00:00:00 2001 From: Katherine Fairchild Date: Wed, 3 Jan 2024 20:06:46 -0500 Subject: [PATCH 7/9] cleanup --- .github/workflows/score_new_plugins.yml | 29 +++++++------------------ 1 file changed, 8 insertions(+), 21 deletions(-) diff --git a/.github/workflows/score_new_plugins.yml b/.github/workflows/score_new_plugins.yml index 2568bdb60..927320ea9 100644 --- a/.github/workflows/score_new_plugins.yml +++ b/.github/workflows/score_new_plugins.yml @@ -13,30 +13,12 @@ on: types: - closed +env: + BSC_DATABASESECRET: secrets.BSC_DATABASESECRET + permissions: write-all jobs: - setup: - name: setup - runs-on: ubuntu-latest - outputs: - BSC_DATABASESECRET: ${{ steps.setenvvars.outputs.BSC_DATABASESECRET }} - steps: - - name: Check out repository code - uses: actions/checkout@v4 - - name: Set up Python 3.7 - uses: actions/setup-python@v4 - with: - python-version: 3.7 - - name: Check if main & set dev/prod vars - id: setenvvars - run: | - if [ ${{ github.ref }} != 'refs/heads/main' ]; then - echo "::set-output name=BSC_DATABASESECRET:: ${{ secrets.BSC_DATABASESECRET_DEV }}" - else - echo "::set-output name=BSC_DATABASESECRET:: ${{ secrets.BSC_DATABASESECRET_PROD }}" - fi - changes_models_or_benchmarks: name: Check if PR makes changes to /models or /benchmarks @@ -50,6 +32,11 @@ jobs: with: fetch-depth: 0 + - name: Set up Python 3.7 + uses: actions/setup-python@v4 + with: + python-version: 3.7 + - name: Save changed files to env var run: echo "CHANGED_FILES=$(git diff --name-only origin/main~1 origin/$GITHUB_HEAD_REF | tr '\n' ' ')" >> $GITHUB_ENV From d2f2a87ad1f0a4449ff011abb0fe250f0462d7ba Mon Sep 17 00:00:00 2001 From: Katherine Fairchild Date: Wed, 3 Jan 2024 20:39:14 -0500 Subject: [PATCH 8/9] cleanup --- .github/workflows/score_new_plugins.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/score_new_plugins.yml b/.github/workflows/score_new_plugins.yml index 927320ea9..9fa5641e2 100644 --- a/.github/workflows/score_new_plugins.yml +++ b/.github/workflows/score_new_plugins.yml @@ -61,7 +61,6 @@ jobs: needs: [setup, changes_models_or_benchmarks] if: ${{ needs.changes_models_or_benchmarks.outputs.RUN_SCORE == 'True' }} env: - BSC_DATABASESECRET: ${{ needs.setup.outputs.BSC_DATABASESECRET }} PLUGIN_INFO: ${{ needs.changes_models_or_benchmarks.outputs.PLUGIN_INFO }} outputs: PLUGIN_INFO: ${{ steps.add_email_to_pluginfo.outputs.PLUGIN_INFO }} @@ -98,7 +97,7 @@ jobs: if: ${{ needs.changes_models_or_benchmarks.outputs.RUN_SCORE == 'True' }} env: PLUGIN_INFO: ${{ needs.get_submitter_info.outputs.PLUGIN_INFO }} - JENKINS_USER: ${{ secrets.JENKINS_USR }} + JENKINS_USER: ${{ secrets.JENKINS_USER }} JENKINS_TOKEN: ${{ secrets.JENKINS_TOKEN }} JENKINS_TRIGGER: ${{ secrets.JENKINS_TRIGGER }} steps: From ed8ba1c70b45a73abc1adbe7329d3c2c1dc96b2b Mon Sep 17 00:00:00 2001 From: Katherine Fairchild Date: Wed, 3 Jan 2024 20:45:01 -0500 Subject: [PATCH 9/9] clarify imported scripts --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index fd1545e71..b4697776b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,8 +25,8 @@ install: - conda info -a - pip list # list installed package versions import: - - brain-score/core:brainscore_core/travis/script.yml@main - - brain-score/core:brainscore_core/travis/submission_failure.yml@main + - brain-score/core:brainscore_core/travis/script.yml@main # run tests + - brain-score/core:brainscore_core/travis/submission_failure.yml@main # if tests fail on web submission, email submitter jobs: include: