Skip to content

brain-score.org submission (user:444) | (public:False) #778

brain-score.org submission (user:444) | (public:False)

brain-score.org submission (user:444) | (public:False) #778

name: Trigger scoring run
# Triggered on all PRs on merge to master
# If changes are made to a subdir of /benchmarks or /models,
# a Jenkins scoring run is triggered for the corresponding plugin
on:
pull_request_target:
branches:
- master
types:
- closed
env:
BSC_DATABASESECRET: secrets.BSC_DATABASESECRET
permissions: write-all
jobs:
process_submission:
name: If triggering PR alters /models or /benchmarks, initiates scoring for relevant plugins
if: github.event.pull_request.merged == true
runs-on: ubuntu-latest
outputs:
RUN_SCORING: ${{ steps.scoringneeded.outputs.RUN_SCORING }}
steps:
- name: Check out repository code
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Set up Python 3.11
uses: actions/setup-python@v4
with:
python-version: 3.11
- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@v1
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: us-east-1
- name: Installing package dependencies
run: |
python -m pip install --upgrade pip setuptools
python -m pip install ".[test]"
- name: Save changed files to env var
run: |
git fetch origin refs/pull/${{ github.event.number }}/head
MERGE_COMMIT=$(git log --format='%H %P' --all | grep "$(git rev-parse FETCH_HEAD)\$" | cut -f1 -d' ')
echo "CHANGED_FILES=$(git diff --name-only origin/master~1 $MERGE_COMMIT | tr '\n' ' ')" >> $GITHUB_ENV
- name: Get plugin info
id: getpluginfo
run: |
echo "PLUGIN_INFO='$(python -c 'from brainscore_core.plugin_management.parse_plugin_changes import get_scoring_info; get_scoring_info("${{ env.CHANGED_FILES }}", "brainscore_vision")')'" >> $GITHUB_OUTPUT
- name: Check if scoring needed
id: scoringneeded
run: |
echo "RUN_SCORING=$(jq -r '.run_score' <<< ${{ steps.getpluginfo.outputs.PLUGIN_INFO }})" >> $GITHUB_OUTPUT
- name: Check for automerge-web label
id: check_label
run: |
LABELS_JSON="${{ toJSON(github.event.pull_request.labels.*.name) }}"
if echo "$LABELS_JSON" | grep -q "automerge-web"; then
echo "has_automerge_web=true" >> $GITHUB_ENV
else
echo "has_automerge_web=false" >> $GITHUB_ENV
fi
- name: Update PLUGIN_INFO based on label
run: |
if [[ "$has_automerge_web" == "true" ]]; then
BS_UID="$(echo '${{ github.event.pull_request.title }}' | sed -E 's/.*\(user:([^)]+)\).*/\1/')"
BS_PUBLIC="$(echo '${{ github.event.pull_request.title }}' | sed -E 's/.*\(public:([^)]+)\).*/\1/')"
PLUGIN_INFO=$(echo ${{ steps.getpluginfo.outputs.PLUGIN_INFO }} | tr -d "'" | jq -c ". + {user_id: \"$BS_UID\", public: \"$BS_PUBLIC\"}")
echo "PLUGIN_INFO=${PLUGIN_INFO}" >> $GITHUB_ENV
else
echo "PLUGIN_INFO=$(echo ${{ steps.getpluginfo.outputs.PLUGIN_INFO }} | tr -d "'")" >> $GITHUB_ENV
fi
- name: Write PLUGIN_INFO to a json file
run: |
echo "$PLUGIN_INFO" > plugin-info.json
- name: Upload PLUGIN_INFO as an artifact
uses: actions/upload-artifact@v2
with:
name: plugin-info
path: plugin-info.json
extract_email:
name: Extracts email for both PRs and web submissions
uses: ./.github/workflows/user_notification_system.yml
needs: process_submission
if: ${{ needs.process_submission.outputs.RUN_SCORING == 'True' }}
with:
pr_username: ${{github.event.pull_request.user.login}}
pr_title: ${{ github.event.pull_request.title }}
is_automerge_web: ${{ contains(github.event.pull_request.labels.*.name, 'automerge-web') }}
action_type: 'extract_email'
secrets: inherit
update_plugin_info:
name: Updates PLUGIN_INFO with various fields (domain, competition, model_type, email)
runs-on: ubuntu-latest
needs: extract_email
steps:
- name: Download PLUGIN_INFO artifact
uses: actions/download-artifact@v2
with:
name: plugin-info
path: artifact-directory
- name: Set PLUGIN_INFO as an environment variable
run: |
PLUGIN_INFO=$(cat artifact-directory/plugin-info.json)
echo "PLUGIN_INFO=${PLUGIN_INFO}" >> $GITHUB_ENV
- name: Decrypt and mask user email
run: |
DECRYPTED_EMAIL=$(echo "${{ needs.extract_email.outputs.extracted_email }}" | openssl enc -aes-256-cbc -a -d -salt -pass pass:${{ secrets.EMAIL_ENCRYPTION_KEY }})
echo "::add-mask::$DECRYPTED_EMAIL"
echo "USER_EMAIL=${DECRYPTED_EMAIL}" >> $GITHUB_ENV
- name: Update PLUGIN_INFO
run: |
PLUGIN_JSON=$(echo "$PLUGIN_INFO" | jq -c '. + {domain: "vision", competition: "None", model_type: "Brain_Model"}')
echo "PLUGIN_INFO=$PLUGIN_JSON" >> $GITHUB_ENV
PLUGIN_JSON=$(echo "$PLUGIN_JSON" | jq -c --arg email "$USER_EMAIL" '. + {email: $email}')
echo "PLUGIN_INFO=$PLUGIN_JSON" >> $GITHUB_ENV
echo "Updated PLUGIN_INFO: $PLUGIN_JSON"
- name: Write PLUGIN_INFO to a json file
run: |
echo "$PLUGIN_INFO" > plugin-info.json
- name: Upload PLUGIN_INFO as an artifact
uses: actions/upload-artifact@v2
with:
name: plugin-info
path: plugin-info.json
run_scoring:
name: Score plugins
runs-on: ubuntu-latest
needs: [process_submission, extract_email, update_plugin_info]
if: needs.process_submission.outputs.RUN_SCORING == 'True'
env:
JENKINS_USER: ${{ secrets.JENKINS_USER }}
JENKINS_TOKEN: ${{ secrets.JENKINS_TOKEN }}
JENKINS_TRIGGER: ${{ secrets.JENKINS_TRIGGER }}
steps:
- name: Download PLUGIN_INFO artifact
uses: actions/download-artifact@v2
with:
name: plugin-info
path: artifact-directory
- name: Set PLUGIN_INFO as an environment variable
run: |
PLUGIN_INFO=$(cat artifact-directory/plugin-info.json)
USER_EMAIL=$(echo "$PLUGIN_INFO" | jq -r '.email')
echo "::add-mask::$USER_EMAIL" # readd a mask when bringing email back from artifact
echo "PLUGIN_INFO=${PLUGIN_INFO}" >> $GITHUB_ENV
- name: Check out repository code
uses: actions/checkout@v4
- name: Set up Python 3.11
uses: actions/setup-python@v4
with:
python-version: 3.11
- name: Build project
run: |
python -m pip install --upgrade pip setuptools
python -m pip install "."
- name: Run scoring
run: |
python -c 'from brainscore_core.submission.endpoints import call_jenkins; call_jenkins('\''${{ env.PLUGIN_INFO }}'\'')'