brain-score.org submission (user:407) | (public:False) #935
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Trigger scoring run | |
| # Triggered on all PRs on merge to master | |
| # If changes are made to a subdir of /benchmarks or /models, | |
| # a Jenkins scoring run is triggered for the corresponding plugin | |
| on: | |
| pull_request_target: | |
| branches: | |
| - master | |
| types: | |
| - closed | |
| env: | |
| BSC_DATABASESECRET: secrets.BSC_DATABASESECRET | |
| permissions: write-all | |
| jobs: | |
| process_submission: | |
| name: If triggering PR alters /models or /benchmarks, initiates scoring for relevant plugins | |
| if: github.event.pull_request.merged == true | |
| runs-on: ubuntu-latest | |
| outputs: | |
| RUN_SCORING: ${{ steps.scoringneeded.outputs.RUN_SCORING }} | |
| steps: | |
| - name: Check out repository code | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Set up Python 3.11 | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: 3.11 | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@v1 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws-region: us-east-1 | |
| - name: Installing package dependencies | |
| run: | | |
| python -m pip install --upgrade pip setuptools | |
| python -m pip install ".[test]" | |
| - name: Save changed files to env var | |
| run: | | |
| git fetch origin refs/pull/${{ github.event.number }}/head | |
| MERGE_COMMIT=$(git log --format='%H %P' --all | grep "$(git rev-parse FETCH_HEAD)\$" | cut -f1 -d' ') | |
| echo "CHANGED_FILES=$(git diff --name-only origin/master~1 $MERGE_COMMIT | tr '\n' ' ')" >> $GITHUB_ENV | |
| - name: Get plugin info | |
| id: getpluginfo | |
| run: | | |
| echo "PLUGIN_INFO='$(python -c 'from brainscore_core.plugin_management.parse_plugin_changes import get_scoring_info; get_scoring_info("${{ env.CHANGED_FILES }}", "brainscore_vision")')'" >> $GITHUB_OUTPUT | |
| - name: Check if scoring needed | |
| id: scoringneeded | |
| run: | | |
| echo "RUN_SCORING=$(jq -r '.run_score' <<< ${{ steps.getpluginfo.outputs.PLUGIN_INFO }})" >> $GITHUB_OUTPUT | |
| - name: Check for automerge-web label | |
| id: check_label | |
| run: | | |
| LABELS_JSON="${{ toJSON(github.event.pull_request.labels.*.name) }}" | |
| if echo "$LABELS_JSON" | grep -q "automerge-web"; then | |
| echo "has_automerge_web=true" >> $GITHUB_ENV | |
| else | |
| echo "has_automerge_web=false" >> $GITHUB_ENV | |
| fi | |
| - name: Update PLUGIN_INFO based on label | |
| run: | | |
| if [[ "$has_automerge_web" == "true" ]]; then | |
| BS_UID="$(echo '${{ github.event.pull_request.title }}' | sed -E 's/.*\(user:([^)]+)\).*/\1/')" | |
| BS_PUBLIC="$(echo '${{ github.event.pull_request.title }}' | sed -E 's/.*\(public:([^)]+)\).*/\1/')" | |
| PLUGIN_INFO=$(echo ${{ steps.getpluginfo.outputs.PLUGIN_INFO }} | tr -d "'" | jq -c ". + {user_id: \"$BS_UID\", public: \"$BS_PUBLIC\"}") | |
| echo "PLUGIN_INFO=${PLUGIN_INFO}" >> $GITHUB_ENV | |
| else | |
| echo "PLUGIN_INFO=$(echo ${{ steps.getpluginfo.outputs.PLUGIN_INFO }} | tr -d "'")" >> $GITHUB_ENV | |
| fi | |
| - name: Write PLUGIN_INFO to a json file | |
| run: | | |
| echo "$PLUGIN_INFO" > plugin-info.json | |
| - name: Upload PLUGIN_INFO as an artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: plugin-info | |
| path: plugin-info.json | |
| extract_email: | |
| name: Extracts email for both PRs and web submissions | |
| uses: ./.github/workflows/user_notification_system.yml | |
| needs: process_submission | |
| if: ${{ needs.process_submission.outputs.RUN_SCORING == 'True' }} | |
| with: | |
| pr_username: ${{github.event.pull_request.user.login}} | |
| pr_title: ${{ github.event.pull_request.title }} | |
| is_automerge_web: ${{ contains(github.event.pull_request.labels.*.name, 'automerge-web') }} | |
| action_type: 'extract_email' | |
| secrets: inherit | |
| update_plugin_info: | |
| name: Updates PLUGIN_INFO with various fields (domain, competition, model_type, email) | |
| runs-on: ubuntu-latest | |
| needs: extract_email | |
| steps: | |
| - name: Download PLUGIN_INFO artifact | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: plugin-info | |
| path: artifact-directory | |
| - name: Set PLUGIN_INFO as an environment variable | |
| run: | | |
| PLUGIN_INFO=$(cat artifact-directory/plugin-info.json) | |
| echo "PLUGIN_INFO=${PLUGIN_INFO}" >> $GITHUB_ENV | |
| - name: Decrypt and mask user email | |
| run: | | |
| DECRYPTED_EMAIL=$(echo "${{ needs.extract_email.outputs.extracted_email }}" | openssl enc -aes-256-cbc -a -d -salt -pass pass:${{ secrets.EMAIL_ENCRYPTION_KEY }}) | |
| echo "::add-mask::$DECRYPTED_EMAIL" | |
| echo "USER_EMAIL=${DECRYPTED_EMAIL}" >> $GITHUB_ENV | |
| - name: Update PLUGIN_INFO | |
| run: | | |
| PLUGIN_JSON=$(echo "$PLUGIN_INFO" | jq -c '. + {domain: "vision", competition: "None", model_type: "Brain_Model"}') | |
| echo "PLUGIN_INFO=$PLUGIN_JSON" >> $GITHUB_ENV | |
| PLUGIN_JSON=$(echo "$PLUGIN_JSON" | jq -c --arg email "$USER_EMAIL" '. + {email: $email}') | |
| echo "PLUGIN_INFO=$PLUGIN_JSON" >> $GITHUB_ENV | |
| echo "Updated PLUGIN_INFO: $PLUGIN_JSON" | |
| - name: Write PLUGIN_INFO to a json file | |
| run: | | |
| echo "$PLUGIN_INFO" > plugin-info2.json | |
| - name: Upload PLUGIN_INFO as an artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: plugin-info2 | |
| path: plugin-info2.json | |
| run_scoring: | |
| name: Score plugins | |
| runs-on: ubuntu-latest | |
| needs: [process_submission, extract_email, update_plugin_info] | |
| if: needs.process_submission.outputs.RUN_SCORING == 'True' | |
| env: | |
| JENKINS_USER: ${{ secrets.JENKINS_USER }} | |
| JENKINS_TOKEN: ${{ secrets.JENKINS_TOKEN }} | |
| JENKINS_TRIGGER: ${{ secrets.JENKINS_TRIGGER }} | |
| steps: | |
| - name: Download PLUGIN_INFO artifact | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: plugin-info2 | |
| path: artifact-directory | |
| - name: Set PLUGIN_INFO as an environment variable | |
| run: | | |
| PLUGIN_INFO=$(cat artifact-directory/plugin-info2.json) | |
| USER_EMAIL=$(echo "$PLUGIN_INFO" | jq -r '.email') | |
| echo "::add-mask::$USER_EMAIL" # readd a mask when bringing email back from artifact | |
| echo "PLUGIN_INFO=${PLUGIN_INFO}" >> $GITHUB_ENV | |
| - name: Check out repository code | |
| uses: actions/checkout@v4 | |
| - name: Set up Python 3.11 | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: 3.11 | |
| - name: Build project | |
| run: | | |
| python -m pip install --upgrade pip setuptools | |
| python -m pip install "." | |
| - name: Run scoring | |
| run: | | |
| python -c 'from brainscore_core.submission.endpoints import call_jenkins; call_jenkins('\''${{ env.PLUGIN_INFO }}'\'')' | |