pipeline_tests.yml

## Disclaimer - This GitHub Actions workflow runs all the tests for changed pipelines.

# Name the workflow
name: pipeline_tests

# Define when it runs
on:
  pull_request:
    paths:
      - 'narps_open/pipelines/team**'

# Jobs that define the workflow
jobs:

  # A job to list the tests to be run
  identify-tests:
    runs-on: ubuntu-latest
    outputs:
      teams: ${{ steps.identify.outputs.teams }}
      tests: ${{ steps.identify.outputs.tests }}
    steps:
    - name: Checkout current branch
      uses: actions/checkout@v3
      with:
        fetch-depth: 0

    - name: Create a list of tests for changed pipelines
      id: identify
      run: |
        # Loop through modified files between PR base and last head
        for file in $(git diff --name-only --diff-filter=d remotes/origin/main...$GITHUB_SHA)
        do
          # echo each file
          echo $file
          # List team id corresponding to team_* files
          if [[ "$file" =~ narps_open/pipelines/team_[A-Z0-9]{4}.py ]]; then
            echo "Modified pipeline = $file"
            tmp=${file#*"team_"} # remove prefix ending in "team_"
            team_id=${tmp%".py"*}   # remove suffix starting with ".py"
            # Populate the lists of test files and teams
            test_files="$test_files tests/pipelines/test_team_$team_id.py"
            teams="$teams $team_id"
          fi
        done
        # Send the test list as job output
        echo $test_files
        echo "tests=$test_files" >> $GITHUB_OUTPUT
        echo "teams=$teams" >> $GITHUB_OUTPUT

  # A job to identify and run the tests
  pytest:
    needs: identify-tests
    runs-on: self-hosted
    timeout-minutes: 2880 # 48h
    steps:
    - name: Checkout PR branch
      uses: actions/checkout@v3

    - name: Load configuration for self-hosted runner
      run: cp /home/neuro/local_testing_config.toml narps_open/utils/configuration/testing_config.toml

    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        pip install .[tests]

    - name: Remove test reports if any existing
      run: rm -f test_pipeline-*.txt

    - name: Execute tests with pytest
      run: |
        if [[ "${{ needs.identify-tests.outputs.tests }}" != "" ]]; then
          pytest -s -q -m "pipeline_test" ${{ needs.identify-tests.outputs.tests }}
        fi

    - name: Export test results
      if: ${{ always() }}
      run: |
        for team in ${{ needs.identify-tests.outputs.teams }}
        do
          report_value=$(<test_pipeline-$team.txt)
          echo "report_team-$team=$report_value" >> $GITHUB_ENV
        done

    - name: Archive pipeline execution failures
      if: ${{ failure() }} # Run only if previous job fails
      uses: actions/upload-artifact@v3
      with:
        name: pipeline_tests-reports
        path: |
          crash-*.pklz
        retention-days: 15

  # A job to report the test results
  tests-report:
    needs: [identify-tests, pytest]
    runs-on: ubuntu-latest
    if: ${{ always() }}
    steps:
    - name: Report results on step summary
      run: |
        # Start report
        echo "# Correlation values" >> $GITHUB_STEP_SUMMARY
        echo "Unthresholded maps, reproduced vs. results" >> $GITHUB_STEP_SUMMARY
        echo "Correlation values are sorted from hypotheses 1 to 9" >> $GITHUB_STEP_SUMMARY

        # Start report table
        echo "| Team    | Number of subjects | Test status | Correlation values |" >> $GITHUB_STEP_SUMMARY
        echo "| -------- | ------- | ------- | ------- |" >> $GITHUB_STEP_SUMMARY

        # Loop through test report files
        for team in ${{ needs.identify-tests.outputs.teams }}
        do
          # Retrieve the report from GITHUB_ENV, as set up by the pytest job
          echo $(report_team-$team) >> $GITHUB_STEP_SUMMARY
        done

    - name: Checkout wiki
      uses: actions/checkout@v3
      with:
        repository: ${{github.repository}}.wiki
        path: wiki

    - name: Report results on the project's wiki
      run: |
        # Loop through test report files & write them to the corresponding wiki page
        for team in ${{ needs.identify-tests.outputs.teams }}
        do
          echo "# Correlation values for pipeline $team" > wiki/pipeline_$team.md
          echo "Unthresholded maps, reproduced vs. results" >> wiki/pipeline_$team.md
          echo "Correlation values are sorted from hypotheses 1 to 9" >> wiki/pipeline_$team.md
          echo "These values were calculated by GitHub Actions run [# $GITHUB_RUN_ID]($GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID)" >> wiki/pipeline_$team.md
          echo "| Team    | Number of subjects | Test status | Correlation values |" >> wiki/pipeline_$team.md
          echo "| -------- | ------- | ------- | ------- |" >> wiki/pipeline_$team.md

          # Retrieve the report from GITHUB_ENV, as set up by the pytest job
          echo $(report_team-$team) >> wiki/pipeline_$team.md
        done

        # Push changes
        cd wiki
        git config user.name github-actions
        git config user.email github-actions@github.com

        # Test if there are changes in the repository before commiting
        if [[ $(git diff --name-only) ]]; then
          git add .
          git commit -m "Pipeline status updated"
          git push
        fi