daily-regression.yml

name: Daily Regression
on:
  schedule:
    # Run this job everyday at 6:30 AM UTC+8
    - cron:  '30 10 * * *'

jobs:
  gen-matrix:
    name: "Generate test matrix"
    runs-on: [self-hosted, linux, nixos, BIGRAM]
    env:
      RUNNERS: 70
    outputs:
      test-plan: ${{ steps.gen-matrix.outputs.matrix }}
    steps:
      # actions/checkout will use the "event" commit to checkout repository,
      # which will lead to an unexpected issue that the "event" commit doesn't belongs to the repository,
      # and causing the derivation build output cannot be cache correctly.
      - uses: actions/checkout@v4
        with:
          ref: ${{ github.event.pull_request.head.sha }}
      - id: gen-matrix
        name: "Generate test matrix"
        run: |
          echo -n matrix= >> "$GITHUB_OUTPUT"
          nix develop -c t1-helper generateRegressionTestPlan --runnersAmount "$RUNNERS" >> "$GITHUB_OUTPUT"

  run-testcases:
    name: "Run testcases"
    needs: [gen-matrix]
    strategy:
      fail-fast: false
      matrix: ${{ fromJSON(needs.gen-matrix.outputs.test-plan) }}
    runs-on: [self-hosted, linux, nixos]
    steps:
      - uses: actions/checkout@v4
        with:
          ref: ${{ github.event.pull_request.head.sha }}
      - name: "Run testcases"
        run: |
          nix develop -c t1-helper runTests --jobs "${{ matrix.jobs }}" \
            --resultDir test-results-$(head -c 10 /dev/urandom | base32) \
            --dontBail

      - uses: actions/upload-artifact@v4
        if: ${{ !cancelled() }}
        with:
          name: test-reports-${{ matrix.id }}
          path: |
            test-results-*/failed-tests.md
            test-results-*/cycle-updates.md

  gen-perf-matrix:
    name: "Generate test matrix for perf cases"
    runs-on: [self-hosted, linux, nixos, BIGRAM]
    env:
      RUNNERS: 70
    outputs:
      perf-plan: ${{ steps.gen-matrix.outputs.matrix }}
    steps:
      # actions/checkout will use the "event" commit to checkout repository,
      # which will lead to an unexpected issue that the "event" commit doesn't belongs to the repository,
      # and causing the derivation build output cannot be cache correctly.
      - uses: actions/checkout@v4
        with:
          ref: ${{ github.event.pull_request.head.sha }}
      - id: gen-matrix
        name: "Generate test matrix"
        run: |
          echo -n matrix= >> "$GITHUB_OUTPUT"
          nix develop -c t1-helper generateCiMatrix --testPlanFile "perf.json" --runnersAmount "$RUNNERS" >> "$GITHUB_OUTPUT"

  run-perfcases:
    name: "Run perf cases"
    needs: [gen-perf-matrix]
    strategy:
      fail-fast: false
      matrix: ${{ fromJSON(needs.gen-perf-matrix.outputs.perf-plan) }}
    runs-on: [self-hosted, linux, nixos]
    steps:
      - uses: actions/checkout@v4
        with:
          ref: ${{ github.event.pull_request.head.sha }}
      - name: "Run testcases"
        run: |
          nix develop -c t1-helper runTests --jobs "${{ matrix.jobs }}" \
            --resultDir test-results-$(head -c 10 /dev/urandom | base32)

      - uses: actions/upload-artifact@v4
        if: ${{ !cancelled() }}
        with:
          name: perf-reports-${{ matrix.id }}
          path: |
            test-results-*/failed-tests.md
            test-results-*/cycle-updates.md
            test-results-*/*_cycle.json

  report:
    name: "Report CI result"
    if: ${{ !cancelled() }}
    needs: [run-testcases]
    runs-on: [self-hosted, linux, nixos]
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
          ref: master
      - uses: actions/download-artifact@v4
        with:
          pattern: '*-reports-*'
          merge-multiple: true
      - run: |
          echo -e "\n## Newly passed tests\n" >> "$GITHUB_STEP_SUMMARY"
          shopt -s nullglob
          cat test-results-*/cycle-updates.md >> "$GITHUB_STEP_SUMMARY"
      - run: |
          echo -e "\n## Still failing tests\n" >> "$GITHUB_STEP_SUMMARY"
          shopt -s nullglob
          cat test-results-*/failed-tests.md >> "$GITHUB_STEP_SUMMARY"
      - name: "Commit cycle updates"
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          nix develop -c t1-helper mergeCycleData --filePat perf.json
          git config user.name github-actions
          git config user.email github-actions@github.com
          changed_cases=$(git diff --name-only '.github/cases/**/perf.json')

          if [ -n "$changed_cases" ]; then
            echo "changed cases: $changed_cases"
            currentDate=$(date +%F)
            branch="cycle-update-$currentDate"
            git checkout -b "$branch"
            git add '.github/cases/**/perf.json'
            git commit -m "[ci] update test case cycle data at $currentDate"
            git push origin "$branch" --force-with-lease
            nix run '.#gh' -- \
              pr create --title "Update perf cycle" --body "Update test cycles."
          else
            echo "No cycle change detect"
          fi
      - uses: geekyeggo/delete-artifact@v4
        with:
          # test-reports has been used, it can be deleted
          name: test-reports-*