bench-turborepo.yml

name: Benchmark Turborepo

on:
  workflow_dispatch:
  workflow_run:
    # Make sure this matches the name of the workflow in ./github/workflows/turborepo-release.yml.
    workflows: [Turborepo Release]
    types:
      - completed
  push:
    branches:
      - main
    paths:
      - "cli/**"
      - crates/turborepo*/**
      # - "benchmark/**" (we don't need to run benchmarks when the benchmark changes, next push will capture it)

jobs:
  # benchmark:
  #   name: Run Benchmarks
  #   timeout-minutes: 60
  #   runs-on: ubuntu-latest

  #   steps:
  #     - uses: actions/checkout@v3
  #     - uses: ./.github/actions/setup-node
  #     - uses: ./.github/actions/setup-go
  #       with:
  #         github-token: "${{ secrets.GITHUB_TOKEN }}"

  #     - uses: ./.github/actions/setup-turborepo-environment
  #       with:
  #         target: ubuntu-latest
  #         github-token: "${{ secrets.GITHUB_TOKEN }}"

  #     - name: Build
  #       run: cd cli && make turbo-prod

  #     - name: Run benchmarks
  #       run: pnpm -F benchmark benchmark

  #     - name: Save to Tinybird
  #       run: |
  #         curl \
  #           -i \
  #           -F "ndjson=@./benchmark/tinybird.ndjson" \
  #           -X POST \
  #           -H 'Authorization: Bearer ${{ secrets.TINYBIRD_TOKEN }}' \
  #           'https://api.us-east.tinybird.co/v0/datasources?format=ndjson&name=turbo_benchmarks&mode=append'

  time-to-first-task:
    name: Time to First Task
    timeout-minutes: 60
    runs-on: ${{ matrix.os.runner }}
    strategy:
      fail-fast: false
      matrix:
        os:
          - name: ubuntu
            runner: ubuntu-latest
          - name: macos
            runner: macos-latest
          - name: windows
            runner: windows-latest

    steps:
      - uses: actions/checkout@v3
      - name: Set filename for profile
        id: filename
        shell: bash
        run: echo 'filename=benchmark/${{ matrix.os.runner }}.json' >> "$GITHUB_OUTPUT"

      - name: Debug filename
        shell: bash
        run: echo "${{ steps.filename.outputs.filename }}"

      - name: Setup Node
        uses: ./.github/actions/setup-node

      # - name: Setup Go
      #   uses: ./.github/actions/setup-go
      #   with:
      #     github-token: "${{ secrets.GITHUB_TOKEN }}"

      # setup Rust on windows, because apparently setup-turborepo-environment doesn't do this
      # TODO: put this logic in ./github/actions/setup-rust and use that instead
      # - name: Add rustup for windows
      #   if: ${{ matrix.os.runner == 'windows-latest'}}
      #   run: rustup target add x86_64-pc-windows-gnu

      # - uses: ./.github/actions/setup-turborepo-environment
      #   with:
      #     target: ${{ matrix.os.runner }}
      #     github-token: "${{ secrets.GITHUB_TOKEN }}"

      # - name: Build Turborepo from source
      #   run: cd cli && make turbo-prod

      - name: Run benchmarks
        shell: bash
        run: pnpm -F benchmark ttft "${{ steps.filename.outputs.filename }}"

      - name: Upload Artifacts
        uses: actions/upload-artifact@v3
        with:
          name: profile-${{matrix.os.runner}}.json
          path: ${{ steps.filename.outputs.filename }}

  # Send each of the profiles generated from the previous job to TinyBird
  # We'll wait for all profiles to complete before sending.
  send-to-tinybird:
    needs: [time-to-first-task]
    runs-on: ubuntu-latest
    env:
      TINYBIRD_TOKEN: ${{secrets.TINYBIRD_TOKEN}}
    strategy:
      fail-fast: false
      matrix:
        # Note: these look like os, but this matrix is _not_ used for the job's runner.
        # It is only used to get each profile's name (which was generated in the previous job)
        profile-name:
          - name: ubuntu-latest
          - name: macos-latest
          - name: windows-latest
    steps:
      - name: Download profiles
        uses: actions/download-artifact@v3
        with:
          path: benchmark/

      - name: Debug downloaded artifacts
        run: ls -la benchmark/

      - name: Process and send to Tinybird
        shell: bash
        run: |
          jq -f benchmark/src/fold.jq < "${{matrix.profile-name}}.json" > benchmark/ttft.json
          cat ttft.json
          node -r esbuild-register ./src/ttft-send.ts ttft.json ${{github.run_id}}

  # # Post results to slack
  # post-to-slack:
  #   needs: [time-to-first-task]
  #   runs-on: ubuntu-latest
  #     # TODO: add some actual data to the message using `with: payload` or `with: payload-path`
  #     # TODO: compare results to previous data and only post regressions
  #   steps:
  #     - name: Post To Slack
  #       uses: slackapi/slack-github-action@v1.23.0
  #       env:
  #         SLACK_WEBHOOK_URL: "${{ secrets.TURBOREPO_PERF_BENCHMARK_SLACK_WEBHOOK_URL }}"