benchmark.yml

name: Benchmark

on:
    workflow_dispatch:
        inputs:
            exclude-slow-benchmarks:
                description: 'Exclude slow benchmarks'
                required: true
                type: boolean
                default: false
            exclude-unstable-benchmarks:
                description: 'Exclude unstable benchmarks'
                required: true
                type: boolean
                default: false
            exclude-release-only-benchmarks:
                description: 'Exclude release benchmarks'
                required: true
                type: boolean
                default: false
            signing-key-id:
                required: true
                type: string
                default: C8B77BCBE16CD2B94B43F9C8757397B82D4ED7B0

jobs:
    get-exclude-dirs:
        name: Get exclude directories
        runs-on: ubuntu-latest
        outputs:
            exclude-dirs: ${{ steps.get-exclude-dirs.outputs.exclude-dirs }}
        steps:
            - uses: actions/checkout@v4

            - id: get-exclude-dirs
              uses: ./.github/actions/get_exclude_dirs
              with:
                  exclude-slow: ${{ github.event.inputs.exclude-slow-benchmarks }}
                  exclude-unstable: ${{ github.event.inputs.exclude-unstable-benchmarks }}
                  exclude-release-only: ${{ github.event.inputs.exclude-release-only-benchmarks }}

    create-branch-prefix:
        uses: ./.github/workflows/create_branch_prefix.yml
        with:
            prefix: 'benchmark'
            version: $(jq -r '.version' package.json)
            signing-key-id: ${{ inputs.signing-key-id }}
        secrets:
            GPG_SIGNING_KEY: ${{ secrets.GPG_SIGNING_KEY }}

    run-benchmarks:
        name: ${{ matrix.benchmark_group.name }}
        needs:
            - get-exclude-dirs
            - create-branch-prefix
        strategy:
            fail-fast: false
            matrix:
                benchmark_group:
                    - {
                          name: 'Stable Demo',
                          directories: './examples/stable/demo'
                      }
                    - {
                          name: 'Experimental Demo',
                          directories: './examples/experimental/demo'
                      }
                    - {
                          name: 'Stable E2E CRPC',
                          directories: './examples/stable/test/end_to_end/candid_rpc'
                      }
                    - {
                          name: 'Experimental E2E CRPC',
                          directories: './examples/experimental/test/end_to_end/candid_rpc'
                      }
                    - {
                          name: 'Experimental E2E HTTP Server',
                          directories: './examples/experimental/test/end_to_end/http_server'
                      }
                    - {
                          name: 'Stable Property CRPC',
                          directories: './examples/stable/test/property/candid_rpc'
                      }
                    - {
                          name: 'Experimental Property CRPC',
                          directories: './examples/experimental/test/property/candid_rpc'
                      }
                    - {
                          name: 'Stable Property IC API',
                          directories: './examples/stable/test/property/ic_api'
                      }
        uses: ./.github/workflows/benchmark_parallel.yml
        secrets:
            GPG_SIGNING_KEY: ${{ secrets.GPG_SIGNING_KEY }}
            GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
            LASTMJS_GITHUB_TOKEN: ${{ secrets.LASTMJS_GITHUB_TOKEN }}
        with:
            base-branch: ${{ needs.create-branch-prefix.outputs.base-branch }}
            branch-prefix: ${{ needs.create-branch-prefix.outputs.branch-prefix }}
            directories: ${{ matrix.benchmark_group.directories }}
            exclude-dirs: ${{ needs.get-exclude-dirs.outputs.exclude-dirs }}
            signing-key-id: ${{ inputs.signing-key-id }}

    squash-branches:
        needs: [run-benchmarks, create-branch-prefix]
        uses: ./.github/workflows/squash_branches.yml
        with:
            base-branch: ${{ needs.create-branch-prefix.outputs.base-branch }}
            branch-prefix: ${{ needs.create-branch-prefix.outputs.branch-prefix }}
            commit-message: 'run benchmarks'
            signing-key-id: ${{ inputs.signing-key-id }}
        secrets:
            GPG_SIGNING_KEY: ${{ secrets.GPG_SIGNING_KEY }}
            GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
            LASTMJS_GITHUB_TOKEN: ${{ secrets.LASTMJS_GITHUB_TOKEN }}

    analyze-benchmarks:
        name: Analyze Benchmark Results
        needs:
            - squash-branches
            - create-branch-prefix
        runs-on: ubuntu-latest
        steps:
            - uses: actions/checkout@v4
              with:
                  ref: ${{ needs.create-branch-prefix.outputs.base-branch }}

            - uses: ./.github/actions/setup_node

            - uses: ./.github/actions/setup_dfx

            - name: Install dependencies
              run: npm install

            - name: Analyze benchmarks
              run: npx tsx scripts/analyze_benchmarks/index.ts

            - uses: ./.github/actions/commit_and_push
              with:
                  gpg-signing-key: ${{ secrets.GPG_SIGNING_KEY }}
                  branch-name: ${{ needs.create-branch-prefix.outputs.base-branch }}
                  commit-message: 'analyze benchmark results'
                  create-branch: 'false'
                  signing-key-id: ${{ inputs.signing-key-id }}

    create-pr:
        needs:
            - analyze-benchmarks
            - create-branch-prefix
        runs-on: ubuntu-latest
        steps:
            - uses: actions/checkout@v4
              with:
                  ref: ${{ github.ref }}

            - name: Create Pull Request
              env:
                  GH_TOKEN: ${{ secrets.LASTMJS_GITHUB_TOKEN }}
              run: |
                  gh pr create \
                    --base main \
                    --head ${{ needs.create-branch-prefix.outputs.base-branch }} \
                    --title "Benchmark Results for ${{ needs.create-branch-prefix.outputs.base-branch }}" \
                    --body "Automated PR for benchmark results"