refactor: README.md #47
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "Run benchmark" | ||
| on: | ||
| pull_request_target: | ||
| types: [assigned, opened, synchronize, reopened, edited] | ||
| push: | ||
| branches: | ||
| - main | ||
| permissions: | ||
| contents: write | ||
| packages: write | ||
| pull-requests: write | ||
| issues: write | ||
| jobs: | ||
| build: | ||
| if: github.event.head_commit.message != 'Update performance results in README.md' | ||
| runs-on: ubuntu-latest | ||
| env: | ||
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | ||
| steps: | ||
| - name: Checkout | ||
| uses: actions/checkout@v4 | ||
| with: | ||
| token: ${{ secrets.GITHUB_TOKEN }} | ||
| ref: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.sha || github.sha }} | ||
| - name: Set up Docker buildx | ||
| uses: docker/setup-buildx-action@v3 | ||
| - name: Login to GitHub Container Registry | ||
| uses: docker/login-action@v3 | ||
| with: | ||
| registry: ghcr.io | ||
| username: ${{ github.repository_owner }} | ||
| password: ${{ secrets.GITHUB_TOKEN }} | ||
| - name: Build and push Docker image | ||
| uses: docker/build-push-action@v6 | ||
| with: | ||
| push: true | ||
| tags: ghcr.io/${{ github.repository_owner }}/federated-benchmark:latest | ||
| cache-from: type=gha | ||
| cache-to: type=gha,mode=max | ||
| benchmark: | ||
| if: github.event.head_commit.message != 'Update performance results in README.md' | ||
| needs: build | ||
| runs-on: ubuntu-latest | ||
| # runs-on: | ||
| # group: benchmarking-runner | ||
| env: | ||
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | ||
| strategy: | ||
| matrix: | ||
| response: [big, medium, small] | ||
| service: [ | ||
| # the theoretical maximum | ||
| # source_graphql, | ||
| # source_rest_api, | ||
| # nginx -> source: the baseline for comparison | ||
| nginx_graphql, | ||
| # nginx_rest_api, | ||
| # tailcall configurations | ||
| tailcall_default, | ||
| # tailcall_tweaks, | ||
| # tailcall_http_cache, | ||
| # tailcall_cache_dir, | ||
| # tailcall_dedupe_op, | ||
| tailcall_full_conf, | ||
| # wundergraph configurations | ||
| wundergraph_no_opt, | ||
| # wundergraph_dedupe, | ||
| wundergraph_default, | ||
| # apollo | ||
| apollo_router, | ||
| # grafbase | ||
| grafbase_default, | ||
| grafbase_cache, | ||
| ] | ||
| steps: | ||
| - name: Login to GitHub Container Registry | ||
| uses: docker/login-action@v3 | ||
| with: | ||
| registry: ghcr.io | ||
| username: ${{ github.repository_owner }} | ||
| password: ${{ secrets.GITHUB_TOKEN }} | ||
| - name: Create "output.log" file | ||
| run: touch ${{ github.workspace }}/output.log | ||
| - id: benchmark | ||
| name: Benchmark | ||
| uses: addnab/docker-run-action@v3 | ||
| with: | ||
| username: ${{ github.repository_owner }} | ||
| password: ${{ secrets.GITHUB_TOKEN }} | ||
| registry: ghcr.io | ||
| image: ghcr.io/${{ github.repository_owner }}/federated-benchmark:latest | ||
| options: -v ${{ github.workspace }}:/work | ||
| run: ./benchmark.sh ${{ matrix.response }} ${{ matrix.service }} >> /work/output.log | ||
| - name: Print result | ||
| run: cat ${{ github.workspace }}/output.log | ||
| - name: Determine if run had errors | ||
| id: extract_has_error | ||
| run: | | ||
| HAS_ERROR=$(grep -q "Error distribution:" ${{ github.workspace }}/output.log && echo true || echo false) | ||
| echo "error=${HAS_ERROR}" | tee -a "$GITHUB_OUTPUT" | ||
| - name: Extract RPS | ||
| id: extract_rps | ||
| run: | | ||
| RPS=$(grep "Requests/sec:" ${{ github.workspace }}/output.log | awk '{printf int($2)}' | sed -E ':a;s/([0-9])([0-9]{3})$/\1,\2/;ta') | ||
| echo "rps=${RPS}" | tee -a "$GITHUB_OUTPUT" | ||
| - name: Extract P95 Latency | ||
| id: extract_p95 | ||
| run: | | ||
| P95=$(grep "95%" ${{ github.workspace }}/output.log | awk '{print $4, $5}') | ||
| echo "p95=${P95}" | tee -a "$GITHUB_OUTPUT" | ||
| - uses: cloudposse/github-action-matrix-outputs-write@v1 | ||
| id: out | ||
| with: | ||
| matrix-step-name: ${{ github.job }} | ||
| matrix-key: ${{ matrix.response }}_${{ matrix.service }} | ||
| outputs: |- | ||
| has_error: ${{ steps.extract_has_error.error }} | ||
| rps: ${{ steps.extract_rps.error }} | ||
| p95: ${{ steps.extract_p95.error }} | ||
| read: | ||
| runs-on: ubuntu-latest | ||
| needs: [benchmark] | ||
| steps: | ||
| - uses: cloudposse/github-action-matrix-outputs-read@v1 | ||
| id: read | ||
| with: | ||
| matrix-step-name: benchmark | ||
| outputs: | ||
| result: "${{ steps.read.outputs.result }}" | ||
| analyze: | ||
| if: github.event.head_commit.message != 'Update performance results in README.md' | ||
| needs: read | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
| - name: Checkout | ||
| uses: actions/checkout@v4 | ||
| with: | ||
| token: ${{ secrets.GITHUB_TOKEN }} | ||
| ref: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.sha || github.sha }} | ||
| - name: Collect RPS from all matrix jobs | ||
| run: | | ||
| echo "| Server | [112,838 bytes](./source/big.json)| [12598 bytes](./source/medium.json)| [362 bytes](./source/small.json) |" > "results_rps.md" | ||
| echo "| ---: | ---: | ---: | ---: |" >> "results_rps.md" | ||
| echo "| [Nginx](https://nginx.org/en/) | \`${{ fromJson(needs.read.outputs.result).big_nginx_graphql.rps }} RPS\` | \`${{ fromJson(needs.read.outputs.result).medium_nginx_graphql.rps }} RPS\` | \`${{ fromJson(needs.read.outputs.result).small_nginx_graphql.rps }} RPS\` |" >> "results_rps.md" | ||
| echo "| **Base** | | | |" >> "results_rps.md" | ||
| echo "| [Tailcall](https://github.com/tailcallhq/tailcall) | \`${{ fromJson(needs.read.outputs.result).big_tailcall_default.rps }} RPS\` | \`${{ fromJson(needs.read.outputs.result).medium_tailcall_default.rps }} RPS\` | \`${{ fromJson(needs.read.outputs.result).small_tailcall_default.rps }} RPS\` |" >> "results_rps.md" | ||
| echo "| [Grafbase](https://github.com/grafbase/grafbase) | \`${{ fromJson(needs.read.outputs.result).big_grafbase_default.rps }} RPS\` | \`${{ fromJson(needs.read.outputs.result).medium_grafbase_default.rps }} RPS\` | \`${{ fromJson(needs.read.outputs.result).small_grafbase_default.rps }} RPS\` |" >> "results_rps.md" | ||
| echo "| [Wundegraph](https://github.com/wundergraph/cosmo) | \`${{ fromJson(needs.read.outputs.result).big_wundergraph_no_opt.rps }} RPS\` | \`${{ fromJson(needs.read.outputs.result).medium_wundergraph_no_opt.rps }} RPS\` | \`${{ fromJson(needs.read.outputs.result).small_wundergraph_no_opt.rps }} RPS\` |" >> "results_rps.md" | ||
| echo "| [Apollo](https://github.com/apollographql/router) | \`${{ fromJson(needs.read.outputs.result).big_apollo_router.rps }} RPS\` | \`${{ fromJson(needs.read.outputs.result).medium_apollo_router.rps }} RPS\` | \`${{ fromJson(needs.read.outputs.result).small_apollo_router.rps }} RPS\` |" >> "results_rps.md" | ||
| echo "| **Cached** | | | |" >> "results_rps.md" | ||
| echo "| [Tailcall](https://github.com/tailcallhq/tailcall) | \`${{ fromJson(needs.read.outputs.result).big_tailcall_full_conf.rps }} RPS\` | \`${{ fromJson(needs.read.outputs.result).medium_tailcall_full_conf.rps }} RPS\` | \`${{ fromJson(needs.read.outputs.result).small_tailcall_full_conf.rps }} RPS\` |" >> "results_rps.md" | ||
| echo "| [Grafbase](https://github.com/grafbase/grafbase) | \`${{ fromJson(needs.read.outputs.result).big_grafbase_cache.rps }} RPS\` | \`${{ fromJson(needs.read.outputs.result).medium_grafbase_cache.rps }} RPS\` | \`${{ fromJson(needs.read.outputs.result).small_grafbase_cache.rps }} RPS\` |" >> "results_rps.md" | ||
| echo "| [Wundegraph](https://github.com/wundergraph/cosmo) | \`${{ fromJson(needs.read.outputs.result).big_wundergraph_default.rps }} RPS\` | \`${{ fromJson(needs.read.outputs.result).medium_wundergraph_default.rps }} RPS\` | \`${{ fromJson(needs.read.outputs.result).small_wundergraph_default.rps }} RPS\` |" >> "results_rps.md" | ||
| - name: Show RPS table | ||
| run: cat "results_rps.md" | ||
| - name: Collect latency from all matrix jobs | ||
| run: | | ||
| echo "| Server | [112,838 bytes](./source/big.json)| [12598 bytes](./source/medium.json)| [362 bytes](./source/small.json) |" > "results_p95.md" | ||
| echo "| ---: | ---: | ---: | ---: |" >> "results_p95.md" | ||
| echo "| [Nginx](https://nginx.org/en/) | \`${{ fromJson(needs.read.outputs.result).big_nginx_graphql.p95 }}\` | \`${{ fromJson(needs.read.outputs.result).medium_nginx_graphql.p95 }}\` | \`${{ fromJson(needs.read.outputs.result).small_nginx_graphql.p95 }}\` |" >> "results_p95.md" | ||
| echo "| **Base** | | | |" >> "results_p95.md" | ||
| echo "| [Tailcall](https://github.com/tailcallhq/tailcall) | \`${{ fromJson(needs.read.outputs.result).big_tailcall_default.p95 }}\` | \`${{ fromJson(needs.read.outputs.result).medium_tailcall_default.p95 }}\` | \`${{ fromJson(needs.read.outputs.result).small_tailcall_default.p95 }}\` |" >> "results_p95.md" | ||
| echo "| [Grafbase](https://github.com/grafbase/grafbase) | \`${{ fromJson(needs.read.outputs.result).big_grafbase_default.p95 }}\` | \`${{ fromJson(needs.read.outputs.result).medium_grafbase_default.p95 }}\` | \`${{ fromJson(needs.read.outputs.result).small_grafbase_default.p95 }}\` |" >> "results_p95.md" | ||
| echo "| [Wundegraph](https://github.com/wundergraph/cosmo) | \`${{ fromJson(needs.read.outputs.result).big_wundergraph_no_opt.p95 }}\` | \`${{ fromJson(needs.read.outputs.result).medium_wundergraph_no_opt.p95 }}\` | \`${{ fromJson(needs.read.outputs.result).small_wundergraph_no_opt.p95 }}\` |" >> "results_p95.md" | ||
| echo "| [Apollo](https://github.com/apollographql/router) | \`${{ fromJson(needs.read.outputs.result).big_apollo_router.p95 }}\` | \`${{ fromJson(needs.read.outputs.result).medium_apollo_router.p95 }}\` | \`${{ fromJson(needs.read.outputs.result).small_apollo_router.p95 }}\` |" >> "results_p95.md" | ||
| echo "| **Cached** | | | |" >> "results_p95.md" | ||
| echo "| [Tailcall](https://github.com/tailcallhq/tailcall) | \`${{ fromJson(needs.read.outputs.result).big_tailcall_full_conf.p95 }}\` | \`${{ fromJson(needs.read.outputs.result).medium_tailcall_full_conf.p95 }}\` | \`${{ fromJson(needs.read.outputs.result).small_tailcall_full_conf.p95 }}\` |" >> "results_p95.md" | ||
| echo "| [Grafbase](https://github.com/grafbase/grafbase) | \`${{ fromJson(needs.read.outputs.result).big_grafbase_cache.p95 }}\` | \`${{ fromJson(needs.read.outputs.result).medium_grafbase_cache.p95 }}\` | \`${{ fromJson(needs.read.outputs.result).small_grafbase_cache.p95 }}\` |" >> "results_p95.md" | ||
| echo "| [Wundegraph](https://github.com/wundergraph/cosmo) | \`${{ fromJson(needs.read.outputs.result).big_wundergraph_default.p95 }}\` | \`${{ fromJson(needs.read.outputs.result).medium_wundergraph_default.p95 }}\` | \`${{ fromJson(needs.read.outputs.result).small_wundergraph_default.p95 }}\` |" >> "results_p95.md" | ||
| - name: Show P95 Latency table | ||
| run: cat "results_p95.md" | ||
| - name: Update `README.md` file with RPS data | ||
| run: | | ||
| sed -i '/<!-- PERFORMANCE_RESULTS_START -->/,/<!-- PERFORMANCE_RESULTS_END -->/ { | ||
| /<!-- PERFORMANCE_RESULTS_START -->/!{ | ||
| /<!-- PERFORMANCE_RESULTS_END -->/!d | ||
| } | ||
| }' README.md | ||
| sed -i '/<!-- PERFORMANCE_RESULTS_START -->/r results_rps.md' README.md | ||
| - name: Update `README.md` file with P95 latency data | ||
| run: | | ||
| sed -i '/<!-- LATENCY_RESULTS_START -->/,/<!-- LATENCY_RESULTS_END -->/ { | ||
| /<!-- LATENCY_RESULTS_START -->/!{ | ||
| /<!-- LATENCY_RESULTS_END -->/!d | ||
| } | ||
| }' README.md | ||
| sed -i '/<!-- LATENCY_RESULTS_START -->/r results_p95.md' README.md | ||
| - name: Comment `README.md` file | ||
| if: github.event_name == 'pull_request_target' | ||
| uses: peter-evans/commit-comment@v3 | ||
| with: | ||
| sha: ${{ github.event.pull_request.head.sha }} | ||
| body-path: "README.md" | ||
| reactions: eyes | ||
| - name: Commit and push changes (on main branch) | ||
| if: github.event_name == 'push' && github.ref == 'refs/heads/main' | ||
| uses: stefanzweifel/git-auto-commit-action@v5 | ||
| with: | ||
| branch: main | ||
| commit_author: Author <[email protected]> | ||
| commit_message: "[ci skip] update performance results in README.md" | ||
| - name: Summarize | ||
| run: | | ||
| echo "### RPS Table" >> "$GITHUB_STEP_SUMMARY" | ||
| cat "results_rps.md" >> "$GITHUB_STEP_SUMMARY" | ||
| echo "### P95 Latency Table" >> "$GITHUB_STEP_SUMMARY" | ||
| cat "results_p95.md" >> "$GITHUB_STEP_SUMMARY" | ||
