refactor: add more info inf README.md #46
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "Run benchmark" | |
| on: | |
| pull_request_target: | |
| types: [assigned, opened, synchronize, reopened, edited] | |
| push: | |
| branches: | |
| - main | |
| permissions: | |
| contents: write | |
| packages: write | |
| pull-requests: write | |
| issues: write | |
| jobs: | |
| build: | |
| if: github.event.head_commit.message != 'Update performance results in README.md' | |
| runs-on: ubuntu-latest | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| ref: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.sha || github.sha }} | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Login to GitHub Container Registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.repository_owner }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Build and Push Docker Image | |
| uses: docker/build-push-action@v6 | |
| with: | |
| push: true | |
| tags: ghcr.io/${{ github.repository_owner }}/federated-benchmark:latest | |
| cache-from: type=gha | |
| cache-to: type=gha,mode=max | |
| benchmark: | |
| if: github.event.head_commit.message != 'Update performance results in README.md' | |
| needs: build | |
| runs-on: | |
| group: benchmarking-runner | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| strategy: | |
| matrix: | |
| response: [big, medium, small] | |
| service: [ | |
| # the theoretical maximum | |
| # source_graphql, | |
| # source_rest_api, | |
| # nginx -> source: the baseline for comparison | |
| nginx_graphql, | |
| # nginx_rest_api, | |
| # tailcall configurations | |
| tailcall_default, | |
| # tailcall_tweaks, | |
| # tailcall_http_cache, | |
| # tailcall_cache_dir, | |
| # tailcall_dedupe_op, | |
| tailcall_full_conf, | |
| # wundergraph configurations | |
| wundergraph_no_opt, | |
| # wundergraph_dedupe, | |
| wundergraph_default, | |
| # apollo | |
| apollo_router, | |
| # grafbase | |
| grafbase_default, | |
| grafbase_cache, | |
| ] | |
| steps: | |
| - name: Login to GitHub Container Registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.repository_owner }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Create "output.log" file | |
| run: touch ${{ github.workspace }}/output.log | |
| - id: benchmark | |
| name: Benchmark | |
| uses: addnab/docker-run-action@v3 | |
| with: | |
| username: ${{ github.repository_owner }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| registry: ghcr.io | |
| image: ghcr.io/${{ github.repository_owner }}/federated-benchmark:latest | |
| options: -v ${{ github.workspace }}:/work | |
| run: ./benchmark.sh ${{ matrix.response }} ${{ matrix.service }} >> /work/output.log | |
| - name: Print result | |
| run: cat ${{ github.workspace }}/output.log | |
| - name: Extract Requests Per Second | |
| id: extract_rps | |
| run: | | |
| RPS=$(grep "Requests/sec:" ${{ github.workspace }}/output.log | awk '{printf int($2)}' | sed -E ':a;s/([0-9])([0-9]{3})$/\1,\2/;ta') | |
| echo "${{ matrix.response }}_${{ matrix.service }}_rps=${RPS}" | tee -a "$GITHUB_OUTPUT" | |
| outputs: | |
| big_nginx_graphql: ${{ steps.extract_rps.outputs.big_nginx_graphql_rps }} | |
| medium_nginx_graphql: ${{ steps.extract_rps.outputs.medium_nginx_graphql_rps }} | |
| small_nginx_graphql: ${{ steps.extract_rps.outputs.small_nginx_graphql_rps }} | |
| big_tailcall_default: ${{ steps.extract_rps.outputs.big_tailcall_default_rps }} | |
| medium_tailcall_default: ${{ steps.extract_rps.outputs.medium_tailcall_default_rps }} | |
| small_tailcall_default: ${{ steps.extract_rps.outputs.small_tailcall_default_rps }} | |
| big_grafbase_default: ${{ steps.extract_rps.outputs.big_grafbase_default_rps }} | |
| medium_grafbase_default: ${{ steps.extract_rps.outputs.medium_grafbase_default_rps }} | |
| small_grafbase_default: ${{ steps.extract_rps.outputs.small_grafbase_default_rps }} | |
| big_wundergraph_no_opt: ${{ steps.extract_rps.outputs.big_wundergraph_no_opt_rps }} | |
| medium_wundergraph_no_opt: ${{ steps.extract_rps.outputs.medium_wundergraph_no_opt_rps }} | |
| small_wundergraph_no_opt: ${{ steps.extract_rps.outputs.small_wundergraph_no_opt_rps }} | |
| big_apollo_router: ${{ steps.extract_rps.outputs.big_apollo_router_rps }} | |
| medium_apollo_router: ${{ steps.extract_rps.outputs.medium_apollo_router_rps }} | |
| small_apollo_router: ${{ steps.extract_rps.outputs.small_apollo_router_rps }} | |
| big_tailcall_full_conf: ${{ steps.extract_rps.outputs.big_tailcall_full_conf_rps }} | |
| medium_tailcall_full_conf: ${{ steps.extract_rps.outputs.medium_tailcall_full_conf_rps }} | |
| small_tailcall_full_conf: ${{ steps.extract_rps.outputs.small_tailcall_full_conf_rps }} | |
| big_grafbase_cache: ${{ steps.extract_rps.outputs.big_grafbase_cache_rps }} | |
| medium_grafbase_cache: ${{ steps.extract_rps.outputs.medium_grafbase_cache_rps }} | |
| small_grafbase_cache: ${{ steps.extract_rps.outputs.small_grafbase_cache_rps }} | |
| big_wundergraph_default: ${{ steps.extract_rps.outputs.big_wundergraph_default_rps }} | |
| medium_wundergraph_default: ${{ steps.extract_rps.outputs.medium_wundergraph_default_rps }} | |
| small_wundergraph_default: ${{ steps.extract_rps.outputs.small_wundergraph_default_rps }} | |
| analyze: | |
| if: github.event.head_commit.message != 'Update performance results in README.md' | |
| needs: benchmark | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| ref: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.sha || github.sha }} | |
| - id: collect | |
| name: Collect Requests Per Second from all matrix jobs | |
| run: | | |
| rm -f results.md | |
| echo "| Server | 112,838 bytes | 12598 bytes | 362 bytes |" > "results.md" | |
| echo "| ---: | ---: | ---: | ---: |" >> "results.md" | |
| echo "| [Nginx](https://nginx.org/en/) | \`${{ needs.benchmark.outputs.big_nginx_graphql }} RPS\` | \`${{ needs.benchmark.outputs.medium_nginx_graphql }} RPS\` | \`${{ needs.benchmark.outputs.small_nginx_graphql }} RPS\` |" >> "results.md" | |
| echo "| **Base** | | | |" >> "results.md" | |
| echo "| [Tailcall](https://github.com/tailcallhq/tailcall) | \`${{ needs.benchmark.outputs.big_tailcall_default }} RPS\` | \`${{ needs.benchmark.outputs.medium_tailcall_default }} RPS\` | \`${{ needs.benchmark.outputs.small_tailcall_default }} RPS\` |" >> "results.md" | |
| echo "| [Grafbase](https://github.com/grafbase/grafbase) | \`${{ needs.benchmark.outputs.big_grafbase_default }} RPS\` | \`${{ needs.benchmark.outputs.medium_grafbase_default }} RPS\` | \`${{ needs.benchmark.outputs.small_grafbase_default }} RPS\` |" >> "results.md" | |
| echo "| [Wundegraph](https://github.com/wundergraph/cosmo) | \`${{ needs.benchmark.outputs.big_wundergraph_no_opt }} RPS\` | \`${{ needs.benchmark.outputs.medium_wundergraph_no_opt }} RPS\` | \`${{ needs.benchmark.outputs.small_wundergraph_no_opt }} RPS\` |" >> "results.md" | |
| echo "| [Apollo](https://github.com/apollographql/router) | \`${{ needs.benchmark.outputs.big_apollo_router }} RPS\` | \`${{ needs.benchmark.outputs.medium_apollo_router }} RPS\` | \`${{ needs.benchmark.outputs.small_apollo_router }} RPS\` |" >> "results.md" | |
| echo "| **Cached** | | | |" >> "results.md" | |
| echo "| [Tailcall](https://github.com/tailcallhq/tailcall) | \`${{ needs.benchmark.outputs.big_tailcall_full_conf }} RPS\` | \`${{ needs.benchmark.outputs.medium_tailcall_full_conf }} RPS\` | \`${{ needs.benchmark.outputs.small_tailcall_full_conf }} RPS\` |" >> "results.md" | |
| echo "| [Grafbase](https://github.com/grafbase/grafbase) | \`${{ needs.benchmark.outputs.big_grafbase_cache }} RPS\` | \`${{ needs.benchmark.outputs.medium_grafbase_cache }} RPS\` | \`${{ needs.benchmark.outputs.small_grafbase_cache }} RPS\` |" >> "results.md" | |
| echo "| [Wundegraph](https://github.com/wundergraph/cosmo) | \`${{ needs.benchmark.outputs.big_wundergraph_default }} RPS\` | \`${{ needs.benchmark.outputs.medium_wundergraph_default }} RPS\` | \`${{ needs.benchmark.outputs.small_wundergraph_default }} RPS\` |" >> "results.md" | |
| - name: Show table | |
| run: cat "results.md" | |
| - name: Update `README.md` file | |
| run: | | |
| sed -i '/<!-- PERFORMANCE_RESULTS_START -->/,/<!-- PERFORMANCE_RESULTS_END -->/ { | |
| /<!-- PERFORMANCE_RESULTS_START -->/!{ | |
| /<!-- PERFORMANCE_RESULTS_END -->/!d | |
| } | |
| }' README.md | |
| sed -i '/<!-- PERFORMANCE_RESULTS_START -->/r results.md' README.md | |
| - name: Comment `results.md` file | |
| if: github.event_name == 'pull_request_target' | |
| uses: peter-evans/commit-comment@v3 | |
| with: | |
| sha: ${{ github.event.pull_request.head.sha }} | |
| body-path: "results.md" | |
| reactions: eyes | |
| - name: Comment `README.md` file | |
| if: github.event_name == 'pull_request_target' | |
| uses: peter-evans/commit-comment@v3 | |
| with: | |
| sha: ${{ github.event.pull_request.head.sha }} | |
| body-path: "README.md" | |
| reactions: eyes | |
| - name: Commit and push changes (on main branch) | |
| if: github.event_name == 'push' && github.ref == 'refs/heads/main' | |
| uses: stefanzweifel/git-auto-commit-action@v5 | |
| with: | |
| branch: main | |
| commit_author: Author <[email protected]> | |
| commit_message: "[ci skip] update performance results in README.md" | |
| - name: Summarize | |
| run: cat "results.md" >> "$GITHUB_STEP_SUMMARY" |