fix: removed files #51
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "Run benchmark" | |
| on: | |
| pull_request_target: | |
| types: [assigned, opened, synchronize, reopened, edited] | |
| push: | |
| branches: | |
| - main | |
| permissions: | |
| contents: write | |
| packages: write | |
| pull-requests: write | |
| issues: write | |
| jobs: | |
| build: | |
| if: github.event.head_commit.message != 'Update performance results in README.md' | |
| runs-on: ubuntu-latest | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| ref: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.sha || github.sha }} | |
| - name: Set up Docker buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Login to GitHub Container Registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.repository_owner }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Build and push Docker image | |
| uses: docker/build-push-action@v6 | |
| env: | |
| DOCKER_BUILD_RECORD_UPLOAD: false | |
| with: | |
| push: true | |
| tags: ghcr.io/${{ github.repository_owner }}/federated-benchmark:latest | |
| cache-from: type=gha | |
| cache-to: type=gha,mode=max | |
| benchmark: | |
| if: github.event.head_commit.message != 'Update performance results in README.md' | |
| needs: build | |
| runs-on: ubuntu-latest | |
| # runs-on: | |
| # group: benchmarking-runner | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| strategy: | |
| matrix: | |
| response: [big, medium, small] | |
| service: [ | |
| # the theoretical maximum | |
| # source_graphql, | |
| # source_rest_api, | |
| # nginx -> source: the baseline for comparison | |
| nginx_graphql, | |
| # nginx_rest_api, | |
| # tailcall configurations | |
| tailcall_default, | |
| # tailcall_tweaks, | |
| # tailcall_http_cache, | |
| # tailcall_cache_dir, | |
| # tailcall_dedupe_op, | |
| tailcall_full_conf, | |
| # wundergraph configurations | |
| wundergraph_no_opt, | |
| # wundergraph_dedupe, | |
| wundergraph_default, | |
| # apollo | |
| apollo_router, | |
| # grafbase | |
| grafbase_default, | |
| grafbase_cache, | |
| ] | |
| steps: | |
| - name: Login to GitHub Container Registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.repository_owner }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Create "output.log" file | |
| run: touch ${{ github.workspace }}/output.log | |
| - id: benchmark | |
| name: Benchmark | |
| uses: addnab/docker-run-action@v3 | |
| with: | |
| username: ${{ github.repository_owner }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| registry: ghcr.io | |
| image: ghcr.io/${{ github.repository_owner }}/federated-benchmark:latest | |
| options: -v ${{ github.workspace }}:/work | |
| run: ./benchmark.sh ${{ matrix.response }} ${{ matrix.service }} >> /work/output.log | |
| - name: Print result | |
| run: cat ${{ github.workspace }}/output.log | |
| - name: Determine if run had errors | |
| id: extract_has_error | |
| run: | | |
| HAS_ERROR=$(grep -q "Error distribution:" ${{ github.workspace }}/output.log && echo true || echo false) | |
| echo "error=${HAS_ERROR}" | tee -a "$GITHUB_OUTPUT" | |
| - name: Extract RPS | |
| id: extract_rps | |
| run: | | |
| RPS=$(grep "Requests/sec:" ${{ github.workspace }}/output.log | awk '{printf int($2)}' | sed -E ':a;s/([0-9])([0-9]{3})$/\1,\2/;ta') | |
| echo "rps=${RPS}" | tee -a "$GITHUB_OUTPUT" | |
| - name: Extract P95 Latency | |
| id: extract_p95 | |
| run: | | |
| P95=$(grep "95%" ${{ github.workspace }}/output.log | awk '{print $3, $4}') | |
| echo "p95=${P95}" | tee -a "$GITHUB_OUTPUT" | |
| - uses: cloudposse/github-action-matrix-outputs-write@v1 | |
| id: out | |
| with: | |
| matrix-step-name: ${{ github.job }} | |
| matrix-key: ${{ matrix.response }}_${{ matrix.service }} | |
| outputs: |- | |
| has_error: ${{ steps.extract_has_error.outputs.error }} | |
| rps: ${{ steps.extract_rps.outputs.rps }} | |
| p95: ${{ steps.extract_p95.outputs.p95 }} | |
| read: | |
| runs-on: ubuntu-latest | |
| needs: [benchmark] | |
| steps: | |
| - uses: cloudposse/github-action-matrix-outputs-read@v1 | |
| id: read | |
| with: | |
| matrix-step-name: benchmark | |
| outputs: | |
| result: "${{ steps.read.outputs.result }}" | |
| analyze: | |
| if: github.event.head_commit.message != 'Update performance results in README.md' | |
| needs: read | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| ref: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.sha || github.sha }} | |
| - name: Print Matrix output | |
| run: echo "${{ needs.read.outputs.result }}" | |
| - name: Collect RPS from all matrix jobs | |
| run: | | |
| echo "| Server | [112,838 bytes](./source/big.json)| [12598 bytes](./source/medium.json)| [362 bytes](./source/small.json) |" > "results_rps.md" | |
| echo "| ---: | ---: | ---: | ---: |" >> "results_rps.md" | |
| echo "| [Nginx](https://nginx.org/en/) | \`${{ fromJson(needs.read.outputs.result).big_nginx_graphql.rps }} RPS\` | \`${{ fromJson(needs.read.outputs.result).medium_nginx_graphql.rps }} RPS\` | \`${{ fromJson(needs.read.outputs.result).small_nginx_graphql.rps }} RPS\` |" >> "results_rps.md" | |
| echo "| **Base** | | | |" >> "results_rps.md" | |
| echo "| [Tailcall](https://github.com/tailcallhq/tailcall) | \`${{ fromJson(needs.read.outputs.result).big_tailcall_default.rps }} RPS\` | \`${{ fromJson(needs.read.outputs.result).medium_tailcall_default.rps }} RPS\` | \`${{ fromJson(needs.read.outputs.result).small_tailcall_default.rps }} RPS\` |" >> "results_rps.md" | |
| echo "| [Grafbase](https://github.com/grafbase/grafbase) | \`${{ fromJson(needs.read.outputs.result).big_grafbase_default.rps }} RPS\` | \`${{ fromJson(needs.read.outputs.result).medium_grafbase_default.rps }} RPS\` | \`${{ fromJson(needs.read.outputs.result).small_grafbase_default.rps }} RPS\` |" >> "results_rps.md" | |
| echo "| [Wundegraph](https://github.com/wundergraph/cosmo) | \`${{ fromJson(needs.read.outputs.result).big_wundergraph_no_opt.rps }} RPS\` | \`${{ fromJson(needs.read.outputs.result).medium_wundergraph_no_opt.rps }} RPS\` | \`${{ fromJson(needs.read.outputs.result).small_wundergraph_no_opt.rps }} RPS\` |" >> "results_rps.md" | |
| echo "| [Apollo](https://github.com/apollographql/router) | \`${{ fromJson(needs.read.outputs.result).big_apollo_router.rps }} RPS\` | \`${{ fromJson(needs.read.outputs.result).medium_apollo_router.rps }} RPS\` | \`${{ fromJson(needs.read.outputs.result).small_apollo_router.rps }} RPS\` |" >> "results_rps.md" | |
| echo "| **Cached** | | | |" >> "results_rps.md" | |
| echo "| [Tailcall](https://github.com/tailcallhq/tailcall) | \`${{ fromJson(needs.read.outputs.result).big_tailcall_full_conf.rps }} RPS\` | \`${{ fromJson(needs.read.outputs.result).medium_tailcall_full_conf.rps }} RPS\` | \`${{ fromJson(needs.read.outputs.result).small_tailcall_full_conf.rps }} RPS\` |" >> "results_rps.md" | |
| echo "| [Grafbase](https://github.com/grafbase/grafbase) | \`${{ fromJson(needs.read.outputs.result).big_grafbase_cache.rps }} RPS\` | \`${{ fromJson(needs.read.outputs.result).medium_grafbase_cache.rps }} RPS\` | \`${{ fromJson(needs.read.outputs.result).small_grafbase_cache.rps }} RPS\` |" >> "results_rps.md" | |
| echo "| [Wundegraph](https://github.com/wundergraph/cosmo) | \`${{ fromJson(needs.read.outputs.result).big_wundergraph_default.rps }} RPS\` | \`${{ fromJson(needs.read.outputs.result).medium_wundergraph_default.rps }} RPS\` | \`${{ fromJson(needs.read.outputs.result).small_wundergraph_default.rps }} RPS\` |" >> "results_rps.md" | |
| - name: Show RPS table | |
| run: cat "results_rps.md" | |
| - name: Collect latency from all matrix jobs | |
| run: | | |
| echo "| Server | [112,838 bytes](./source/big.json)| [12598 bytes](./source/medium.json)| [362 bytes](./source/small.json) |" > "results_p95.md" | |
| echo "| ---: | ---: | ---: | ---: |" >> "results_p95.md" | |
| echo "| [Nginx](https://nginx.org/en/) | \`${{ fromJson(needs.read.outputs.result).big_nginx_graphql.p95 }}\` | \`${{ fromJson(needs.read.outputs.result).medium_nginx_graphql.p95 }}\` | \`${{ fromJson(needs.read.outputs.result).small_nginx_graphql.p95 }}\` |" >> "results_p95.md" | |
| echo "| **Base** | | | |" >> "results_p95.md" | |
| echo "| [Tailcall](https://github.com/tailcallhq/tailcall) | \`${{ fromJson(needs.read.outputs.result).big_tailcall_default.p95 }}\` | \`${{ fromJson(needs.read.outputs.result).medium_tailcall_default.p95 }}\` | \`${{ fromJson(needs.read.outputs.result).small_tailcall_default.p95 }}\` |" >> "results_p95.md" | |
| echo "| [Grafbase](https://github.com/grafbase/grafbase) | \`${{ fromJson(needs.read.outputs.result).big_grafbase_default.p95 }}\` | \`${{ fromJson(needs.read.outputs.result).medium_grafbase_default.p95 }}\` | \`${{ fromJson(needs.read.outputs.result).small_grafbase_default.p95 }}\` |" >> "results_p95.md" | |
| echo "| [Wundegraph](https://github.com/wundergraph/cosmo) | \`${{ fromJson(needs.read.outputs.result).big_wundergraph_no_opt.p95 }}\` | \`${{ fromJson(needs.read.outputs.result).medium_wundergraph_no_opt.p95 }}\` | \`${{ fromJson(needs.read.outputs.result).small_wundergraph_no_opt.p95 }}\` |" >> "results_p95.md" | |
| echo "| [Apollo](https://github.com/apollographql/router) | \`${{ fromJson(needs.read.outputs.result).big_apollo_router.p95 }}\` | \`${{ fromJson(needs.read.outputs.result).medium_apollo_router.p95 }}\` | \`${{ fromJson(needs.read.outputs.result).small_apollo_router.p95 }}\` |" >> "results_p95.md" | |
| echo "| **Cached** | | | |" >> "results_p95.md" | |
| echo "| [Tailcall](https://github.com/tailcallhq/tailcall) | \`${{ fromJson(needs.read.outputs.result).big_tailcall_full_conf.p95 }}\` | \`${{ fromJson(needs.read.outputs.result).medium_tailcall_full_conf.p95 }}\` | \`${{ fromJson(needs.read.outputs.result).small_tailcall_full_conf.p95 }}\` |" >> "results_p95.md" | |
| echo "| [Grafbase](https://github.com/grafbase/grafbase) | \`${{ fromJson(needs.read.outputs.result).big_grafbase_cache.p95 }}\` | \`${{ fromJson(needs.read.outputs.result).medium_grafbase_cache.p95 }}\` | \`${{ fromJson(needs.read.outputs.result).small_grafbase_cache.p95 }}\` |" >> "results_p95.md" | |
| echo "| [Wundegraph](https://github.com/wundergraph/cosmo) | \`${{ fromJson(needs.read.outputs.result).big_wundergraph_default.p95 }}\` | \`${{ fromJson(needs.read.outputs.result).medium_wundergraph_default.p95 }}\` | \`${{ fromJson(needs.read.outputs.result).small_wundergraph_default.p95 }}\` |" >> "results_p95.md" | |
| - name: Show P95 Latency table | |
| run: cat "results_p95.md" | |
| - name: Update `README.md` file with RPS data | |
| run: | | |
| sed -i '/<!-- PERFORMANCE_RESULTS_START -->/,/<!-- PERFORMANCE_RESULTS_END -->/ { | |
| /<!-- PERFORMANCE_RESULTS_START -->/!{ | |
| /<!-- PERFORMANCE_RESULTS_END -->/!d | |
| } | |
| }' README.md | |
| sed -i '/<!-- PERFORMANCE_RESULTS_START -->/r results_rps.md' README.md | |
| - name: Update `README.md` file with P95 latency data | |
| run: | | |
| sed -i '/<!-- LATENCY_RESULTS_START -->/,/<!-- LATENCY_RESULTS_END -->/ { | |
| /<!-- LATENCY_RESULTS_START -->/!{ | |
| /<!-- LATENCY_RESULTS_END -->/!d | |
| } | |
| }' README.md | |
| sed -i '/<!-- LATENCY_RESULTS_START -->/r results_p95.md' README.md | |
| - name: Summarize | |
| run: | | |
| echo "### RPS Table" >> "$GITHUB_STEP_SUMMARY" | |
| cat "results_rps.md" >> "$GITHUB_STEP_SUMMARY" | |
| echo "### P95 Latency Table" >> "$GITHUB_STEP_SUMMARY" | |
| cat "results_p95.md" >> "$GITHUB_STEP_SUMMARY" | |
| - name: Cleanup files | |
| run: | | |
| rm "results_rps.md" | |
| rm "results_p95.md" | |
| - name: Comment `README.md` file | |
| if: github.event_name == 'pull_request_target' | |
| uses: peter-evans/commit-comment@v3 | |
| with: | |
| sha: ${{ github.event.pull_request.head.sha }} | |
| body-path: "README.md" | |
| reactions: eyes | |
| - name: Commit and push changes (on main branch) | |
| if: github.event_name == 'push' && github.ref == 'refs/heads/main' | |
| uses: stefanzweifel/git-auto-commit-action@v5 | |
| with: | |
| branch: main | |
| commit_author: Author <[email protected]> | |
| commit_message: "[ci skip] update performance results in README.md" |