diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index fa4bc97a..e4ee18c0 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -1,42 +1,73 @@ name: "Run benchmark" - on: pull_request_target: types: [assigned, opened, synchronize, reopened, edited] push: branches: - main - permissions: contents: write pull-requests: write issues: write - jobs: build: runs-on: benchmarking-runner if: github.event.head_commit.message != 'Update performance results in README.md' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + strategy: + matrix: + service: [apollo_server, caliban, netflix_dgs, gqlgen, tailcall, async_graphql, hasura, graphql_jit] steps: - name: Checkout (GitHub) uses: actions/checkout@v4 with: token: ${{ secrets.GITHUB_TOKEN }} ref: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.sha || github.sha }} - - - name: Build devcontainer and run benchmarks + + - name: Build devcontainer and run setup and benchmark uses: devcontainers/ci@v0.3 with: imageName: graphql-benchmarks push: never runCmd: | - bash ./setup.sh - bash ./run_benchmarks.sh + bash ./graphql/${{ matrix.service }}/setup.sh + bash run_benchmarks.sh ${{ matrix.service }} + + - name: List benchmark files + run: | + ls -la bench*.txt || echo "No matching files found" + + - name: Upload benchmark results + uses: actions/upload-artifact@v3 + with: + name: benchmark-results + path: bench*.txt + + + analyze: + needs: benchmark + runs-on: benchmarking-runner + steps: + - name: Checkout (GitHub) + uses: actions/checkout@v4 + + - name: Download all benchmark results + uses: actions/download-artifact@v3 + with: + name: benchmark-results + path: . + + - name: List downloaded artifacts + run: ls -la bench*.txt || echo "No matching files found" + + - name: Analyze results + run: | + bash run_analyze_script.sh - name: Print benchmark results run: cat ./results.md - + - name: Comment benchmark results on PR if: github.event_name == 'pull_request_target' uses: peter-evans/commit-comment@v3 @@ -44,7 +75,7 @@ jobs: sha: ${{ github.event.pull_request.head.sha }} body-path: "results.md" reactions: eyes - + - name: Commit and push changes (on main branch) if: github.event_name == 'push' && github.ref == 'refs/heads/main' uses: stefanzweifel/git-auto-commit-action@v5 diff --git a/analyze.sh b/analyze.sh index 17bbbce1..a52e8471 100755 --- a/analyze.sh +++ b/analyze.sh @@ -1,5 +1,8 @@ #!/bin/bash +# Install gnuplot +sudo apt-get update && sudo apt-get install -y gnuplot + function extractMetric() { local file="$1" local metric="$2" @@ -161,4 +164,4 @@ mv $latencyHistogramFile assets/ # Delete the result TXT files for file in "${resultFiles[@]}"; do rm "$file" -done +done \ No newline at end of file diff --git a/run_analyze_script.sh b/run_analyze_script.sh index 45eeec16..84f7206a 100755 --- a/run_analyze_script.sh +++ b/run_analyze_script.sh @@ -1,13 +1,36 @@ #!/bin/bash -rm results.md +# Update and install gnuplot +sudo apt-get update && sudo apt-get install -y gnuplot +# Remove existing results file +rm -f results.md + +services=("apollo" "caliban" "netflixdgs" "gqlgen" "tailcall" "async_graphql" "hasura" "graphql_jit") + +# Loop through each benchmark (1, 2, 3) for bench in 1 2 3; do - if ls bench${bench}*.txt &> /dev/null; then - echo "Processing files for bench${bench}:" - bash analyze.sh bench${bench}*.txt - echo "Files processed: $(ls bench${bench}*.txt)" - else - echo "No matching files found for bench${bench}*.txt" - fi + echo "Processing files for bench${bench}:" + + # Construct the command for each benchmark + cmd="bash analyze.sh" + + # Loop through each service + for service in "${services[@]}"; do + # Convert service name to match file naming convention + case $service in + "apollo") file_service="apollo_server" ;; + "netflixdgs") file_service="netflix_dgs" ;; + *) file_service=$service ;; + esac + + # Add files for current service to the command + for run in 1 2 3; do + cmd+=" bench${bench}_result${run}_graphql_${file_service}_run.sh.txt" + done + done + + # Execute the command + echo "Executing: $cmd" + eval $cmd done \ No newline at end of file diff --git a/run_benchmarks.sh b/run_benchmarks.sh index 37280277..9c1b1aa8 100755 --- a/run_benchmarks.sh +++ b/run_benchmarks.sh @@ -12,6 +12,7 @@ function killServerOnPort() { echo "No process found running on port $port" fi } + bench1Results=() bench2Results=() bench3Results=() @@ -72,17 +73,29 @@ function runBenchmark() { rm "results.md" -for service in "apollo_server" "caliban" "netflix_dgs" "gqlgen" "tailcall" "async_graphql" "hasura" "graphql_jit"; do - runBenchmark "graphql/${service}/run.sh" - if [ "$service" == "apollo_server" ]; then +# Main script +if [ $# -eq 0 ]; then + echo "Usage: $0 " + echo "Available services: apollo_server, caliban, netflix_dgs, gqlgen, tailcall, async_graphql, hasura, graphql_jit" + exit 1 +fi + +service="$1" +valid_services=("apollo_server" "caliban" "netflix_dgs" "gqlgen" "tailcall" "async_graphql" "hasura" "graphql_jit") + +if [[ ! " ${valid_services[@]} " =~ " ${service} " ]]; then + echo "Invalid service name. Available services: ${valid_services[*]}" + exit 1 +fi + + + +runBenchmark "graphql/${service}/run.sh" + +if [ "$service" == "apollo_server" ]; then cd graphql/apollo_server/ npm stop cd ../../ - elif [ "$service" == "hasura" ]; then +elif [ "$service" == "hasura" ]; then bash "graphql/hasura/kill.sh" - fi -done - -bash analyze.sh "${bench1Results[@]}" -bash analyze.sh "${bench2Results[@]}" -bash analyze.sh "${bench3Results[@]}" +fi \ No newline at end of file