Benchmarks in parallel (#357)

.github/workflows/bench.yml

@@ -1,50 +1,81 @@
 name: "Run benchmark"
-
 on:
   pull_request_target:
     types: [assigned, opened, synchronize, reopened, edited]
   push:
     branches:
       - main
-
 permissions:
   contents: write
   pull-requests: write
   issues: write
-
 jobs:
   build:
     runs-on: benchmarking-runner
     if: github.event.head_commit.message != 'Update performance results in README.md'
     env:
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    strategy:
+      matrix:
+        service: [apollo_server, caliban, netflix_dgs, gqlgen, tailcall, async_graphql, hasura, graphql_jit]
     steps:
       - name: Checkout (GitHub)
         uses: actions/checkout@v4
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
           ref: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.sha || github.sha }}
-
-      - name: Build devcontainer and run benchmarks
+      
+      - name: Build devcontainer and run setup and benchmark
         uses: devcontainers/[email protected]
         with:
           imageName: graphql-benchmarks
           push: never
           runCmd: |
-            bash ./setup.sh
-            bash ./run_benchmarks.sh
+            bash ./graphql/${{ matrix.service }}/setup.sh
+            bash run_benchmarks.sh ${{ matrix.service }}
+
+      - name: List benchmark files
+        run: |
+          ls -la bench*.txt || echo "No matching files found"
+          
+      - name: Upload benchmark results
+        uses: actions/upload-artifact@v3
+        with:
+          name: benchmark-results
+          path: bench*.txt
+
+
+  analyze:
+    needs: benchmark
+    runs-on: benchmarking-runner
+    steps:
+      - name: Checkout (GitHub)
+        uses: actions/checkout@v4
+
+      - name: Download all benchmark results
+        uses: actions/download-artifact@v3
+        with:
+          name: benchmark-results
+          path: .
+
+      - name: List downloaded artifacts
+        run: ls -la bench*.txt || echo "No matching files found"
+
+      - name: Analyze results
+        run: |
+          bash run_analyze_script.sh
 
       - name: Print benchmark results
         run: cat ./results.md
-
+      
       - name: Comment benchmark results on PR
         if: github.event_name == 'pull_request_target'
         uses: peter-evans/commit-comment@v3
         with:
           sha: ${{ github.event.pull_request.head.sha }}
           body-path: "results.md"
           reactions: eyes
-
+      
       - name: Commit and push changes (on main branch)
         if: github.event_name == 'push' && github.ref == 'refs/heads/main'
         uses: stefanzweifel/git-auto-commit-action@v5

analyze.sh

@@ -1,5 +1,8 @@
 #!/bin/bash
 
+# Install gnuplot
+sudo apt-get update && sudo apt-get install -y gnuplot
+
 function extractMetric() {
   local file="$1"
   local metric="$2"
@@ -161,4 +164,4 @@ mv $latencyHistogramFile assets/
 # Delete the result TXT files
 for file in "${resultFiles[@]}"; do
   rm "$file"
-done
+done

run_analyze_script.sh

@@ -1,13 +1,36 @@
 #!/bin/bash
 
-rm results.md
+# Update and install gnuplot
+sudo apt-get update && sudo apt-get install -y gnuplot
 
+# Remove existing results file
+rm -f results.md
+
+services=("apollo" "caliban" "netflixdgs" "gqlgen" "tailcall" "async_graphql" "hasura" "graphql_jit")
+
+# Loop through each benchmark (1, 2, 3)
 for bench in 1 2 3; do
-    if ls bench${bench}*.txt &> /dev/null; then
-        echo "Processing files for bench${bench}:"
-        bash analyze.sh bench${bench}*.txt
-        echo "Files processed: $(ls bench${bench}*.txt)"
-    else
-        echo "No matching files found for bench${bench}*.txt"
-    fi
+    echo "Processing files for bench${bench}:"
+
+    # Construct the command for each benchmark
+    cmd="bash analyze.sh"
+
+    # Loop through each service 
+    for service in "${services[@]}"; do
+        # Convert service name to match file naming convention 
+        case $service in
+            "apollo") file_service="apollo_server" ;;
+            "netflixdgs") file_service="netflix_dgs" ;;
+            *) file_service=$service ;;
+        esac
+
+        # Add files for current service to the command
+        for run in 1 2 3; do
+            cmd+=" bench${bench}_result${run}_graphql_${file_service}_run.sh.txt"
+        done
+    done
+
+    # Execute the command
+    echo "Executing: $cmd"
+    eval $cmd
 done

run_benchmarks.sh

@@ -12,6 +12,7 @@ function killServerOnPort() {
     echo "No process found running on port $port"
   fi
 }
+
 bench1Results=()
 bench2Results=()
 bench3Results=()
@@ -72,17 +73,29 @@ function runBenchmark() {
 
 rm "results.md"
 
-for service in "apollo_server" "caliban" "netflix_dgs" "gqlgen" "tailcall" "async_graphql" "hasura" "graphql_jit"; do
-  runBenchmark "graphql/${service}/run.sh"
-  if [ "$service" == "apollo_server" ]; then
+# Main script
+if [ $# -eq 0 ]; then
+    echo "Usage: $0 <service_name>"
+    echo "Available services: apollo_server, caliban, netflix_dgs, gqlgen, tailcall, async_graphql, hasura, graphql_jit"
+    exit 1
+fi  
+
+service="$1"
+valid_services=("apollo_server" "caliban" "netflix_dgs" "gqlgen" "tailcall" "async_graphql" "hasura" "graphql_jit")
+
+if [[ ! " ${valid_services[@]} " =~ " ${service} " ]]; then
+    echo "Invalid service name. Available services: ${valid_services[*]}"
+    exit 1
+fi
+
+
+
+runBenchmark "graphql/${service}/run.sh"
+
+if [ "$service" == "apollo_server" ]; then
     cd graphql/apollo_server/
     npm stop
     cd ../../
-  elif [ "$service" == "hasura" ]; then
+elif [ "$service" == "hasura" ]; then
     bash "graphql/hasura/kill.sh"
-  fi
-done
-
-bash analyze.sh "${bench1Results[@]}"
-bash analyze.sh "${bench2Results[@]}"
-bash analyze.sh "${bench3Results[@]}"
+fi