Merge branch 'main' into nginx-delay

.devcontainer/devcontainer.json

@@ -10,7 +10,7 @@
     "dockerfile": "Dockerfile",
     "context": ".."
   },
-  "runArgs": ["-v", "/var/run/docker.sock:/var/run/docker.sock"],
+  "runArgs": ["-v", "/var/run/docker.sock:/var/run/docker.sock", "--network=host"],
   "forwardPorts": [8000, 3000],
   "postCreateCommand": "bash ./setup.sh"
 }

.github/workflows/bench.yml

@@ -1,38 +1,78 @@
 name: "Run benchmark"
-
 on:
   pull_request_target:
     types: [assigned, opened, synchronize, reopened, edited]
   push:
     branches:
       - main
-
 permissions:
   contents: write
   pull-requests: write
   issues: write
-
 jobs:
   build:
     runs-on: benchmarking-runner
     if: github.event.head_commit.message != 'Update performance results in README.md'
     env:
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    strategy:
+      matrix:
+        service:
+          [
+            apollo_server,
+            caliban,
+            netflix_dgs,
+            gqlgen,
+            tailcall,
+            async_graphql,
+            hasura,
+            graphql_jit,
+          ]
     steps:
       - name: Checkout (GitHub)
         uses: actions/checkout@v4
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
           ref: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.sha || github.sha }}
 
-      - name: Build devcontainer and run benchmarks
+      - name: Build devcontainer and run setup and benchmark
         uses: devcontainers/[email protected]
         with:
           imageName: graphql-benchmarks
           push: never
           runCmd: |
-            bash ./setup.sh
-            bash ./run_benchmarks.sh
+            bash ./graphql/${{ matrix.service }}/setup.sh
+            bash run_benchmarks.sh ${{ matrix.service }}
+
+      - name: List benchmark files
+        run: |
+          ls -la bench*.txt || echo "No matching files found"
+
+      - name: Upload benchmark results
+        uses: actions/upload-artifact@v3
+        with:
+          name: benchmark-results
+          path: bench*.txt
+
+  analyze:
+    needs: build
+    runs-on: benchmarking-runner
+    steps:
+      - name: Checkout (GitHub)
+        uses: actions/checkout@v4
+
+      - name: Download all benchmark results
+        uses: actions/download-artifact@v3
+        with:
+          name: benchmark-results
+          path: .
+
+      - name: List downloaded artifacts
+        run: ls -la bench*.txt || echo "No matching files found"
+
+      - name: Analyze results
+        run: |
+          bash run_analyze_script.sh
 
       - name: Print benchmark results
         run: cat ./results.md
@@ -51,4 +91,4 @@ jobs:
         with:
           branch: main
           commit_author: Author <[email protected]>
-          commit_message: "Update performance results in README.md"
+          commit_message: "[ci skip] update performance results in README.md"

README.md

@@ -51,32 +51,32 @@ Get started with the benchmarks:
 | Query | Server | Requests/sec | Latency (ms) | Relative |
 |-------:|--------:|--------------:|--------------:|---------:|
 | 1 | `{ posts { id userId title user { id name email }}}` |
-|| [Tailcall] | `29,865.00` | `3.34` | `112.98x` |
-|| [async-graphql] | `1,824.45` | `54.68` | `6.90x` |
-|| [Caliban] | `1,631.74` | `60.96` | `6.17x` |
-|| [Hasura] | `1,513.32` | `65.92` | `5.72x` |
-|| [GraphQL JIT] | `1,308.30` | `76.09` | `4.95x` |
-|| [Gqlgen] | `771.42` | `128.66` | `2.92x` |
-|| [Netflix DGS] | `360.56` | `174.79` | `1.36x` |
-|| [Apollo GraphQL] | `264.34` | `370.65` | `1.00x` |
+|| [Tailcall] | `27,894.20` | `3.57` | `116.40x` |
+|| [async-graphql] | `1,925.61` | `52.56` | `8.04x` |
+|| [Caliban] | `1,755.13` | `56.75` | `7.32x` |
+|| [GraphQL JIT] | `1,260.61` | `79.00` | `5.26x` |
+|| [Gqlgen] | `774.95` | `128.05` | `3.23x` |
+|| [Netflix DGS] | `363.59` | `177.99` | `1.52x` |
+|| [Apollo GraphQL] | `269.21` | `364.28` | `1.12x` |
+|| [Hasura] | `239.63` | `411.81` | `1.00x` |
 | 2 | `{ posts { title }}` |
-|| [Tailcall] | `61,611.40` | `1.61` | `45.11x` |
-|| [Caliban] | `9,406.21` | `10.99` | `6.89x` |
-|| [async-graphql] | `9,366.90` | `10.76` | `6.86x` |
-|| [Hasura] | `2,450.62` | `40.79` | `1.79x` |
-|| [Gqlgen] | `2,150.47` | `48.08` | `1.57x` |
-|| [Apollo GraphQL] | `1,745.86` | `57.22` | `1.28x` |
-|| [Netflix DGS] | `1,602.77` | `69.19` | `1.17x` |
-|| [GraphQL JIT] | `1,365.94` | `73.10` | `1.00x` |
+|| [Tailcall] | `57,871.20` | `1.72` | `78.98x` |
+|| [Caliban] | `10,019.30` | `10.31` | `13.67x` |
+|| [async-graphql] | `9,838.23` | `10.26` | `13.43x` |
+|| [Gqlgen] | `2,181.29` | `47.51` | `2.98x` |
+|| [Apollo GraphQL] | `1,743.58` | `57.29` | `2.38x` |
+|| [Netflix DGS] | `1,587.87` | `71.10` | `2.17x` |
+|| [GraphQL JIT] | `1,325.13` | `75.35` | `1.81x` |
+|| [Hasura] | `732.74` | `137.01` | `1.00x` |
 | 3 | `{ greet }` |
-|| [Caliban] | `69,058.50` | `1.06` | `27.21x` |
-|| [Tailcall] | `63,879.10` | `1.58` | `25.17x` |
-|| [async-graphql] | `50,746.10` | `2.10` | `20.00x` |
-|| [Gqlgen] | `46,935.70` | `5.11` | `18.50x` |
-|| [Netflix DGS] | `8,233.67` | `14.43` | `3.24x` |
-|| [Apollo GraphQL] | `8,028.37` | `12.66` | `3.16x` |
-|| [GraphQL JIT] | `5,097.42` | `19.59` | `2.01x` |
-|| [Hasura] | `2,537.66` | `39.39` | `1.00x` |
+|| [Caliban] | `67,294.20` | `1.11` | `26.65x` |
+|| [Tailcall] | `58,715.80` | `1.71` | `23.25x` |
+|| [Gqlgen] | `47,750.10` | `5.05` | `18.91x` |
+|| [async-graphql] | `46,947.60` | `2.15` | `18.59x` |
+|| [Netflix DGS] | `8,157.51` | `14.95` | `3.23x` |
+|| [Apollo GraphQL] | `8,061.32` | `12.58` | `3.19x` |
+|| [GraphQL JIT] | `5,041.54` | `19.80` | `2.00x` |
+|| [Hasura] | `2,524.95` | `40.06` | `1.00x` |
 
 <!-- PERFORMANCE_RESULTS_END -->
 

analyze.sh

@@ -1,5 +1,8 @@
 #!/bin/bash
 
+# Install gnuplot
+sudo apt-get update && sudo apt-get install -y gnuplot
+
 function extractMetric() {
   local file="$1"
   local metric="$2"
@@ -161,4 +164,4 @@ mv $latencyHistogramFile assets/
 # Delete the result TXT files
 for file in "${resultFiles[@]}"; do
   rm "$file"
-done
+done

graphql/apollo_server/setup.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+# For apollo server:
+cd graphql/apollo_server
+npm i
+cd ../../