diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index fa4bc97a..e4ee18c0 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -1,42 +1,73 @@ name: "Run benchmark" - on: pull_request_target: types: [assigned, opened, synchronize, reopened, edited] push: branches: - main - permissions: contents: write pull-requests: write issues: write - jobs: build: runs-on: benchmarking-runner if: github.event.head_commit.message != 'Update performance results in README.md' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + strategy: + matrix: + service: [apollo_server, caliban, netflix_dgs, gqlgen, tailcall, async_graphql, hasura, graphql_jit] steps: - name: Checkout (GitHub) uses: actions/checkout@v4 with: token: ${{ secrets.GITHUB_TOKEN }} ref: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.sha || github.sha }} - - - name: Build devcontainer and run benchmarks + + - name: Build devcontainer and run setup and benchmark uses: devcontainers/ci@v0.3 with: imageName: graphql-benchmarks push: never runCmd: | - bash ./setup.sh - bash ./run_benchmarks.sh + bash ./graphql/${{ matrix.service }}/setup.sh + bash run_benchmarks.sh ${{ matrix.service }} + + - name: List benchmark files + run: | + ls -la bench*.txt || echo "No matching files found" + + - name: Upload benchmark results + uses: actions/upload-artifact@v3 + with: + name: benchmark-results + path: bench*.txt + + + analyze: + needs: benchmark + runs-on: benchmarking-runner + steps: + - name: Checkout (GitHub) + uses: actions/checkout@v4 + + - name: Download all benchmark results + uses: actions/download-artifact@v3 + with: + name: benchmark-results + path: . + + - name: List downloaded artifacts + run: ls -la bench*.txt || echo "No matching files found" + + - name: Analyze results + run: | + bash run_analyze_script.sh - name: Print benchmark results run: cat ./results.md - + - name: Comment benchmark results on PR if: github.event_name == 'pull_request_target' uses: peter-evans/commit-comment@v3 @@ -44,7 +75,7 @@ jobs: sha: ${{ github.event.pull_request.head.sha }} body-path: "results.md" reactions: eyes - + - name: Commit and push changes (on main branch) if: github.event_name == 'push' && github.ref == 'refs/heads/main' uses: stefanzweifel/git-auto-commit-action@v5 diff --git a/README.md b/README.md index 998aac57..9337e111 100644 --- a/README.md +++ b/README.md @@ -51,32 +51,32 @@ Get started with the benchmarks: | Query | Server | Requests/sec | Latency (ms) | Relative | |-------:|--------:|--------------:|--------------:|---------:| | 1 | `{ posts { id userId title user { id name email }}}` | -|| [Tailcall] | `28,320.00` | `3.52` | `175.34x` | -|| [async-graphql] | `1,784.61` | `57.51` | `11.05x` | -|| [Caliban] | `1,523.52` | `65.29` | `9.43x` | -|| [GraphQL JIT] | `1,311.50` | `75.93` | `8.12x` | -|| [Gqlgen] | `744.41` | `133.27` | `4.61x` | -|| [Netflix DGS] | `354.53` | `192.91` | `2.20x` | -|| [Apollo GraphQL] | `264.73` | `370.46` | `1.64x` | -|| [Hasura] | `161.51` | `496.54` | `1.00x` | +|| [Tailcall] | `29,496.60` | `3.38` | `196.74x` | +|| [async-graphql] | `1,805.45` | `55.70` | `12.04x` | +|| [Caliban] | `1,547.91` | `64.33` | `10.32x` | +|| [GraphQL JIT] | `1,335.26` | `74.60` | `8.91x` | +|| [Gqlgen] | `772.80` | `128.41` | `5.15x` | +|| [Netflix DGS] | `358.06` | `177.42` | `2.39x` | +|| [Apollo GraphQL] | `268.25` | `366.16` | `1.79x` | +|| [Hasura] | `149.93` | `549.07` | `1.00x` | | 2 | `{ posts { title }}` | -|| [Tailcall] | `58,223.00` | `1.71` | `68.82x` | -|| [Caliban] | `9,102.02` | `11.33` | `10.76x` | -|| [async-graphql] | `9,042.00` | `11.20` | `10.69x` | -|| [Gqlgen] | `2,136.74` | `48.34` | `2.53x` | -|| [Apollo GraphQL] | `1,742.58` | `57.30` | `2.06x` | -|| [Netflix DGS] | `1,584.25` | `69.49` | `1.87x` | -|| [GraphQL JIT] | `1,337.30` | `74.68` | `1.58x` | -|| [Hasura] | `845.98` | `117.95` | `1.00x` | +|| [Tailcall] | `58,939.60` | `1.69` | `67.77x` | +|| [Caliban] | `9,238.76` | `11.16` | `10.62x` | +|| [async-graphql] | `9,138.76` | `11.34` | `10.51x` | +|| [Gqlgen] | `2,136.19` | `48.48` | `2.46x` | +|| [Apollo GraphQL] | `1,775.87` | `56.26` | `2.04x` | +|| [Netflix DGS] | `1,593.80` | `69.82` | `1.83x` | +|| [GraphQL JIT] | `1,399.68` | `71.35` | `1.61x` | +|| [Hasura] | `869.69` | `114.81` | `1.00x` | | 3 | `{ greet }` | -|| [Caliban] | `66,940.20` | `1.09` | `26.59x` | -|| [Tailcall] | `59,316.40` | `1.70` | `23.56x` | -|| [async-graphql] | `47,630.90` | `2.14` | `18.92x` | -|| [Gqlgen] | `46,908.30` | `5.23` | `18.63x` | -|| [Netflix DGS] | `8,151.83` | `15.19` | `3.24x` | -|| [Apollo GraphQL] | `7,931.63` | `12.75` | `3.15x` | -|| [GraphQL JIT] | `5,147.67` | `19.40` | `2.04x` | -|| [Hasura] | `2,517.36` | `39.64` | `1.00x` | +|| [Caliban] | `67,637.00` | `1.07` | `25.83x` | +|| [Tailcall] | `59,557.80` | `1.69` | `22.75x` | +|| [Gqlgen] | `47,525.20` | `5.21` | `18.15x` | +|| [async-graphql] | `47,150.60` | `2.21` | `18.01x` | +|| [Netflix DGS] | `8,261.44` | `15.02` | `3.16x` | +|| [Apollo GraphQL] | `8,186.12` | `12.54` | `3.13x` | +|| [GraphQL JIT] | `5,273.45` | `18.93` | `2.01x` | +|| [Hasura] | `2,618.18` | `38.13` | `1.00x` | diff --git a/analyze.sh b/analyze.sh new file mode 100755 index 00000000..a52e8471 --- /dev/null +++ b/analyze.sh @@ -0,0 +1,167 @@ +#!/bin/bash + +# Install gnuplot +sudo apt-get update && sudo apt-get install -y gnuplot + +function extractMetric() { + local file="$1" + local metric="$2" + grep "$metric" "$file" | awk '{print $2}' | sed 's/ms//' +} + +function average() { + echo "$@" | awk '{for(i=1;i<=NF;i++) s+=$i; print s/NF}' +} + +declare -A formattedServerNames +formattedServerNames=( + ["tailcall"]="Tailcall" + ["gqlgen"]="Gqlgen" + ["apollo"]="Apollo GraphQL" + ["netflixdgs"]="Netflix DGS" + ["caliban"]="Caliban" + ["async_graphql"]="async-graphql" + ["hasura"]="Hasura" + ["graphql_jit"]="GraphQL JIT" +) + +servers=("apollo" "caliban" "netflixdgs" "gqlgen" "tailcall" "async_graphql" "hasura" "graphql_jit") +resultFiles=("$@") +declare -A avgReqSecs +declare -A avgLatencies + +# Extract metrics and calculate averages +for idx in "${!servers[@]}"; do + startIdx=$((idx * 3)) + reqSecVals=() + latencyVals=() + for j in 0 1 2; do + fileIdx=$((startIdx + j)) + reqSecVals+=($(extractMetric "${resultFiles[$fileIdx]}" "Requests/sec")) + latencyVals+=($(extractMetric "${resultFiles[$fileIdx]}" "Latency")) + done + avgReqSecs[${servers[$idx]}]=$(average "${reqSecVals[@]}") + avgLatencies[${servers[$idx]}]=$(average "${latencyVals[@]}") +done + +# Generating data files for gnuplot +reqSecData="/tmp/reqSec.dat" +latencyData="/tmp/latency.dat" + +echo "Server Value" >"$reqSecData" +for server in "${servers[@]}"; do + echo "$server ${avgReqSecs[$server]}" >>"$reqSecData" +done + +echo "Server Value" >"$latencyData" +for server in "${servers[@]}"; do + echo "$server ${avgLatencies[$server]}" >>"$latencyData" +done + +whichBench=1 +if [[ $1 == bench2* ]]; then + whichBench=2 +elif [[ $1 == bench3* ]]; then + whichBench=3 +fi + +reqSecHistogramFile="req_sec_histogram${whichBench}.png" +latencyHistogramFile="latency_histogram${whichBench}.png" + +# Plotting using gnuplot +gnuplot <<-EOF + set term pngcairo size 1280,720 enhanced font "Courier,12" + set output "$reqSecHistogramFile" + set style data histograms + set style histogram cluster gap 1 + set style fill solid border -1 + set xtics rotate by -45 + set boxwidth 0.9 + set title "Requests/Sec" + stats "$reqSecData" using 2 nooutput + set yrange [0:STATS_max*1.2] + set key outside right top + plot "$reqSecData" using 2:xtic(1) title "Req/Sec" + + set output "$latencyHistogramFile" + set title "Latency (in ms)" + stats "$latencyData" using 2 nooutput + set yrange [0:STATS_max*1.2] + plot "$latencyData" using 2:xtic(1) title "Latency" +EOF + +# Move PNGs to assets +mkdir -p assets +mv $reqSecHistogramFile assets/ +mv $latencyHistogramFile assets/ + +# Declare an associative array for server RPS +declare -A serverRPS + +# Populate the serverRPS array +for server in "${servers[@]}"; do + serverRPS[$server]=${avgReqSecs[$server]} +done + +# Get the servers sorted by RPS in descending order +IFS=$'\n' sortedServers=($(for server in "${!serverRPS[@]}"; do echo "$server ${serverRPS[$server]}"; done | sort -rn -k2 | cut -d' ' -f1)) + +echo "Sorted servers: ${sortedServers[@]}" +lastServer="${sortedServers[-1]}" +lastServerReqSecs=${avgReqSecs[$lastServer]} + +# Start building the resultsTable +if [[ $whichBench == 1 ]]; then + resultsTable="\n\n| Query | Server | Requests/sec | Latency (ms) | Relative |\n|-------:|--------:|--------------:|--------------:|---------:|\n| $whichBench | \`{ posts { id userId title user { id name email }}}\` |" +elif [[ $whichBench == 2 ]]; then + resultsTable="| $whichBench | \`{ posts { title }}\` |" +elif [[ $whichBench == 3 ]]; then + resultsTable="| $whichBench | \`{ greet }\` |" +fi + +# Build the resultsTable with sorted servers and formatted numbers +for server in "${sortedServers[@]}"; do + formattedReqSecs=$(printf "%.2f" ${avgReqSecs[$server]} | perl -pe 's/(?<=\d)(?=(\d{3})+(\.\d*)?$)/,/g') + formattedLatencies=$(printf "%.2f" ${avgLatencies[$server]} | perl -pe 's/(?<=\d)(?=(\d{3})+(\.\d*)?$)/,/g') + # Calculate the relative performance + relativePerformance=$(echo "${avgReqSecs[$server]} $lastServerReqSecs" | awk '{printf "%.2f", $1 / $2}') + + resultsTable+="\n|| [${formattedServerNames[$server]}] | \`${formattedReqSecs}\` | \`${formattedLatencies}\` | \`${relativePerformance}x\` |" +done + +if [[ $whichBench == 3 ]]; then + resultsTable+="\n\n" +fi + +echo "resultsTable: $resultsTable" + +# Print the results table in a new file +resultsFile="results.md" +echo -e $resultsTable >> $resultsFile + + +if [[ $whichBench == 3 ]]; then + finalResults=$(printf '%s\n' "$(cat $resultsFile)" | sed 's/$/\\n/'| tr -d '\n') + # Remove the last newline character + finalResults=${finalResults::-2} + + # Print the results as a table in the terminal + echo -e $finalResults | sed "s///;s///" + # Check if the markers are present + if grep -q "PERFORMANCE_RESULTS_START" README.md; then + # Replace the old results with the new results + sed -i "/PERFORMANCE_RESULTS_START/,/PERFORMANCE_RESULTS_END/c\\$finalResults" README.md + else + # Append the results at the end of the README.md file + echo -e "\n$finalResults" >> README.md + fi +fi + +# Move the generated images to the assets folder +mv $reqSecHistogramFile assets/ +mv $latencyHistogramFile assets/ + +# Delete the result TXT files +for file in "${resultFiles[@]}"; do + rm "$file" +done \ No newline at end of file diff --git a/assets/latency_histogram1.png b/assets/latency_histogram1.png index 114166ef..62a4f151 100644 Binary files a/assets/latency_histogram1.png and b/assets/latency_histogram1.png differ diff --git a/assets/latency_histogram2.png b/assets/latency_histogram2.png index 016104ed..c1cb2e49 100644 Binary files a/assets/latency_histogram2.png and b/assets/latency_histogram2.png differ diff --git a/assets/latency_histogram3.png b/assets/latency_histogram3.png index 686c860f..83b040d9 100644 Binary files a/assets/latency_histogram3.png and b/assets/latency_histogram3.png differ diff --git a/assets/req_sec_histogram1.png b/assets/req_sec_histogram1.png index fbc5a27d..5f9c618d 100644 Binary files a/assets/req_sec_histogram1.png and b/assets/req_sec_histogram1.png differ diff --git a/assets/req_sec_histogram2.png b/assets/req_sec_histogram2.png index fa8e4ae7..11cde851 100644 Binary files a/assets/req_sec_histogram2.png and b/assets/req_sec_histogram2.png differ diff --git a/assets/req_sec_histogram3.png b/assets/req_sec_histogram3.png index 2b1502cc..eea730f9 100644 Binary files a/assets/req_sec_histogram3.png and b/assets/req_sec_histogram3.png differ diff --git a/graphql/apollo_server/package-lock.json b/graphql/apollo_server/package-lock.json index 211e157c..dff89e3f 100644 --- a/graphql/apollo_server/package-lock.json +++ b/graphql/apollo_server/package-lock.json @@ -50,9 +50,9 @@ } }, "node_modules/@apollo/server": { - "version": "4.10.4", - "resolved": "https://registry.npmjs.org/@apollo/server/-/server-4.10.4.tgz", - "integrity": "sha512-HS12CUa1wq8f5zKXOKJRwRdESFp4por9AINecpcsEUV9jsCP/NqPILgx0hCOOFJuKxmnaL7070xO6l5xmOq4Fw==", + "version": "4.10.5", + "resolved": "https://registry.npmjs.org/@apollo/server/-/server-4.10.5.tgz", + "integrity": "sha512-I+Oi1CnphXExSAAsZbclgRDN4k4CEMxyKRzYg3bir5N8OmydEhzXDlIGAOETP/TKblxR7HPXGvwn2cJKzbl46w==", "dependencies": { "@apollo/cache-control-types": "^1.0.3", "@apollo/server-gateway-interface": "^1.1.1", @@ -65,7 +65,6 @@ "@apollo/utils.usagereporting": "^2.1.0", "@apollo/utils.withrequired": "^2.0.0", "@graphql-tools/schema": "^9.0.0", - "@josephg/resolvable": "^1.0.0", "@types/express": "^4.17.13", "@types/express-serve-static-core": "^4.17.30", "@types/node-fetch": "^2.6.1", @@ -288,11 +287,6 @@ "graphql": "^0.8.0 || ^0.9.0 || ^0.10.0 || ^0.11.0 || ^0.12.0 || ^0.13.0 || ^14.0.0 || ^15.0.0 || ^16.0.0 || ^17.0.0" } }, - "node_modules/@josephg/resolvable": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/@josephg/resolvable/-/resolvable-1.0.1.tgz", - "integrity": "sha512-CtzORUwWTTOTqfVtHaKRJ0I1kNQd1bpn3sUh8I3nJDVY+5/M/Oe1DnEWzPQvqq/xPIIkzzzIP7mfCoAjFRvDhg==" - }, "node_modules/@pm2/agent": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/@pm2/agent/-/agent-2.0.3.tgz", @@ -2888,9 +2882,9 @@ } }, "@apollo/server": { - "version": "4.10.4", - "resolved": "https://registry.npmjs.org/@apollo/server/-/server-4.10.4.tgz", - "integrity": "sha512-HS12CUa1wq8f5zKXOKJRwRdESFp4por9AINecpcsEUV9jsCP/NqPILgx0hCOOFJuKxmnaL7070xO6l5xmOq4Fw==", + "version": "4.10.5", + "resolved": "https://registry.npmjs.org/@apollo/server/-/server-4.10.5.tgz", + "integrity": "sha512-I+Oi1CnphXExSAAsZbclgRDN4k4CEMxyKRzYg3bir5N8OmydEhzXDlIGAOETP/TKblxR7HPXGvwn2cJKzbl46w==", "requires": { "@apollo/cache-control-types": "^1.0.3", "@apollo/server-gateway-interface": "^1.1.1", @@ -2903,7 +2897,6 @@ "@apollo/utils.usagereporting": "^2.1.0", "@apollo/utils.withrequired": "^2.0.0", "@graphql-tools/schema": "^9.0.0", - "@josephg/resolvable": "^1.0.0", "@types/express": "^4.17.13", "@types/express-serve-static-core": "^4.17.30", "@types/node-fetch": "^2.6.1", @@ -3056,11 +3049,6 @@ "integrity": "sha512-mB9oAsNCm9aM3/SOv4YtBMqZbYj10R7dkq8byBqxGY/ncFwhf2oQzMV+LCRlWoDSEBJ3COiR1yeDvMtsoOsuFQ==", "requires": {} }, - "@josephg/resolvable": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/@josephg/resolvable/-/resolvable-1.0.1.tgz", - "integrity": "sha512-CtzORUwWTTOTqfVtHaKRJ0I1kNQd1bpn3sUh8I3nJDVY+5/M/Oe1DnEWzPQvqq/xPIIkzzzIP7mfCoAjFRvDhg==" - }, "@pm2/agent": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/@pm2/agent/-/agent-2.0.3.tgz", diff --git a/results.md b/results.md index 23b72c8a..ab23b548 100644 --- a/results.md +++ b/results.md @@ -3,31 +3,31 @@ | Query | Server | Requests/sec | Latency (ms) | Relative | |-------:|--------:|--------------:|--------------:|---------:| | 1 | `{ posts { id userId title user { id name email }}}` | -|| [Tailcall] | `28,320.00` | `3.52` | `175.34x` | -|| [async-graphql] | `1,784.61` | `57.51` | `11.05x` | -|| [Caliban] | `1,523.52` | `65.29` | `9.43x` | -|| [GraphQL JIT] | `1,311.50` | `75.93` | `8.12x` | -|| [Gqlgen] | `744.41` | `133.27` | `4.61x` | -|| [Netflix DGS] | `354.53` | `192.91` | `2.20x` | -|| [Apollo GraphQL] | `264.73` | `370.46` | `1.64x` | -|| [Hasura] | `161.51` | `496.54` | `1.00x` | +|| [Tailcall] | `29,496.60` | `3.38` | `196.74x` | +|| [async-graphql] | `1,805.45` | `55.70` | `12.04x` | +|| [Caliban] | `1,547.91` | `64.33` | `10.32x` | +|| [GraphQL JIT] | `1,335.26` | `74.60` | `8.91x` | +|| [Gqlgen] | `772.80` | `128.41` | `5.15x` | +|| [Netflix DGS] | `358.06` | `177.42` | `2.39x` | +|| [Apollo GraphQL] | `268.25` | `366.16` | `1.79x` | +|| [Hasura] | `149.93` | `549.07` | `1.00x` | | 2 | `{ posts { title }}` | -|| [Tailcall] | `58,223.00` | `1.71` | `68.82x` | -|| [Caliban] | `9,102.02` | `11.33` | `10.76x` | -|| [async-graphql] | `9,042.00` | `11.20` | `10.69x` | -|| [Gqlgen] | `2,136.74` | `48.34` | `2.53x` | -|| [Apollo GraphQL] | `1,742.58` | `57.30` | `2.06x` | -|| [Netflix DGS] | `1,584.25` | `69.49` | `1.87x` | -|| [GraphQL JIT] | `1,337.30` | `74.68` | `1.58x` | -|| [Hasura] | `845.98` | `117.95` | `1.00x` | +|| [Tailcall] | `58,939.60` | `1.69` | `67.77x` | +|| [Caliban] | `9,238.76` | `11.16` | `10.62x` | +|| [async-graphql] | `9,138.76` | `11.34` | `10.51x` | +|| [Gqlgen] | `2,136.19` | `48.48` | `2.46x` | +|| [Apollo GraphQL] | `1,775.87` | `56.26` | `2.04x` | +|| [Netflix DGS] | `1,593.80` | `69.82` | `1.83x` | +|| [GraphQL JIT] | `1,399.68` | `71.35` | `1.61x` | +|| [Hasura] | `869.69` | `114.81` | `1.00x` | | 3 | `{ greet }` | -|| [Caliban] | `66,940.20` | `1.09` | `26.59x` | -|| [Tailcall] | `59,316.40` | `1.70` | `23.56x` | -|| [async-graphql] | `47,630.90` | `2.14` | `18.92x` | -|| [Gqlgen] | `46,908.30` | `5.23` | `18.63x` | -|| [Netflix DGS] | `8,151.83` | `15.19` | `3.24x` | -|| [Apollo GraphQL] | `7,931.63` | `12.75` | `3.15x` | -|| [GraphQL JIT] | `5,147.67` | `19.40` | `2.04x` | -|| [Hasura] | `2,517.36` | `39.64` | `1.00x` | +|| [Caliban] | `67,637.00` | `1.07` | `25.83x` | +|| [Tailcall] | `59,557.80` | `1.69` | `22.75x` | +|| [Gqlgen] | `47,525.20` | `5.21` | `18.15x` | +|| [async-graphql] | `47,150.60` | `2.21` | `18.01x` | +|| [Netflix DGS] | `8,261.44` | `15.02` | `3.16x` | +|| [Apollo GraphQL] | `8,186.12` | `12.54` | `3.13x` | +|| [GraphQL JIT] | `5,273.45` | `18.93` | `2.01x` | +|| [Hasura] | `2,618.18` | `38.13` | `1.00x` | diff --git a/run_analyze_script.sh b/run_analyze_script.sh index 45eeec16..84f7206a 100755 --- a/run_analyze_script.sh +++ b/run_analyze_script.sh @@ -1,13 +1,36 @@ #!/bin/bash -rm results.md +# Update and install gnuplot +sudo apt-get update && sudo apt-get install -y gnuplot +# Remove existing results file +rm -f results.md + +services=("apollo" "caliban" "netflixdgs" "gqlgen" "tailcall" "async_graphql" "hasura" "graphql_jit") + +# Loop through each benchmark (1, 2, 3) for bench in 1 2 3; do - if ls bench${bench}*.txt &> /dev/null; then - echo "Processing files for bench${bench}:" - bash analyze.sh bench${bench}*.txt - echo "Files processed: $(ls bench${bench}*.txt)" - else - echo "No matching files found for bench${bench}*.txt" - fi + echo "Processing files for bench${bench}:" + + # Construct the command for each benchmark + cmd="bash analyze.sh" + + # Loop through each service + for service in "${services[@]}"; do + # Convert service name to match file naming convention + case $service in + "apollo") file_service="apollo_server" ;; + "netflixdgs") file_service="netflix_dgs" ;; + *) file_service=$service ;; + esac + + # Add files for current service to the command + for run in 1 2 3; do + cmd+=" bench${bench}_result${run}_graphql_${file_service}_run.sh.txt" + done + done + + # Execute the command + echo "Executing: $cmd" + eval $cmd done \ No newline at end of file diff --git a/run_benchmarks.sh b/run_benchmarks.sh index d1762d04..9c1b1aa8 100755 --- a/run_benchmarks.sh +++ b/run_benchmarks.sh @@ -12,6 +12,7 @@ function killServerOnPort() { echo "No process found running on port $port" fi } + bench1Results=() bench2Results=() bench3Results=() @@ -72,17 +73,29 @@ function runBenchmark() { rm "results.md" -for service in "apollo_server" "caliban" "netflix_dgs" "gqlgen" "tailcall" "async_graphql" "hasura" "graphql_jit"; do - runBenchmark "graphql/${service}/run.sh" - if [ "$service" == "apollo_server" ]; then +# Main script +if [ $# -eq 0 ]; then + echo "Usage: $0 " + echo "Available services: apollo_server, caliban, netflix_dgs, gqlgen, tailcall, async_graphql, hasura, graphql_jit" + exit 1 +fi + +service="$1" +valid_services=("apollo_server" "caliban" "netflix_dgs" "gqlgen" "tailcall" "async_graphql" "hasura" "graphql_jit") + +if [[ ! " ${valid_services[@]} " =~ " ${service} " ]]; then + echo "Invalid service name. Available services: ${valid_services[*]}" + exit 1 +fi + + + +runBenchmark "graphql/${service}/run.sh" + +if [ "$service" == "apollo_server" ]; then cd graphql/apollo_server/ npm stop cd ../../ - elif [ "$service" == "hasura" ]; then +elif [ "$service" == "hasura" ]; then bash "graphql/hasura/kill.sh" - fi -done - -node analyze.js "${bench1Results[@]}" -node analyze.js "${bench2Results[@]}" -node analyze.js "${bench3Results[@]}" +fi \ No newline at end of file