From 04f4fcfbe962bde7706492bea1249385d8c62bd9 Mon Sep 17 00:00:00 2001 From: daveads Date: Wed, 24 Jul 2024 14:34:28 +0100 Subject: [PATCH 1/3] Rewrote run_benchmarks.sh in javascript - rewrite makes use of javascript asynchronous programming techniques using async/await... // benchmark 1, 2, 3 run without waiting for each other... for each services... --- run_benchmarks.js | 120 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100644 run_benchmarks.js diff --git a/run_benchmarks.js b/run_benchmarks.js new file mode 100644 index 00000000..894ed1a2 --- /dev/null +++ b/run_benchmarks.js @@ -0,0 +1,120 @@ +const { exec, execSync } = require('child_process'); +const fs = require('fs'); +const path = require('path'); +const util = require('util'); + +const execAsync = util.promisify(exec); + +// Start services and run benchmarks +function killServerOnPort(port) { + try { + const pid = execSync(`lsof -t -i:${port}`).toString().trim(); + if (pid) { + execSync(`kill ${pid}`); + console.log(`Killed process running on port ${port}`); + } else { + console.log(`No process found running on port ${port}`); + } + } catch (error) { + console.error(`Error killing server on port ${port}:`, error.message); + } +} + +const bench1Results = []; +const bench2Results = []; +const bench3Results = []; + +killServerOnPort(3000); +execSync('sh nginx/run.sh'); + +async function runBenchmarkAsync(serviceScript, bench) { + let graphqlEndpoint = 'http://localhost:8000/graphql'; + if (serviceScript.includes('hasura')) { + graphqlEndpoint = 'http://127.0.0.1:8080/v1/graphql'; + } + + const benchmarkScript = 'wrk/bench.sh'; + const sanitizedServiceScriptName = serviceScript.replace(/\//g, '_'); + const resultFiles = [ + `result1_${sanitizedServiceScriptName}.txt`, + `result2_${sanitizedServiceScriptName}.txt`, + `result3_${sanitizedServiceScriptName}.txt` + ]; + + await execAsync(`bash test_query${bench}.sh ${graphqlEndpoint}`); + + // Warmup run + for (let i = 0; i < 3; i++) { + await execAsync(`bash ${benchmarkScript} ${graphqlEndpoint} ${bench} > /dev/null`); + await new Promise(resolve => setTimeout(resolve, 1000)); + } + + // 3 benchmark runs + for (const resultFile of resultFiles) { + console.log(`Running benchmark ${bench} for ${serviceScript}`); + const outputFile = `bench${bench}_${resultFile}`; + await execAsync(`bash ${benchmarkScript} ${graphqlEndpoint} ${bench} > ${outputFile}`); + + if (bench === 1) { + bench1Results.push(outputFile); + } else if (bench === 2) { + bench2Results.push(outputFile); + } else if (bench === 3) { + bench3Results.push(outputFile); + } + } +} + +async function runBenchmark(serviceScript) { + killServerOnPort(8000); + execSync('sleep 5'); + + if (serviceScript.includes('hasura')) { + execSync(`bash ${serviceScript}`, { stdio: 'inherit' }); + } else { + execSync(`bash ${serviceScript} &`, { stdio: 'inherit' }); + } + + execSync('sleep 15'); + + const benchmarks = [1, 2, 3]; + const benchmarkPromises = benchmarks.map(bench => runBenchmarkAsync(serviceScript, bench)); + + await Promise.all(benchmarkPromises); +} + +// Main script +if (process.argv.length < 3) { + console.log('Usage: node script.js '); + console.log('Available services: apollo_server, caliban, netflix_dgs, gqlgen, tailcall, async_graphql, hasura, graphql_jit'); + process.exit(1); +} + +const service = process.argv[2]; +const validServices = ['apollo_server', 'caliban', 'netflix_dgs', 'gqlgen', 'tailcall', 'async_graphql', 'hasura', 'graphql_jit']; + +if (!validServices.includes(service)) { + console.log(`Invalid service name. Available services: ${validServices.join(', ')}`); + process.exit(1); +} + +if (fs.existsSync('results.md')) { + fs.unlinkSync('results.md'); +} + +async function main() { + await runBenchmark(`graphql/${service}/run.sh`); + + if (service === 'apollo_server') { + process.chdir('graphql/apollo_server'); + execSync('npm stop'); + process.chdir('../../'); + } else if (service === 'hasura') { + execSync('bash graphql/hasura/kill.sh'); + } +} + +main().catch(error => { + console.error("An error occurred:", error); + process.exit(1); +}); \ No newline at end of file From a441d1678e7fc68bf2c2869f7904b27438da7e12 Mon Sep 17 00:00:00 2001 From: daveads Date: Wed, 24 Jul 2024 14:38:43 +0100 Subject: [PATCH 2/3] updated workflow --- .github/workflows/bench.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 84b7b283..d64145ab 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -42,7 +42,7 @@ jobs: push: never runCmd: | bash ./graphql/${{ matrix.service }}/setup.sh - bash run_benchmarks.sh ${{ matrix.service }} + node run_benchmarks.js ${{ matrix.service }} - name: List benchmark files run: | From 2914bbb714df3d72f313ea67a2e088d722eca4ea Mon Sep 17 00:00:00 2001 From: daveads Date: Sun, 28 Jul 2024 21:03:06 +0100 Subject: [PATCH 3/3] typescript --- .github/workflows/bench.yml | 8 ++++- run_benchmarks.js => run_benchmarks.ts | 42 +++++++++++--------------- 2 files changed, 25 insertions(+), 25 deletions(-) rename run_benchmarks.js => run_benchmarks.ts (73%) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index d64145ab..28f75211 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -41,7 +41,13 @@ jobs: imageName: graphql-benchmarks push: never runCmd: | - bash ./graphql/${{ matrix.service }}/setup.sh + + bash ./graphql/${{ matrix.service }}/setup.sha + + npm install + sudo npm install -g typescript + + tsc run_benchmarks.tx node run_benchmarks.js ${{ matrix.service }} - name: List benchmark files diff --git a/run_benchmarks.js b/run_benchmarks.ts similarity index 73% rename from run_benchmarks.js rename to run_benchmarks.ts index 894ed1a2..a07fe563 100644 --- a/run_benchmarks.js +++ b/run_benchmarks.ts @@ -1,14 +1,14 @@ -const { exec, execSync } = require('child_process'); -const fs = require('fs'); -const path = require('path'); -const util = require('util'); +import { exec, execSync } from 'child_process'; +import * as fs from 'fs'; +import * as path from 'path'; +import * as util from 'util'; const execAsync = util.promisify(exec); // Start services and run benchmarks -function killServerOnPort(port) { +function killServerOnPort(port: number): void { try { - const pid = execSync(`lsof -t -i:${port}`).toString().trim(); + const pid: string = execSync(`lsof -t -i:${port}`).toString().trim(); if (pid) { execSync(`kill ${pid}`); console.log(`Killed process running on port ${port}`); @@ -16,23 +16,22 @@ function killServerOnPort(port) { console.log(`No process found running on port ${port}`); } } catch (error) { - console.error(`Error killing server on port ${port}:`, error.message); + console.error(`Error killing server on port ${port}:`, (error as Error).message); } } -const bench1Results = []; -const bench2Results = []; -const bench3Results = []; +const bench1Results: string[] = []; +const bench2Results: string[] = []; +const bench3Results: string[] = []; killServerOnPort(3000); execSync('sh nginx/run.sh'); -async function runBenchmarkAsync(serviceScript, bench) { +async function runBenchmarkAsync(serviceScript: string, bench: number): Promise { let graphqlEndpoint = 'http://localhost:8000/graphql'; if (serviceScript.includes('hasura')) { graphqlEndpoint = 'http://127.0.0.1:8080/v1/graphql'; } - const benchmarkScript = 'wrk/bench.sh'; const sanitizedServiceScriptName = serviceScript.replace(/\//g, '_'); const resultFiles = [ @@ -54,7 +53,6 @@ async function runBenchmarkAsync(serviceScript, bench) { console.log(`Running benchmark ${bench} for ${serviceScript}`); const outputFile = `bench${bench}_${resultFile}`; await execAsync(`bash ${benchmarkScript} ${graphqlEndpoint} ${bench} > ${outputFile}`); - if (bench === 1) { bench1Results.push(outputFile); } else if (bench === 2) { @@ -65,21 +63,18 @@ async function runBenchmarkAsync(serviceScript, bench) { } } -async function runBenchmark(serviceScript) { +async function runBenchmark(serviceScript: string): Promise { killServerOnPort(8000); execSync('sleep 5'); - if (serviceScript.includes('hasura')) { execSync(`bash ${serviceScript}`, { stdio: 'inherit' }); } else { execSync(`bash ${serviceScript} &`, { stdio: 'inherit' }); } - execSync('sleep 15'); const benchmarks = [1, 2, 3]; - const benchmarkPromises = benchmarks.map(bench => runBenchmarkAsync(serviceScript, bench)); - + const benchmarkPromises: Promise[] = benchmarks.map(bench => runBenchmarkAsync(serviceScript, bench)); await Promise.all(benchmarkPromises); } @@ -90,8 +85,8 @@ if (process.argv.length < 3) { process.exit(1); } -const service = process.argv[2]; -const validServices = ['apollo_server', 'caliban', 'netflix_dgs', 'gqlgen', 'tailcall', 'async_graphql', 'hasura', 'graphql_jit']; +const service: string = process.argv[2]; +const validServices: string[] = ['apollo_server', 'caliban', 'netflix_dgs', 'gqlgen', 'tailcall', 'async_graphql', 'hasura', 'graphql_jit']; if (!validServices.includes(service)) { console.log(`Invalid service name. Available services: ${validServices.join(', ')}`); @@ -102,9 +97,8 @@ if (fs.existsSync('results.md')) { fs.unlinkSync('results.md'); } -async function main() { +async function main(): Promise { await runBenchmark(`graphql/${service}/run.sh`); - if (service === 'apollo_server') { process.chdir('graphql/apollo_server'); execSync('npm stop'); @@ -114,7 +108,7 @@ async function main() { } } -main().catch(error => { +main().catch((error: Error) => { console.error("An error occurred:", error); process.exit(1); -}); \ No newline at end of file +});