From a9d7285009666721a34a2496b01b22e18795c293 Mon Sep 17 00:00:00 2001 From: asr2003 <162500856+asr2003@users.noreply.github.com> Date: Tue, 9 Jul 2024 21:49:41 +0530 Subject: [PATCH 1/6] reimplement analyze.sh to analyze.js --- wrk/analyze.js | 127 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 127 insertions(+) create mode 100644 wrk/analyze.js diff --git a/wrk/analyze.js b/wrk/analyze.js new file mode 100644 index 00000000..f621f0b9 --- /dev/null +++ b/wrk/analyze.js @@ -0,0 +1,127 @@ +#!/usr/bin/env node + +const fs = require("fs"); +const { execSync } = require("child_process"); + +const average = (arr) => arr.reduce((a, b) => a + b, 0) / arr.length; + +const servers = ["apollo", "netflixdgs", "gqlgen", "tailcall"]; + +const resultFiles = process.argv + .slice(2) + .filter((arg) => arg.startsWith("result") && fs.existsSync(arg)); + +const extractMetrics = (files) => { + return files.map((file) => { + const { summary } = JSON.parse(fs.readFileSync(file, "utf8")); + return { + requestsPerSec: summary.requestsPerSec, + latency: summary.average * 1000, + }; + }); +}; + +const calculateAverages = (metrics) => { + return servers.reduce((acc, server, index) => { + const serverMetrics = metrics.slice(index * 3, (index + 1) * 3); + acc[server] = { + avgReqSec: average(serverMetrics.map((m) => m.requestsPerSec)), + avgLatency: average(serverMetrics.map((m) => m.latency)), + }; + return acc; + }, {}); +}; + +const writeDataFile = (filename, data) => { + fs.writeFileSync(filename, "Server Value\n"); + Object.entries(data).forEach(([server, value]) => { + fs.appendFileSync(filename, `${server} ${value}\n`); + }); +}; + +const generateGnuplotScript = (reqSecData, latencyData) => ` +set term pngcairo size 1280,720 enhanced font "Courier,12" +set style data histograms +set style histogram cluster gap 1 +set style fill solid border -1 +set xtics rotate by -45 +set boxwidth 0.9 +set key outside right top +set output "req_sec_histogram.png" +set title "Requests/Sec" +stats "${reqSecData}" using 2 nooutput +set yrange [0:*] +plot "${reqSecData}" using 2:xtic(1) title "Req/Sec" +set output "latency_histogram.png" +set title "Latency (in ms)" +stats "${latencyData}" using 2 nooutput +set yrange [0:*] +plot "${latencyData}" using 2:xtic(1) title "Latency" +`; + +const generateResultsTable = (averages) => { + let table = + "\n| Server | Requests/sec | Latency (ms) |\n|--------|--------------|--------------|"; + Object.entries(averages).forEach(([server, { avgReqSec, avgLatency }]) => { + table += `\n| ${server} | ${avgReqSec} | ${avgLatency} |`; + }); + return table + "\n"; +}; + +const updateReadme = (resultsTable) => { + const readmePath = "README.md"; + let content = fs.readFileSync(readmePath, "utf8"); + const regex = + /[\s\S]*/; + content = content.includes("PERFORMANCE_RESULTS_START") + ? content.replace(regex, resultsTable) + : content + "\n\n" + resultsTable; + fs.writeFileSync(readmePath, content); +}; + +const main = () => { + const metrics = extractMetrics(resultFiles); + const averages = calculateAverages(metrics); + + resultFiles.forEach(fs.unlinkSync); + + const reqSecData = "/tmp/reqSec.dat"; + const latencyData = "/tmp/latency.dat"; + + writeDataFile( + reqSecData, + Object.fromEntries( + Object.entries(averages).map(([k, v]) => [k, v.avgReqSec]) + ) + ); + writeDataFile( + latencyData, + Object.fromEntries( + Object.entries(averages).map(([k, v]) => [k, v.avgLatency]) + ) + ); + + const gnuplotScript = generateGnuplotScript(reqSecData, latencyData); + fs.writeFileSync("/tmp/gnuplot_script", gnuplotScript); + execSync("gnuplot /tmp/gnuplot_script"); + + if (!fs.existsSync("assets")) fs.mkdirSync("assets"); + ["req_sec_histogram.png", "latency_histogram.png"].forEach((file) => + fs.renameSync(file, `assets/${file}`) + ); + + const resultsTable = generateResultsTable(averages); + updateReadme(resultsTable); + + console.log(resultsTable.replace(/\n?/g, "")); + + execSync( + "git add README.md assets/req_sec_histogram.png assets/latency_histogram.png" + ); + execSync('git commit -m "Updated performance results in README.md"'); + execSync("git push"); + + resultFiles.forEach(fs.unlinkSync); +}; + +main(); From b0000fe0fa6141cbc01bf71d8f689275ac6d178c Mon Sep 17 00:00:00 2001 From: asr2003 <162500856+asr2003@users.noreply.github.com> Date: Tue, 9 Jul 2024 21:57:27 +0530 Subject: [PATCH 2/6] move to root --- wrk/analyze.js | 127 ------------------------------------------------- 1 file changed, 127 deletions(-) delete mode 100644 wrk/analyze.js diff --git a/wrk/analyze.js b/wrk/analyze.js deleted file mode 100644 index f621f0b9..00000000 --- a/wrk/analyze.js +++ /dev/null @@ -1,127 +0,0 @@ -#!/usr/bin/env node - -const fs = require("fs"); -const { execSync } = require("child_process"); - -const average = (arr) => arr.reduce((a, b) => a + b, 0) / arr.length; - -const servers = ["apollo", "netflixdgs", "gqlgen", "tailcall"]; - -const resultFiles = process.argv - .slice(2) - .filter((arg) => arg.startsWith("result") && fs.existsSync(arg)); - -const extractMetrics = (files) => { - return files.map((file) => { - const { summary } = JSON.parse(fs.readFileSync(file, "utf8")); - return { - requestsPerSec: summary.requestsPerSec, - latency: summary.average * 1000, - }; - }); -}; - -const calculateAverages = (metrics) => { - return servers.reduce((acc, server, index) => { - const serverMetrics = metrics.slice(index * 3, (index + 1) * 3); - acc[server] = { - avgReqSec: average(serverMetrics.map((m) => m.requestsPerSec)), - avgLatency: average(serverMetrics.map((m) => m.latency)), - }; - return acc; - }, {}); -}; - -const writeDataFile = (filename, data) => { - fs.writeFileSync(filename, "Server Value\n"); - Object.entries(data).forEach(([server, value]) => { - fs.appendFileSync(filename, `${server} ${value}\n`); - }); -}; - -const generateGnuplotScript = (reqSecData, latencyData) => ` -set term pngcairo size 1280,720 enhanced font "Courier,12" -set style data histograms -set style histogram cluster gap 1 -set style fill solid border -1 -set xtics rotate by -45 -set boxwidth 0.9 -set key outside right top -set output "req_sec_histogram.png" -set title "Requests/Sec" -stats "${reqSecData}" using 2 nooutput -set yrange [0:*] -plot "${reqSecData}" using 2:xtic(1) title "Req/Sec" -set output "latency_histogram.png" -set title "Latency (in ms)" -stats "${latencyData}" using 2 nooutput -set yrange [0:*] -plot "${latencyData}" using 2:xtic(1) title "Latency" -`; - -const generateResultsTable = (averages) => { - let table = - "\n| Server | Requests/sec | Latency (ms) |\n|--------|--------------|--------------|"; - Object.entries(averages).forEach(([server, { avgReqSec, avgLatency }]) => { - table += `\n| ${server} | ${avgReqSec} | ${avgLatency} |`; - }); - return table + "\n"; -}; - -const updateReadme = (resultsTable) => { - const readmePath = "README.md"; - let content = fs.readFileSync(readmePath, "utf8"); - const regex = - /[\s\S]*/; - content = content.includes("PERFORMANCE_RESULTS_START") - ? content.replace(regex, resultsTable) - : content + "\n\n" + resultsTable; - fs.writeFileSync(readmePath, content); -}; - -const main = () => { - const metrics = extractMetrics(resultFiles); - const averages = calculateAverages(metrics); - - resultFiles.forEach(fs.unlinkSync); - - const reqSecData = "/tmp/reqSec.dat"; - const latencyData = "/tmp/latency.dat"; - - writeDataFile( - reqSecData, - Object.fromEntries( - Object.entries(averages).map(([k, v]) => [k, v.avgReqSec]) - ) - ); - writeDataFile( - latencyData, - Object.fromEntries( - Object.entries(averages).map(([k, v]) => [k, v.avgLatency]) - ) - ); - - const gnuplotScript = generateGnuplotScript(reqSecData, latencyData); - fs.writeFileSync("/tmp/gnuplot_script", gnuplotScript); - execSync("gnuplot /tmp/gnuplot_script"); - - if (!fs.existsSync("assets")) fs.mkdirSync("assets"); - ["req_sec_histogram.png", "latency_histogram.png"].forEach((file) => - fs.renameSync(file, `assets/${file}`) - ); - - const resultsTable = generateResultsTable(averages); - updateReadme(resultsTable); - - console.log(resultsTable.replace(/\n?/g, "")); - - execSync( - "git add README.md assets/req_sec_histogram.png assets/latency_histogram.png" - ); - execSync('git commit -m "Updated performance results in README.md"'); - execSync("git push"); - - resultFiles.forEach(fs.unlinkSync); -}; - -main(); From 2d6cc0a6cc899917f56bb30baa7a67e5d35541aa Mon Sep 17 00:00:00 2001 From: asr2003 <162500856+asr2003@users.noreply.github.com> Date: Tue, 9 Jul 2024 21:59:05 +0530 Subject: [PATCH 3/6] implement analyze.js --- analyze.js | 137 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 137 insertions(+) create mode 100644 analyze.js diff --git a/analyze.js b/analyze.js new file mode 100644 index 00000000..6909a2ed --- /dev/null +++ b/analyze.js @@ -0,0 +1,137 @@ +#!/usr/bin/env node + +const fs = require("fs"); +const { execSync } = require("child_process"); + +const average = (arr) => arr.reduce((a, b) => a + b, 0) / arr.length; + +const servers = ["apollo", "caliban", "netflixdgs", "gqlgen", "tailcall", "async_graphql", "hasura", "graphql_jit"]; +const formattedServerNames = { + "tailcall": "Tailcall", + "gqlgen": "Gqlgen", + "apollo": "Apollo GraphQL", + "netflixdgs": "Netflix DGS", + "caliban": "Caliban", + "async_graphql": "async-graphql", + "hasura": "Hasura", + "graphql_jit": "GraphQL JIT" +}; + +const resultFiles = process.argv + .slice(2) + .filter((arg) => arg.startsWith("result") && fs.existsSync(arg)); + +const extractMetrics = (files) => { + return files.map((file) => { + const { summary } = JSON.parse(fs.readFileSync(file, "utf8")); + return { + requestsPerSec: summary.requestsPerSec, + latency: summary.average * 1000, + }; + }); +}; + +const calculateAverages = (metrics) => { + return servers.reduce((acc, server, index) => { + const serverMetrics = metrics.slice(index * 3, (index + 1) * 3); + acc[server] = { + avgReqSec: average(serverMetrics.map((m) => m.requestsPerSec)), + avgLatency: average(serverMetrics.map((m) => m.latency)), + }; + return acc; + }, {}); +}; + +const writeDataFile = (filename, data) => { + fs.writeFileSync(filename, "Server Value\n"); + Object.entries(data).forEach(([server, value]) => { + fs.appendFileSync(filename, `${server} ${value}\n`); + }); +}; + +const generateGnuplotScript = (reqSecData, latencyData) => ` +set term pngcairo size 1280,720 enhanced font "Courier,12" +set style data histograms +set style histogram cluster gap 1 +set style fill solid border -1 +set xtics rotate by -45 +set boxwidth 0.9 +set key outside right top +set output "req_sec_histogram.png" +set title "Requests/Sec" +stats "${reqSecData}" using 2 nooutput +set yrange [0:*] +plot "${reqSecData}" using 2:xtic(1) title "Req/Sec" +set output "latency_histogram.png" +set title "Latency (in ms)" +stats "${latencyData}" using 2 nooutput +set yrange [0:*] +plot "${latencyData}" using 2:xtic(1) title "Latency" +`; + +const generateResultsTable = (averages) => { + let table = + "\n| Server | Requests/sec | Latency (ms) |\n|--------|--------------|--------------|"; + Object.entries(averages).forEach(([server, { avgReqSec, avgLatency }]) => { + table += `\n| ${server} | ${avgReqSec} | ${avgLatency} |`; + }); + return table + "\n"; +}; + +const updateReadme = (resultsTable) => { + const readmePath = "README.md"; + let content = fs.readFileSync(readmePath, "utf8"); + const regex = + /[\s\S]*/; + content = content.includes("PERFORMANCE_RESULTS_START") + ? content.replace(regex, resultsTable) + : content + "\n\n" + resultsTable; + fs.writeFileSync(readmePath, content); +}; + +const main = () => { + const metrics = extractMetrics(resultFiles); + const averages = calculateAverages(metrics); + + resultFiles.forEach(fs.unlinkSync); + + const reqSecData = "/tmp/reqSec.dat"; + const latencyData = "/tmp/latency.dat"; + + writeDataFile( + reqSecData, + Object.fromEntries( + Object.entries(averages).map(([k, v]) => [k, v.avgReqSec]) + ) + ); + writeDataFile( + latencyData, + Object.fromEntries( + Object.entries(averages).map(([k, v]) => [k, v.avgLatency]) + ) + ); + + const gnuplotScript = generateGnuplotScript(reqSecData, latencyData); + fs.writeFileSync("/tmp/gnuplot_script", gnuplotScript); + execSync("gnuplot /tmp/gnuplot_script"); + + if (!fs.existsSync("assets")) fs.mkdirSync("assets"); + ["req_sec_histogram.png", "latency_histogram.png"].forEach((file) => + fs.renameSync(file, `assets/${file}`) + ); + + const resultsTable = generateResultsTable(averages); + updateReadme(resultsTable); + + console.log(resultsTable.replace(/\n?/g, "")); + + execSync( + "git add README.md assets/req_sec_histogram.png assets/latency_histogram.png" + ); + execSync('git commit -m "Updated performance results in README.md"'); + execSync("git push"); + + resultFiles.forEach(fs.unlinkSync); +}; + +main(); From 64deeb802241b8a8754e0d7f844c175313b94ebe Mon Sep 17 00:00:00 2001 From: asr2003 <162500856+asr2003@users.noreply.github.com> Date: Tue, 9 Jul 2024 22:41:59 +0530 Subject: [PATCH 4/6] preserve comments for better functionality --- analyze.js | 234 ++++++++++++++++++++++++++++++++++------------------- 1 file changed, 153 insertions(+), 81 deletions(-) diff --git a/analyze.js b/analyze.js index 6909a2ed..fc69a597 100644 --- a/analyze.js +++ b/analyze.js @@ -1,55 +1,79 @@ #!/usr/bin/env node -const fs = require("fs"); -const { execSync } = require("child_process"); +const fs = require('fs'); +const { execSync } = require('child_process'); + +const extractMetric = (file, metric) => { + const data = fs.readFileSync(file, 'utf8'); + const line = data.split('\n').find((line) => line.includes(metric)); + if (!line) return null; + return parseFloat(line.split(' ')[1].replace('ms', '')); +}; const average = (arr) => arr.reduce((a, b) => a + b, 0) / arr.length; -const servers = ["apollo", "caliban", "netflixdgs", "gqlgen", "tailcall", "async_graphql", "hasura", "graphql_jit"]; +const servers = [ + 'apollo', + 'caliban', + 'netflixdgs', + 'gqlgen', + 'tailcall', + 'async_graphql', + 'hasura', + 'graphql_jit', +]; + +// Associative array for formatted server names const formattedServerNames = { - "tailcall": "Tailcall", - "gqlgen": "Gqlgen", - "apollo": "Apollo GraphQL", - "netflixdgs": "Netflix DGS", - "caliban": "Caliban", - "async_graphql": "async-graphql", - "hasura": "Hasura", - "graphql_jit": "GraphQL JIT" + tailcall: 'Tailcall', + gqlgen: 'Gqlgen', + apollo: 'Apollo GraphQL', + netflixdgs: 'Netflix DGZ', + caliban: 'Caliban', + async_graphql: 'async-graphql', + hasura: 'Hasura', + graphql_jit: 'GraphQL JIT', }; -const resultFiles = process.argv - .slice(2) - .filter((arg) => arg.startsWith("result") && fs.existsSync(arg)); - -const extractMetrics = (files) => { - return files.map((file) => { - const { summary } = JSON.parse(fs.readFileSync(file, "utf8")); - return { - requestsPerSec: summary.requestsPerSec, - latency: summary.average * 1000, - }; +const resultFiles = process.argv.slice(2).filter((arg) => fs.existsSync(arg)); + +// Extract metrics and calculate averages +const calculateAverages = () => { + const avgReqSecs = {}; + const avgLatencies = {}; + + servers.forEach((server, idx) => { + const startIdx = idx * 3; + const reqSecVals = []; + const latencyVals = []; + + for (let j = 0; j < 3; j++) { + const fileIdx = startIdx + j; + reqSecVals.push(extractMetric(resultFiles[fileIdx], 'Requests/sec')); + latencyVals.push(extractMetric(resultFiles[fileIdx], 'Latency')); + } + + avgReqSecs[server] = average(reqSecVals); + avgLatencies[server] = average(latencyVals); }); -}; -const calculateAverages = (metrics) => { - return servers.reduce((acc, server, index) => { - const serverMetrics = metrics.slice(index * 3, (index + 1) * 3); - acc[server] = { - avgReqSec: average(serverMetrics.map((m) => m.requestsPerSec)), - avgLatency: average(serverMetrics.map((m) => m.latency)), - }; - return acc; - }, {}); + return { avgReqSecs, avgLatencies }; }; const writeDataFile = (filename, data) => { - fs.writeFileSync(filename, "Server Value\n"); + fs.writeFileSync(filename, 'Server Value\n'); Object.entries(data).forEach(([server, value]) => { fs.appendFileSync(filename, `${server} ${value}\n`); }); }; -const generateGnuplotScript = (reqSecData, latencyData) => ` +// Plotting using gnuplot +const generateGnuplotScript = ( + reqSecData, + latencyData, + reqSecHistogramFile, + latencyHistogramFile, +) => ` set term pngcairo size 1280,720 enhanced font "Courier,12" set style data histograms set style histogram cluster gap 1 @@ -57,81 +81,129 @@ set style fill solid border -1 set xtics rotate by -45 set boxwidth 0.9 set key outside right top -set output "req_sec_histogram.png" +set output "${reqSecHistogramFile}" set title "Requests/Sec" stats "${reqSecData}" using 2 nooutput -set yrange [0:*] +set yrange [0:STATS_max*1.2] plot "${reqSecData}" using 2:xtic(1) title "Req/Sec" -set output "latency_histogram.png" + +set output "${latencyHistogramFile}" set title "Latency (in ms)" stats "${latencyData}" using 2 nooutput -set yrange [0:*] +set yrange [0:STATS_max*1.2] plot "${latencyData}" using 2:xtic(1) title "Latency" `; -const generateResultsTable = (averages) => { - let table = - "\n| Server | Requests/sec | Latency (ms) |\n|--------|--------------|--------------|"; - Object.entries(averages).forEach(([server, { avgReqSec, avgLatency }]) => { - table += `\n| ${server} | ${avgReqSec} | ${avgLatency} |`; +// Generate the results table for README +const generateResultsTable = (avgReqSecs, avgLatencies, whichBench) => { + const sortedServers = Object.keys(avgReqSecs).sort( + (a, b) => avgReqSecs[b] - avgReqSecs[a], + ); + const lastServer = sortedServers[sortedServers.length - 1]; + const lastServerReqSecs = avgReqSecs[lastServer]; + + console.log(`Sorted servers: ${sortedServers.join(', ')}`); + + // Start building the resultsTable + let resultsTable; + if (whichBench === 1) { + resultsTable = + '\n\n| Query | Server | Requests/sec | Latency (ms) | Relative |\n|-------:|--------:|--------------:|--------------:|---------:|\n| ' + + whichBench + + ' | `{ posts { id userId title user { id name email }}}` |'; + } else if (whichBench === 2) { + resultsTable = '| ' + whichBench + ' | `{ posts { title }}` |'; + } else if (whichBench === 3) { + resultsTable = '| ' + whichBench + ' | `{ greet }` |'; + } + + // Build the resultsTable with sorted servers and formatted numbers + sortedServers.forEach((server) => { + const formattedReqSecs = avgReqSecs[server].toLocaleString(undefined, { + minimumFractionDigits: 2, + maximumFractionDigits: 2, + }); + const formattedLatencies = avgLatencies[server].toLocaleString(undefined, { + minimumFractionDigits: 2, + maximumFractionDigits: 2, + }); + const relativePerformance = ( + avgReqSecs[server] / lastServerReqSecs + ).toFixed(2); + + resultsTable += `\n| [${formattedServerNames[server]}] | \`${formattedReqSecs}\` | \`${formattedLatencies}\` | \`${relativePerformance}x\` |`; }); - return table + "\n"; + + if (whichBench === 3) { + resultsTable += '\n\n'; + } + + return resultsTable; }; +// Print the results table in a new file const updateReadme = (resultsTable) => { - const readmePath = "README.md"; - let content = fs.readFileSync(readmePath, "utf8"); + const readmePath = 'README.md'; + let content = fs.readFileSync(readmePath, 'utf8'); const regex = /[\s\S]*/; - content = content.includes("PERFORMANCE_RESULTS_START") + content = content.includes('PERFORMANCE_RESULTS_START') ? content.replace(regex, resultsTable) - : content + "\n\n" + resultsTable; + : content + '\n\n' + resultsTable; fs.writeFileSync(readmePath, content); }; const main = () => { - const metrics = extractMetrics(resultFiles); - const averages = calculateAverages(metrics); + const { avgReqSecs, avgLatencies } = calculateAverages(); + + // Generating data files for gnuplot + const reqSecData = '/tmp/reqSec.dat'; + const latencyData = '/tmp/latency.dat'; + + writeDataFile(reqSecData, avgReqSecs); + writeDataFile(latencyData, avgLatencies); - resultFiles.forEach(fs.unlinkSync); + let whichBench = 1; + if (resultFiles[0].startsWith('bench2')) { + whichBench = 2; + } else if (resultFiles[0].startsWith('bench3')) { + whichBench = 3; + } - const reqSecData = "/tmp/reqSec.dat"; - const latencyData = "/tmp/latency.dat"; + // Move the generated images to the assets folder + const reqSecHistogramFile = `req_sec_histogram${whichBench}.png`; + const latencyHistogramFile = `latency_histogram${whichBench}.png`; - writeDataFile( + const gnuplotScript = generateGnuplotScript( reqSecData, - Object.fromEntries( - Object.entries(averages).map(([k, v]) => [k, v.avgReqSec]) - ) - ); - writeDataFile( latencyData, - Object.fromEntries( - Object.entries(averages).map(([k, v]) => [k, v.avgLatency]) - ) + reqSecHistogramFile, + latencyHistogramFile, ); - - const gnuplotScript = generateGnuplotScript(reqSecData, latencyData); - fs.writeFileSync("/tmp/gnuplot_script", gnuplotScript); - execSync("gnuplot /tmp/gnuplot_script"); - - if (!fs.existsSync("assets")) fs.mkdirSync("assets"); - ["req_sec_histogram.png", "latency_histogram.png"].forEach((file) => - fs.renameSync(file, `assets/${file}`) + fs.writeFileSync('/tmp/gnuplot_script', gnuplotScript); + execSync('gnuplot /tmp/gnuplot_script'); + + // Move PNGs to assets + if (!fs.existsSync('assets')) fs.mkdirSync('assets'); + fs.renameSync(reqSecHistogramFile, `assets/${reqSecHistogramFile}`); + fs.renameSync(latencyHistogramFile, `assets/${latencyHistogramFile}`); + + const resultsTable = generateResultsTable( + avgReqSecs, + avgLatencies, + whichBench, ); + const resultsFile = 'results.md'; + fs.writeFileSync(resultsFile, resultsTable); - const resultsTable = generateResultsTable(averages); - updateReadme(resultsTable); - - console.log(resultsTable.replace(/\n?/g, "")); - - execSync( - "git add README.md assets/req_sec_histogram.png assets/latency_histogram.png" - ); - execSync('git commit -m "Updated performance results in README.md"'); - execSync("git push"); + if (whichBench === 3) { + const finalResults = resultsTable.replace(/\n?/g, ''); + console.log(finalResults); + updateReadme(resultsTable); + } - resultFiles.forEach(fs.unlinkSync); + // Delete the result TXT files + resultFiles.forEach((file) => fs.unlinkSync(file)); }; main(); From 4f293bcb9951ba0e90f0591227853281c3787802 Mon Sep 17 00:00:00 2001 From: asr2003 <162500856+asr2003@users.noreply.github.com> Date: Tue, 9 Jul 2024 22:45:18 +0530 Subject: [PATCH 5/6] Delete analyze.sh --- analyze.sh | 164 ----------------------------------------------------- 1 file changed, 164 deletions(-) delete mode 100755 analyze.sh diff --git a/analyze.sh b/analyze.sh deleted file mode 100755 index 17bbbce1..00000000 --- a/analyze.sh +++ /dev/null @@ -1,164 +0,0 @@ -#!/bin/bash - -function extractMetric() { - local file="$1" - local metric="$2" - grep "$metric" "$file" | awk '{print $2}' | sed 's/ms//' -} - -function average() { - echo "$@" | awk '{for(i=1;i<=NF;i++) s+=$i; print s/NF}' -} - -declare -A formattedServerNames -formattedServerNames=( - ["tailcall"]="Tailcall" - ["gqlgen"]="Gqlgen" - ["apollo"]="Apollo GraphQL" - ["netflixdgs"]="Netflix DGS" - ["caliban"]="Caliban" - ["async_graphql"]="async-graphql" - ["hasura"]="Hasura" - ["graphql_jit"]="GraphQL JIT" -) - -servers=("apollo" "caliban" "netflixdgs" "gqlgen" "tailcall" "async_graphql" "hasura" "graphql_jit") -resultFiles=("$@") -declare -A avgReqSecs -declare -A avgLatencies - -# Extract metrics and calculate averages -for idx in "${!servers[@]}"; do - startIdx=$((idx * 3)) - reqSecVals=() - latencyVals=() - for j in 0 1 2; do - fileIdx=$((startIdx + j)) - reqSecVals+=($(extractMetric "${resultFiles[$fileIdx]}" "Requests/sec")) - latencyVals+=($(extractMetric "${resultFiles[$fileIdx]}" "Latency")) - done - avgReqSecs[${servers[$idx]}]=$(average "${reqSecVals[@]}") - avgLatencies[${servers[$idx]}]=$(average "${latencyVals[@]}") -done - -# Generating data files for gnuplot -reqSecData="/tmp/reqSec.dat" -latencyData="/tmp/latency.dat" - -echo "Server Value" >"$reqSecData" -for server in "${servers[@]}"; do - echo "$server ${avgReqSecs[$server]}" >>"$reqSecData" -done - -echo "Server Value" >"$latencyData" -for server in "${servers[@]}"; do - echo "$server ${avgLatencies[$server]}" >>"$latencyData" -done - -whichBench=1 -if [[ $1 == bench2* ]]; then - whichBench=2 -elif [[ $1 == bench3* ]]; then - whichBench=3 -fi - -reqSecHistogramFile="req_sec_histogram${whichBench}.png" -latencyHistogramFile="latency_histogram${whichBench}.png" - -# Plotting using gnuplot -gnuplot <<-EOF - set term pngcairo size 1280,720 enhanced font "Courier,12" - set output "$reqSecHistogramFile" - set style data histograms - set style histogram cluster gap 1 - set style fill solid border -1 - set xtics rotate by -45 - set boxwidth 0.9 - set title "Requests/Sec" - stats "$reqSecData" using 2 nooutput - set yrange [0:STATS_max*1.2] - set key outside right top - plot "$reqSecData" using 2:xtic(1) title "Req/Sec" - - set output "$latencyHistogramFile" - set title "Latency (in ms)" - stats "$latencyData" using 2 nooutput - set yrange [0:STATS_max*1.2] - plot "$latencyData" using 2:xtic(1) title "Latency" -EOF - -# Move PNGs to assets -mkdir -p assets -mv $reqSecHistogramFile assets/ -mv $latencyHistogramFile assets/ - -# Declare an associative array for server RPS -declare -A serverRPS - -# Populate the serverRPS array -for server in "${servers[@]}"; do - serverRPS[$server]=${avgReqSecs[$server]} -done - -# Get the servers sorted by RPS in descending order -IFS=$'\n' sortedServers=($(for server in "${!serverRPS[@]}"; do echo "$server ${serverRPS[$server]}"; done | sort -rn -k2 | cut -d' ' -f1)) - -echo "Sorted servers: ${sortedServers[@]}" -lastServer="${sortedServers[-1]}" -lastServerReqSecs=${avgReqSecs[$lastServer]} - -# Start building the resultsTable -if [[ $whichBench == 1 ]]; then - resultsTable="\n\n| Query | Server | Requests/sec | Latency (ms) | Relative |\n|-------:|--------:|--------------:|--------------:|---------:|\n| $whichBench | \`{ posts { id userId title user { id name email }}}\` |" -elif [[ $whichBench == 2 ]]; then - resultsTable="| $whichBench | \`{ posts { title }}\` |" -elif [[ $whichBench == 3 ]]; then - resultsTable="| $whichBench | \`{ greet }\` |" -fi - -# Build the resultsTable with sorted servers and formatted numbers -for server in "${sortedServers[@]}"; do - formattedReqSecs=$(printf "%.2f" ${avgReqSecs[$server]} | perl -pe 's/(?<=\d)(?=(\d{3})+(\.\d*)?$)/,/g') - formattedLatencies=$(printf "%.2f" ${avgLatencies[$server]} | perl -pe 's/(?<=\d)(?=(\d{3})+(\.\d*)?$)/,/g') - # Calculate the relative performance - relativePerformance=$(echo "${avgReqSecs[$server]} $lastServerReqSecs" | awk '{printf "%.2f", $1 / $2}') - - resultsTable+="\n|| [${formattedServerNames[$server]}] | \`${formattedReqSecs}\` | \`${formattedLatencies}\` | \`${relativePerformance}x\` |" -done - -if [[ $whichBench == 3 ]]; then - resultsTable+="\n\n" -fi - -echo "resultsTable: $resultsTable" - -# Print the results table in a new file -resultsFile="results.md" -echo -e $resultsTable >> $resultsFile - - -if [[ $whichBench == 3 ]]; then - finalResults=$(printf '%s\n' "$(cat $resultsFile)" | sed 's/$/\\n/'| tr -d '\n') - # Remove the last newline character - finalResults=${finalResults::-2} - - # Print the results as a table in the terminal - echo -e $finalResults | sed "s///;s///" - # Check if the markers are present - if grep -q "PERFORMANCE_RESULTS_START" README.md; then - # Replace the old results with the new results - sed -i "/PERFORMANCE_RESULTS_START/,/PERFORMANCE_RESULTS_END/c\\$finalResults" README.md - else - # Append the results at the end of the README.md file - echo -e "\n$finalResults" >> README.md - fi -fi - -# Move the generated images to the assets folder -mv $reqSecHistogramFile assets/ -mv $latencyHistogramFile assets/ - -# Delete the result TXT files -for file in "${resultFiles[@]}"; do - rm "$file" -done From 0b7a75604aff06f843762d5e374db9e435f6fa89 Mon Sep 17 00:00:00 2001 From: asr2003 <162500856+asr2003@users.noreply.github.com> Date: Wed, 10 Jul 2024 16:05:39 +0530 Subject: [PATCH 6/6] Replacing `analyze.sh` with `analyze.js` to analyze the benchmark results --- run_benchmarks.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/run_benchmarks.sh b/run_benchmarks.sh index 263e6f69..511bfebd 100755 --- a/run_benchmarks.sh +++ b/run_benchmarks.sh @@ -83,6 +83,7 @@ for service in "apollo_server" "caliban" "netflix_dgs" "gqlgen" "tailcall" "asyn fi done -bash analyze.sh "${bench1Results[@]}" -bash analyze.sh "${bench2Results[@]}" -bash analyze.sh "${bench3Results[@]}" +# Run the analysis using analyze.js +node analyze.js "${bench1Results[@]}" +node analyze.js "${bench2Results[@]}" +node analyze.js "${bench3Results[@]}"