From f12abde0668aa657c1cf50bd74cb74d1416d9f9d Mon Sep 17 00:00:00 2001 From: daveads Date: Thu, 18 Jul 2024 23:09:20 +0100 Subject: [PATCH 1/8] Port analyze.sh to analyze.js - Converted Bash script to JavaScript for Node.js environment - Maintained core functionality of metric extraction and analysis - Utilized child_process to execute shell commands for file parsing - Implemented Gnuplot execution via Node.js - Ensured compatibility with existing benchmark result files --- analyze.js | 198 ++++++++++++++++++++++++++++++++++++++++++++++ analyze.sh | 164 -------------------------------------- run_benchmarks.sh | 6 +- 3 files changed, 201 insertions(+), 167 deletions(-) create mode 100644 analyze.js delete mode 100755 analyze.sh diff --git a/analyze.js b/analyze.js new file mode 100644 index 00000000..7bc07c96 --- /dev/null +++ b/analyze.js @@ -0,0 +1,198 @@ +#!/usr/bin/env node + +const { execSync } = require('child_process'); +const fs = require('fs'); +const path = require('path'); + +function extractMetric(file, metric) { + try { + const command = `grep "${metric}" "${file}" | awk '{print $2}' | sed 's/ms//'`; + const result = execSync(command, { encoding: 'utf-8' }).trim(); + return result; + } catch (error) { + console.error(`Error extracting metric from ${file}: ${error.message}`); + return null; + } +} + +function average(values) { + const sum = values.reduce((a, b) => parseFloat(a) + parseFloat(b), 0); + return sum / values.length; +} + +const formattedServerNames = { + tailcall: "Tailcall", + gqlgen: "Gqlgen", + apollo: "Apollo GraphQL", + netflixdgs: "Netflix DGS", + caliban: "Caliban", + async_graphql: "async-graphql", + hasura: "Hasura", + graphql_jit: "GraphQL JIT", +}; + +const servers = ["apollo", "caliban", "netflixdgs", "gqlgen", "tailcall", "async_graphql", "hasura", "graphql_jit"]; +const resultFiles = process.argv.slice(2); +const avgReqSecs = {}; +const avgLatencies = {}; + +servers.forEach((server, idx) => { + const startIdx = idx * 3; + const reqSecVals = []; + const latencyVals = []; + for (let j = 0; j < 3; j++) { + const fileIdx = startIdx + j; + const reqSec = extractMetric(resultFiles[fileIdx], "Requests/sec"); + const latency = extractMetric(resultFiles[fileIdx], "Latency"); + if (reqSec !== null) reqSecVals.push(reqSec); + if (latency !== null) latencyVals.push(latency); + } + avgReqSecs[server] = average(reqSecVals); + avgLatencies[server] = average(latencyVals); +}); + +const reqSecData = "/tmp/reqSec.dat"; +const latencyData = "/tmp/latency.dat"; + +fs.writeFileSync(reqSecData, "Server Value\n" + servers.map(server => `${server} ${avgReqSecs[server]}`).join('\n')); +fs.writeFileSync(latencyData, "Server Value\n" + servers.map(server => `${server} ${avgLatencies[server]}`).join('\n')); + +let whichBench = 1; +if (resultFiles[0].startsWith("bench2")) { + whichBench = 2; +} else if (resultFiles[0].startsWith("bench3")) { + whichBench = 3; +} + +const reqSecHistogramFile = `req_sec_histogram${whichBench}.png`; +const latencyHistogramFile = `latency_histogram${whichBench}.png`; + +function getMaxValue(data) { + return Math.max(...data.split('\n').slice(1).map(line => parseFloat(line.split(' ')[1]))); +} + +const reqSecMax = getMaxValue(fs.readFileSync(reqSecData, 'utf-8')) * 1.2; +const latencyMax = getMaxValue(fs.readFileSync(latencyData, 'utf-8')) * 1.2; + +const gnuplotScript = ` +set term pngcairo size 1280,720 enhanced font 'Courier,12' +set output '${reqSecHistogramFile}' +set style data histograms +set style histogram cluster gap 1 +set style fill solid border -1 +set xtics rotate by -45 +set boxwidth 0.9 +set title 'Requests/Sec' +set yrange [0:${reqSecMax}] +set key outside right top +plot '${reqSecData}' using 2:xtic(1) title 'Req/Sec' + +set output '${latencyHistogramFile}' +set title 'Latency (in ms)' +set yrange [0:${latencyMax}] +plot '${latencyData}' using 2:xtic(1) title 'Latency' +`; + +const gnuplotScriptFile = '/tmp/gnuplot_script.gp'; +fs.writeFileSync(gnuplotScriptFile, gnuplotScript); + +try { + execSync(`gnuplot ${gnuplotScriptFile}`, { stdio: 'inherit' }); + console.log('Gnuplot executed successfully'); +} catch (error) { + console.error('Error executing gnuplot:', error.message); + process.exit(1); +} + +const assetsDir = path.join(__dirname, "assets"); +if (!fs.existsSync(assetsDir)) { + fs.mkdirSync(assetsDir); +} + +function moveFile(source, destination) { + try { + if (fs.existsSync(source)) { + fs.renameSync(source, destination); + console.log(`Moved ${source} to ${destination}`); + } else { + console.log(`Source file ${source} does not exist`); + } + } catch (error) { + console.error(`Error moving file ${source}: ${error.message}`); + } +} + +moveFile(reqSecHistogramFile, path.join(assetsDir, reqSecHistogramFile)); +moveFile(latencyHistogramFile, path.join(assetsDir, latencyHistogramFile)); + +const serverRPS = {}; +servers.forEach((server) => { + serverRPS[server] = avgReqSecs[server]; +}); + +const sortedServers = Object.keys(serverRPS).sort( + (a, b) => serverRPS[b] - serverRPS[a] +); +const lastServer = sortedServers[sortedServers.length - 1]; +const lastServerReqSecs = avgReqSecs[lastServer]; + +const resultsFile = "results.md"; + +if (!fs.existsSync(resultsFile) || fs.readFileSync(resultsFile, 'utf8').trim() === '') { + fs.writeFileSync(resultsFile, ` + +| Query | Server | Requests/sec | Latency (ms) | Relative | +|-------:|--------:|--------------:|--------------:|---------:|`); +} + +let resultsTable = ""; + +if (whichBench === 1) { + resultsTable += `\n| ${whichBench} | \`{ posts { id userId title user { id name email }}}\` |`; +} else if (whichBench === 2) { + resultsTable += `\n| ${whichBench} | \`{ posts { title }}\` |`; +} else if (whichBench === 3) { + resultsTable += `\n| ${whichBench} | \`{ greet }\` |`; +} + +sortedServers.forEach((server) => { + const formattedReqSecs = avgReqSecs[server].toLocaleString(undefined, { + minimumFractionDigits: 2, + maximumFractionDigits: 2, + }); + const formattedLatencies = avgLatencies[server].toLocaleString(undefined, { + minimumFractionDigits: 2, + maximumFractionDigits: 2, + }); + const relativePerformance = (avgReqSecs[server] / lastServerReqSecs).toFixed(2); + + resultsTable += `\n|| [${formattedServerNames[server]}] | \`${formattedReqSecs}\` | \`${formattedLatencies}\` | \`${relativePerformance}x\` |`; +}); + +fs.appendFileSync(resultsFile, resultsTable + "\n"); + +if (whichBench === 3) { + fs.appendFileSync(resultsFile, "\n"); + + const finalResults = fs + .readFileSync(resultsFile, "utf-8") + .replace(/(\r\n|\n|\r)/gm, "\\n"); + + const readmePath = "README.md"; + let readmeContent = fs.readFileSync(readmePath, "utf-8"); + const performanceResultsRegex = + /[\s\S]*/; + if (performanceResultsRegex.test(readmeContent)) { + readmeContent = readmeContent.replace( + performanceResultsRegex, + finalResults + ); + } else { + readmeContent += `\n${finalResults}`; + } + fs.writeFileSync(readmePath, readmeContent); +} + +resultFiles.forEach((file) => { + fs.unlinkSync(file); +}); diff --git a/analyze.sh b/analyze.sh deleted file mode 100755 index 17bbbce1..00000000 --- a/analyze.sh +++ /dev/null @@ -1,164 +0,0 @@ -#!/bin/bash - -function extractMetric() { - local file="$1" - local metric="$2" - grep "$metric" "$file" | awk '{print $2}' | sed 's/ms//' -} - -function average() { - echo "$@" | awk '{for(i=1;i<=NF;i++) s+=$i; print s/NF}' -} - -declare -A formattedServerNames -formattedServerNames=( - ["tailcall"]="Tailcall" - ["gqlgen"]="Gqlgen" - ["apollo"]="Apollo GraphQL" - ["netflixdgs"]="Netflix DGS" - ["caliban"]="Caliban" - ["async_graphql"]="async-graphql" - ["hasura"]="Hasura" - ["graphql_jit"]="GraphQL JIT" -) - -servers=("apollo" "caliban" "netflixdgs" "gqlgen" "tailcall" "async_graphql" "hasura" "graphql_jit") -resultFiles=("$@") -declare -A avgReqSecs -declare -A avgLatencies - -# Extract metrics and calculate averages -for idx in "${!servers[@]}"; do - startIdx=$((idx * 3)) - reqSecVals=() - latencyVals=() - for j in 0 1 2; do - fileIdx=$((startIdx + j)) - reqSecVals+=($(extractMetric "${resultFiles[$fileIdx]}" "Requests/sec")) - latencyVals+=($(extractMetric "${resultFiles[$fileIdx]}" "Latency")) - done - avgReqSecs[${servers[$idx]}]=$(average "${reqSecVals[@]}") - avgLatencies[${servers[$idx]}]=$(average "${latencyVals[@]}") -done - -# Generating data files for gnuplot -reqSecData="/tmp/reqSec.dat" -latencyData="/tmp/latency.dat" - -echo "Server Value" >"$reqSecData" -for server in "${servers[@]}"; do - echo "$server ${avgReqSecs[$server]}" >>"$reqSecData" -done - -echo "Server Value" >"$latencyData" -for server in "${servers[@]}"; do - echo "$server ${avgLatencies[$server]}" >>"$latencyData" -done - -whichBench=1 -if [[ $1 == bench2* ]]; then - whichBench=2 -elif [[ $1 == bench3* ]]; then - whichBench=3 -fi - -reqSecHistogramFile="req_sec_histogram${whichBench}.png" -latencyHistogramFile="latency_histogram${whichBench}.png" - -# Plotting using gnuplot -gnuplot <<-EOF - set term pngcairo size 1280,720 enhanced font "Courier,12" - set output "$reqSecHistogramFile" - set style data histograms - set style histogram cluster gap 1 - set style fill solid border -1 - set xtics rotate by -45 - set boxwidth 0.9 - set title "Requests/Sec" - stats "$reqSecData" using 2 nooutput - set yrange [0:STATS_max*1.2] - set key outside right top - plot "$reqSecData" using 2:xtic(1) title "Req/Sec" - - set output "$latencyHistogramFile" - set title "Latency (in ms)" - stats "$latencyData" using 2 nooutput - set yrange [0:STATS_max*1.2] - plot "$latencyData" using 2:xtic(1) title "Latency" -EOF - -# Move PNGs to assets -mkdir -p assets -mv $reqSecHistogramFile assets/ -mv $latencyHistogramFile assets/ - -# Declare an associative array for server RPS -declare -A serverRPS - -# Populate the serverRPS array -for server in "${servers[@]}"; do - serverRPS[$server]=${avgReqSecs[$server]} -done - -# Get the servers sorted by RPS in descending order -IFS=$'\n' sortedServers=($(for server in "${!serverRPS[@]}"; do echo "$server ${serverRPS[$server]}"; done | sort -rn -k2 | cut -d' ' -f1)) - -echo "Sorted servers: ${sortedServers[@]}" -lastServer="${sortedServers[-1]}" -lastServerReqSecs=${avgReqSecs[$lastServer]} - -# Start building the resultsTable -if [[ $whichBench == 1 ]]; then - resultsTable="\n\n| Query | Server | Requests/sec | Latency (ms) | Relative |\n|-------:|--------:|--------------:|--------------:|---------:|\n| $whichBench | \`{ posts { id userId title user { id name email }}}\` |" -elif [[ $whichBench == 2 ]]; then - resultsTable="| $whichBench | \`{ posts { title }}\` |" -elif [[ $whichBench == 3 ]]; then - resultsTable="| $whichBench | \`{ greet }\` |" -fi - -# Build the resultsTable with sorted servers and formatted numbers -for server in "${sortedServers[@]}"; do - formattedReqSecs=$(printf "%.2f" ${avgReqSecs[$server]} | perl -pe 's/(?<=\d)(?=(\d{3})+(\.\d*)?$)/,/g') - formattedLatencies=$(printf "%.2f" ${avgLatencies[$server]} | perl -pe 's/(?<=\d)(?=(\d{3})+(\.\d*)?$)/,/g') - # Calculate the relative performance - relativePerformance=$(echo "${avgReqSecs[$server]} $lastServerReqSecs" | awk '{printf "%.2f", $1 / $2}') - - resultsTable+="\n|| [${formattedServerNames[$server]}] | \`${formattedReqSecs}\` | \`${formattedLatencies}\` | \`${relativePerformance}x\` |" -done - -if [[ $whichBench == 3 ]]; then - resultsTable+="\n\n" -fi - -echo "resultsTable: $resultsTable" - -# Print the results table in a new file -resultsFile="results.md" -echo -e $resultsTable >> $resultsFile - - -if [[ $whichBench == 3 ]]; then - finalResults=$(printf '%s\n' "$(cat $resultsFile)" | sed 's/$/\\n/'| tr -d '\n') - # Remove the last newline character - finalResults=${finalResults::-2} - - # Print the results as a table in the terminal - echo -e $finalResults | sed "s///;s///" - # Check if the markers are present - if grep -q "PERFORMANCE_RESULTS_START" README.md; then - # Replace the old results with the new results - sed -i "/PERFORMANCE_RESULTS_START/,/PERFORMANCE_RESULTS_END/c\\$finalResults" README.md - else - # Append the results at the end of the README.md file - echo -e "\n$finalResults" >> README.md - fi -fi - -# Move the generated images to the assets folder -mv $reqSecHistogramFile assets/ -mv $latencyHistogramFile assets/ - -# Delete the result TXT files -for file in "${resultFiles[@]}"; do - rm "$file" -done diff --git a/run_benchmarks.sh b/run_benchmarks.sh index 263e6f69..09d625e3 100755 --- a/run_benchmarks.sh +++ b/run_benchmarks.sh @@ -83,6 +83,6 @@ for service in "apollo_server" "caliban" "netflix_dgs" "gqlgen" "tailcall" "asyn fi done -bash analyze.sh "${bench1Results[@]}" -bash analyze.sh "${bench2Results[@]}" -bash analyze.sh "${bench3Results[@]}" +node analyze.js "${bench1Results[@]}" +node analyze.js "${bench2Results[@]}" +node analyze.js "${bench3Results[@]}" From af52228f3c152017586ebfd4db9b56f2c8fb9dc4 Mon Sep 17 00:00:00 2001 From: daveads Date: Fri, 19 Jul 2024 03:24:10 +0100 Subject: [PATCH 2/8] newline at the end of resultsTable --- analyze.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/analyze.js b/analyze.js index 7bc07c96..262c6997 100644 --- a/analyze.js +++ b/analyze.js @@ -169,10 +169,10 @@ sortedServers.forEach((server) => { resultsTable += `\n|| [${formattedServerNames[server]}] | \`${formattedReqSecs}\` | \`${formattedLatencies}\` | \`${relativePerformance}x\` |`; }); -fs.appendFileSync(resultsFile, resultsTable + "\n"); +fs.appendFileSync(resultsFile, resultsTable); if (whichBench === 3) { - fs.appendFileSync(resultsFile, "\n"); + fs.appendFileSync(resultsFile, "\n\n"); const finalResults = fs .readFileSync(resultsFile, "utf-8") From e2cec2a3cf0a0c83d9695d0d8a3a3727280457e6 Mon Sep 17 00:00:00 2001 From: daveads Date: Mon, 22 Jul 2024 19:22:26 +0100 Subject: [PATCH 3/8] A fix the result output to README.md file. --- analyze.js | 124 ++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 84 insertions(+), 40 deletions(-) diff --git a/analyze.js b/analyze.js index 262c6997..78a40454 100644 --- a/analyze.js +++ b/analyze.js @@ -1,5 +1,4 @@ #!/usr/bin/env node - const { execSync } = require('child_process'); const fs = require('fs'); const path = require('path'); @@ -16,6 +15,7 @@ function extractMetric(file, metric) { } function average(values) { + if (values.length === 0) return 0; const sum = values.reduce((a, b) => parseFloat(a) + parseFloat(b), 0); return sum / values.length; } @@ -42,10 +42,12 @@ servers.forEach((server, idx) => { const latencyVals = []; for (let j = 0; j < 3; j++) { const fileIdx = startIdx + j; - const reqSec = extractMetric(resultFiles[fileIdx], "Requests/sec"); - const latency = extractMetric(resultFiles[fileIdx], "Latency"); - if (reqSec !== null) reqSecVals.push(reqSec); - if (latency !== null) latencyVals.push(latency); + if (fileIdx < resultFiles.length) { + const reqSec = extractMetric(resultFiles[fileIdx], "Requests/sec"); + const latency = extractMetric(resultFiles[fileIdx], "Latency"); + if (reqSec !== null) reqSecVals.push(reqSec); + if (latency !== null) latencyVals.push(latency); + } } avgReqSecs[server] = average(reqSecVals); avgLatencies[server] = average(latencyVals); @@ -54,25 +56,43 @@ servers.forEach((server, idx) => { const reqSecData = "/tmp/reqSec.dat"; const latencyData = "/tmp/latency.dat"; -fs.writeFileSync(reqSecData, "Server Value\n" + servers.map(server => `${server} ${avgReqSecs[server]}`).join('\n')); -fs.writeFileSync(latencyData, "Server Value\n" + servers.map(server => `${server} ${avgLatencies[server]}`).join('\n')); +try { + fs.writeFileSync(reqSecData, "Server Value\n" + servers.map(server => `${server} ${avgReqSecs[server]}`).join('\n')); + fs.writeFileSync(latencyData, "Server Value\n" + servers.map(server => `${server} ${avgLatencies[server]}`).join('\n')); +} catch (error) { + console.error(`Error writing data files: ${error.message}`); +} let whichBench = 1; -if (resultFiles[0].startsWith("bench2")) { - whichBench = 2; -} else if (resultFiles[0].startsWith("bench3")) { - whichBench = 3; +if (resultFiles.length > 0) { + if (resultFiles[0].startsWith("bench2")) { + whichBench = 2; + } else if (resultFiles[0].startsWith("bench3")) { + whichBench = 3; + } } const reqSecHistogramFile = `req_sec_histogram${whichBench}.png`; const latencyHistogramFile = `latency_histogram${whichBench}.png`; function getMaxValue(data) { - return Math.max(...data.split('\n').slice(1).map(line => parseFloat(line.split(' ')[1]))); + try { + return Math.max(...data.split('\n').slice(1).map(line => parseFloat(line.split(' ')[1]))); + } catch (error) { + console.error(`Error getting max value: ${error.message}`); + return 0; + } } -const reqSecMax = getMaxValue(fs.readFileSync(reqSecData, 'utf-8')) * 1.2; -const latencyMax = getMaxValue(fs.readFileSync(latencyData, 'utf-8')) * 1.2; +let reqSecMax, latencyMax; +try { + reqSecMax = getMaxValue(fs.readFileSync(reqSecData, 'utf-8')) * 1.2; + latencyMax = getMaxValue(fs.readFileSync(latencyData, 'utf-8')) * 1.2; +} catch (error) { + console.error(`Error reading data files: ${error.message}`); + reqSecMax = 0; + latencyMax = 0; +} const gnuplotScript = ` set term pngcairo size 1280,720 enhanced font 'Courier,12' @@ -94,19 +114,26 @@ plot '${latencyData}' using 2:xtic(1) title 'Latency' `; const gnuplotScriptFile = '/tmp/gnuplot_script.gp'; -fs.writeFileSync(gnuplotScriptFile, gnuplotScript); +try { + fs.writeFileSync(gnuplotScriptFile, gnuplotScript); +} catch (error) { + console.error(`Error writing gnuplot script: ${error.message}`); +} try { execSync(`gnuplot ${gnuplotScriptFile}`, { stdio: 'inherit' }); console.log('Gnuplot executed successfully'); } catch (error) { console.error('Error executing gnuplot:', error.message); - process.exit(1); } const assetsDir = path.join(__dirname, "assets"); if (!fs.existsSync(assetsDir)) { - fs.mkdirSync(assetsDir); + try { + fs.mkdirSync(assetsDir); + } catch (error) { + console.error(`Error creating assets directory: ${error.message}`); + } } function moveFile(source, destination) { @@ -138,11 +165,15 @@ const lastServerReqSecs = avgReqSecs[lastServer]; const resultsFile = "results.md"; -if (!fs.existsSync(resultsFile) || fs.readFileSync(resultsFile, 'utf8').trim() === '') { - fs.writeFileSync(resultsFile, ` +try { + if (!fs.existsSync(resultsFile) || fs.readFileSync(resultsFile, 'utf8').trim() === '') { + fs.writeFileSync(resultsFile, ` | Query | Server | Requests/sec | Latency (ms) | Relative | |-------:|--------:|--------------:|--------------:|---------:|`); + } +} catch (error) { + console.error(`Error initializing results file: ${error.message}`); } let resultsTable = ""; @@ -169,30 +200,43 @@ sortedServers.forEach((server) => { resultsTable += `\n|| [${formattedServerNames[server]}] | \`${formattedReqSecs}\` | \`${formattedLatencies}\` | \`${relativePerformance}x\` |`; }); -fs.appendFileSync(resultsFile, resultsTable); +try { + fs.appendFileSync(resultsFile, resultsTable); +} catch (error) { + console.error(`Error appending to results file: ${error.message}`); +} if (whichBench === 3) { - fs.appendFileSync(resultsFile, "\n\n"); - - const finalResults = fs - .readFileSync(resultsFile, "utf-8") - .replace(/(\r\n|\n|\r)/gm, "\\n"); - - const readmePath = "README.md"; - let readmeContent = fs.readFileSync(readmePath, "utf-8"); - const performanceResultsRegex = - /[\s\S]*/; - if (performanceResultsRegex.test(readmeContent)) { - readmeContent = readmeContent.replace( - performanceResultsRegex, - finalResults - ); - } else { - readmeContent += `\n${finalResults}`; + try { + fs.appendFileSync(resultsFile, "\n\n"); + + const finalResults = fs + .readFileSync(resultsFile, "utf-8") + .replace(/\\/g, ''); // Remove backslashes + + const readmePath = "README.md"; + let readmeContent = fs.readFileSync(readmePath, "utf-8"); + const performanceResultsRegex = + /[\s\S]*/; + if (performanceResultsRegex.test(readmeContent)) { + readmeContent = readmeContent.replace( + performanceResultsRegex, + finalResults + ); + } else { + readmeContent += `\n${finalResults}`; + } + fs.writeFileSync(readmePath, readmeContent); + console.log("README.md updated successfully"); + } catch (error) { + console.error(`Error updating README: ${error.message}`); } - fs.writeFileSync(readmePath, readmeContent); } resultFiles.forEach((file) => { - fs.unlinkSync(file); -}); + try { + fs.unlinkSync(file); + } catch (error) { + console.error(`Error deleting file ${file}: ${error.message}`); + } +}); \ No newline at end of file From 14fb08050954e40c2e5a108ba77043d58b0d168d Mon Sep 17 00:00:00 2001 From: daveads Date: Tue, 23 Jul 2024 13:23:36 +0100 Subject: [PATCH 4/8] updates --- run_analyze_script.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run_analyze_script.sh b/run_analyze_script.sh index 84f7206a..dd284135 100755 --- a/run_analyze_script.sh +++ b/run_analyze_script.sh @@ -13,7 +13,7 @@ for bench in 1 2 3; do echo "Processing files for bench${bench}:" # Construct the command for each benchmark - cmd="bash analyze.sh" + cmd="node analyze.js" # Loop through each service for service in "${services[@]}"; do From 4c10b3458b4f733aa7a8fd2ca6119d765f26fe76 Mon Sep 17 00:00:00 2001 From: daveads Date: Sun, 28 Jul 2024 13:00:15 +0100 Subject: [PATCH 5/8] based off the review --- analyze.js | 242 ------------------------------------ analyze.ts | 76 +++++++++++ analyze/config.js | 14 +++ analyze/config.ts | 14 +++ analyze/dataFileWriter.js | 10 ++ analyze/dataFileWriter.ts | 10 ++ analyze/fileUtils.js | 48 +++++++ analyze/fileUtils.ts | 42 +++++++ analyze/gnuplotGenerator.js | 14 +++ analyze/gnuplotGenerator.ts | 30 +++++ analyze/parser.js | 49 ++++++++ analyze/parser.ts | 48 +++++++ analyze/resultsFormatter.js | 70 +++++++++++ analyze/resultsFormatter.ts | 83 +++++++++++++ analyze/types.js | 2 + analyze/types.ts | 8 ++ run_analyze_script.sh | 7 +- tsconfig.json | 9 ++ 18 files changed, 531 insertions(+), 245 deletions(-) delete mode 100644 analyze.js create mode 100644 analyze.ts create mode 100644 analyze/config.js create mode 100644 analyze/config.ts create mode 100644 analyze/dataFileWriter.js create mode 100644 analyze/dataFileWriter.ts create mode 100644 analyze/fileUtils.js create mode 100644 analyze/fileUtils.ts create mode 100644 analyze/gnuplotGenerator.js create mode 100644 analyze/gnuplotGenerator.ts create mode 100644 analyze/parser.js create mode 100644 analyze/parser.ts create mode 100644 analyze/resultsFormatter.js create mode 100644 analyze/resultsFormatter.ts create mode 100644 analyze/types.js create mode 100644 analyze/types.ts create mode 100644 tsconfig.json diff --git a/analyze.js b/analyze.js deleted file mode 100644 index 78a40454..00000000 --- a/analyze.js +++ /dev/null @@ -1,242 +0,0 @@ -#!/usr/bin/env node -const { execSync } = require('child_process'); -const fs = require('fs'); -const path = require('path'); - -function extractMetric(file, metric) { - try { - const command = `grep "${metric}" "${file}" | awk '{print $2}' | sed 's/ms//'`; - const result = execSync(command, { encoding: 'utf-8' }).trim(); - return result; - } catch (error) { - console.error(`Error extracting metric from ${file}: ${error.message}`); - return null; - } -} - -function average(values) { - if (values.length === 0) return 0; - const sum = values.reduce((a, b) => parseFloat(a) + parseFloat(b), 0); - return sum / values.length; -} - -const formattedServerNames = { - tailcall: "Tailcall", - gqlgen: "Gqlgen", - apollo: "Apollo GraphQL", - netflixdgs: "Netflix DGS", - caliban: "Caliban", - async_graphql: "async-graphql", - hasura: "Hasura", - graphql_jit: "GraphQL JIT", -}; - -const servers = ["apollo", "caliban", "netflixdgs", "gqlgen", "tailcall", "async_graphql", "hasura", "graphql_jit"]; -const resultFiles = process.argv.slice(2); -const avgReqSecs = {}; -const avgLatencies = {}; - -servers.forEach((server, idx) => { - const startIdx = idx * 3; - const reqSecVals = []; - const latencyVals = []; - for (let j = 0; j < 3; j++) { - const fileIdx = startIdx + j; - if (fileIdx < resultFiles.length) { - const reqSec = extractMetric(resultFiles[fileIdx], "Requests/sec"); - const latency = extractMetric(resultFiles[fileIdx], "Latency"); - if (reqSec !== null) reqSecVals.push(reqSec); - if (latency !== null) latencyVals.push(latency); - } - } - avgReqSecs[server] = average(reqSecVals); - avgLatencies[server] = average(latencyVals); -}); - -const reqSecData = "/tmp/reqSec.dat"; -const latencyData = "/tmp/latency.dat"; - -try { - fs.writeFileSync(reqSecData, "Server Value\n" + servers.map(server => `${server} ${avgReqSecs[server]}`).join('\n')); - fs.writeFileSync(latencyData, "Server Value\n" + servers.map(server => `${server} ${avgLatencies[server]}`).join('\n')); -} catch (error) { - console.error(`Error writing data files: ${error.message}`); -} - -let whichBench = 1; -if (resultFiles.length > 0) { - if (resultFiles[0].startsWith("bench2")) { - whichBench = 2; - } else if (resultFiles[0].startsWith("bench3")) { - whichBench = 3; - } -} - -const reqSecHistogramFile = `req_sec_histogram${whichBench}.png`; -const latencyHistogramFile = `latency_histogram${whichBench}.png`; - -function getMaxValue(data) { - try { - return Math.max(...data.split('\n').slice(1).map(line => parseFloat(line.split(' ')[1]))); - } catch (error) { - console.error(`Error getting max value: ${error.message}`); - return 0; - } -} - -let reqSecMax, latencyMax; -try { - reqSecMax = getMaxValue(fs.readFileSync(reqSecData, 'utf-8')) * 1.2; - latencyMax = getMaxValue(fs.readFileSync(latencyData, 'utf-8')) * 1.2; -} catch (error) { - console.error(`Error reading data files: ${error.message}`); - reqSecMax = 0; - latencyMax = 0; -} - -const gnuplotScript = ` -set term pngcairo size 1280,720 enhanced font 'Courier,12' -set output '${reqSecHistogramFile}' -set style data histograms -set style histogram cluster gap 1 -set style fill solid border -1 -set xtics rotate by -45 -set boxwidth 0.9 -set title 'Requests/Sec' -set yrange [0:${reqSecMax}] -set key outside right top -plot '${reqSecData}' using 2:xtic(1) title 'Req/Sec' - -set output '${latencyHistogramFile}' -set title 'Latency (in ms)' -set yrange [0:${latencyMax}] -plot '${latencyData}' using 2:xtic(1) title 'Latency' -`; - -const gnuplotScriptFile = '/tmp/gnuplot_script.gp'; -try { - fs.writeFileSync(gnuplotScriptFile, gnuplotScript); -} catch (error) { - console.error(`Error writing gnuplot script: ${error.message}`); -} - -try { - execSync(`gnuplot ${gnuplotScriptFile}`, { stdio: 'inherit' }); - console.log('Gnuplot executed successfully'); -} catch (error) { - console.error('Error executing gnuplot:', error.message); -} - -const assetsDir = path.join(__dirname, "assets"); -if (!fs.existsSync(assetsDir)) { - try { - fs.mkdirSync(assetsDir); - } catch (error) { - console.error(`Error creating assets directory: ${error.message}`); - } -} - -function moveFile(source, destination) { - try { - if (fs.existsSync(source)) { - fs.renameSync(source, destination); - console.log(`Moved ${source} to ${destination}`); - } else { - console.log(`Source file ${source} does not exist`); - } - } catch (error) { - console.error(`Error moving file ${source}: ${error.message}`); - } -} - -moveFile(reqSecHistogramFile, path.join(assetsDir, reqSecHistogramFile)); -moveFile(latencyHistogramFile, path.join(assetsDir, latencyHistogramFile)); - -const serverRPS = {}; -servers.forEach((server) => { - serverRPS[server] = avgReqSecs[server]; -}); - -const sortedServers = Object.keys(serverRPS).sort( - (a, b) => serverRPS[b] - serverRPS[a] -); -const lastServer = sortedServers[sortedServers.length - 1]; -const lastServerReqSecs = avgReqSecs[lastServer]; - -const resultsFile = "results.md"; - -try { - if (!fs.existsSync(resultsFile) || fs.readFileSync(resultsFile, 'utf8').trim() === '') { - fs.writeFileSync(resultsFile, ` - -| Query | Server | Requests/sec | Latency (ms) | Relative | -|-------:|--------:|--------------:|--------------:|---------:|`); - } -} catch (error) { - console.error(`Error initializing results file: ${error.message}`); -} - -let resultsTable = ""; - -if (whichBench === 1) { - resultsTable += `\n| ${whichBench} | \`{ posts { id userId title user { id name email }}}\` |`; -} else if (whichBench === 2) { - resultsTable += `\n| ${whichBench} | \`{ posts { title }}\` |`; -} else if (whichBench === 3) { - resultsTable += `\n| ${whichBench} | \`{ greet }\` |`; -} - -sortedServers.forEach((server) => { - const formattedReqSecs = avgReqSecs[server].toLocaleString(undefined, { - minimumFractionDigits: 2, - maximumFractionDigits: 2, - }); - const formattedLatencies = avgLatencies[server].toLocaleString(undefined, { - minimumFractionDigits: 2, - maximumFractionDigits: 2, - }); - const relativePerformance = (avgReqSecs[server] / lastServerReqSecs).toFixed(2); - - resultsTable += `\n|| [${formattedServerNames[server]}] | \`${formattedReqSecs}\` | \`${formattedLatencies}\` | \`${relativePerformance}x\` |`; -}); - -try { - fs.appendFileSync(resultsFile, resultsTable); -} catch (error) { - console.error(`Error appending to results file: ${error.message}`); -} - -if (whichBench === 3) { - try { - fs.appendFileSync(resultsFile, "\n\n"); - - const finalResults = fs - .readFileSync(resultsFile, "utf-8") - .replace(/\\/g, ''); // Remove backslashes - - const readmePath = "README.md"; - let readmeContent = fs.readFileSync(readmePath, "utf-8"); - const performanceResultsRegex = - /[\s\S]*/; - if (performanceResultsRegex.test(readmeContent)) { - readmeContent = readmeContent.replace( - performanceResultsRegex, - finalResults - ); - } else { - readmeContent += `\n${finalResults}`; - } - fs.writeFileSync(readmePath, readmeContent); - console.log("README.md updated successfully"); - } catch (error) { - console.error(`Error updating README: ${error.message}`); - } -} - -resultFiles.forEach((file) => { - try { - fs.unlinkSync(file); - } catch (error) { - console.error(`Error deleting file ${file}: ${error.message}`); - } -}); \ No newline at end of file diff --git a/analyze.ts b/analyze.ts new file mode 100644 index 00000000..518719f9 --- /dev/null +++ b/analyze.ts @@ -0,0 +1,76 @@ +#!/usr/bin/env node +import { execSync } from 'child_process'; +import * as path from 'path'; +import * as fs from 'fs'; +import { servers, formattedServerNames } from './analyze/config'; +import { createDirectoryIfNotExists, moveFile, readFileContent } from './analyze/fileUtils'; +import { parseServerMetrics } from './analyze/parser'; +import { writeMetricsDataFiles } from './analyze/dataFileWriter'; +import { generateGnuplotScript, writeGnuplotScript } from './analyze/gnuplotGenerator'; +import { formatResults, writeResults } from './analyze/resultsFormatter'; + +const resultFiles: string[] = process.argv.slice(2); + +// Read content of result files +const resultContents: string[] = resultFiles.map(file => readFileContent(file)); + +const serverMetrics = parseServerMetrics(servers, resultContents); + +writeMetricsDataFiles(serverMetrics, servers); + +let whichBench = 1; +if (resultFiles.length > 0) { + if (resultFiles[0].startsWith("bench2")) { + whichBench = 2; + } else if (resultFiles[0].startsWith("bench3")) { + whichBench = 3; + } +} + +function getMaxValue(data: string): number { + try { + return Math.max(...data.split('\n') + .slice(1) + .map(line => { + const [, valueStr] = line.split(' '); + const value = parseFloat(valueStr); + if (isNaN(value)) { + throw new Error(`Invalid number in data: ${valueStr}`); + } + return value; + })); + } catch (error) { + console.error(`Error getting max value: ${(error as Error).message}`); + return 0; + } +} + +const reqSecMax = getMaxValue(readFileContent('/tmp/reqSec.dat')) * 1.2; +const latencyMax = getMaxValue(readFileContent('/tmp/latency.dat')) * 1.2; + +const gnuplotScript = generateGnuplotScript(whichBench, reqSecMax, latencyMax); +writeGnuplotScript(gnuplotScript); + +try { + execSync(`gnuplot /tmp/gnuplot_script.gp`, { stdio: 'inherit' }); + console.log('Gnuplot executed successfully'); +} catch (error) { + console.error('Error executing gnuplot:', (error as Error).message); +} + +const assetsDir = path.join(__dirname, "assets"); +createDirectoryIfNotExists(assetsDir); + +moveFile(`req_sec_histogram${whichBench}.png`, path.join(assetsDir, `req_sec_histogram${whichBench}.png`)); +moveFile(`latency_histogram${whichBench}.png`, path.join(assetsDir, `latency_histogram${whichBench}.png`)); + +const resultsTable = formatResults(serverMetrics, formattedServerNames, whichBench); +writeResults(resultsTable, whichBench); + +resultFiles.forEach((file) => { + try { + fs.unlinkSync(file); + } catch (error) { + console.error(`Error deleting file ${file}: ${(error as Error).message}`); + } +}); diff --git a/analyze/config.js b/analyze/config.js new file mode 100644 index 00000000..393684fa --- /dev/null +++ b/analyze/config.js @@ -0,0 +1,14 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.servers = exports.formattedServerNames = void 0; +exports.formattedServerNames = { + tailcall: "Tailcall", + gqlgen: "Gqlgen", + apollo: "Apollo GraphQL", + netflixdgs: "Netflix DGS", + caliban: "Caliban", + async_graphql: "async-graphql", + hasura: "Hasura", + graphql_jit: "GraphQL JIT", +}; +exports.servers = ["apollo", "caliban", "netflixdgs", "gqlgen", "tailcall", "async_graphql", "hasura", "graphql_jit"]; diff --git a/analyze/config.ts b/analyze/config.ts new file mode 100644 index 00000000..0b84b675 --- /dev/null +++ b/analyze/config.ts @@ -0,0 +1,14 @@ +import { FormattedServerNames } from './types'; + +export const formattedServerNames: FormattedServerNames = { + tailcall: "Tailcall", + gqlgen: "Gqlgen", + apollo: "Apollo GraphQL", + netflixdgs: "Netflix DGS", + caliban: "Caliban", + async_graphql: "async-graphql", + hasura: "Hasura", + graphql_jit: "GraphQL JIT", +}; + +export const servers: string[] = ["apollo", "caliban", "netflixdgs", "gqlgen", "tailcall", "async_graphql", "hasura", "graphql_jit"]; diff --git a/analyze/dataFileWriter.js b/analyze/dataFileWriter.js new file mode 100644 index 00000000..67653355 --- /dev/null +++ b/analyze/dataFileWriter.js @@ -0,0 +1,10 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.writeMetricsDataFiles = writeMetricsDataFiles; +var fileUtils_1 = require("./fileUtils"); +function writeMetricsDataFiles(serverMetrics, servers) { + var reqSecData = "/tmp/reqSec.dat"; + var latencyData = "/tmp/latency.dat"; + (0, fileUtils_1.writeDataFile)(reqSecData, "Server Value\n" + servers.map(function (server) { return "".concat(server, " ").concat(serverMetrics[server].reqSec); }).join('\n')); + (0, fileUtils_1.writeDataFile)(latencyData, "Server Value\n" + servers.map(function (server) { return "".concat(server, " ").concat(serverMetrics[server].latency); }).join('\n')); +} diff --git a/analyze/dataFileWriter.ts b/analyze/dataFileWriter.ts new file mode 100644 index 00000000..3812d5d0 --- /dev/null +++ b/analyze/dataFileWriter.ts @@ -0,0 +1,10 @@ +import { writeDataFile } from './fileUtils'; +import { ServerMetrics } from './types'; + +export function writeMetricsDataFiles(serverMetrics: Record, servers: string[]): void { + const reqSecData = "/tmp/reqSec.dat"; + const latencyData = "/tmp/latency.dat"; + + writeDataFile(reqSecData, "Server Value\n" + servers.map(server => `${server} ${serverMetrics[server].reqSec}`).join('\n')); + writeDataFile(latencyData, "Server Value\n" + servers.map(server => `${server} ${serverMetrics[server].latency}`).join('\n')); +} diff --git a/analyze/fileUtils.js b/analyze/fileUtils.js new file mode 100644 index 00000000..a7459d1f --- /dev/null +++ b/analyze/fileUtils.js @@ -0,0 +1,48 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.writeDataFile = writeDataFile; +exports.readFileContent = readFileContent; +exports.moveFile = moveFile; +exports.createDirectoryIfNotExists = createDirectoryIfNotExists; +var fs = require("fs"); +function writeDataFile(filename, data) { + try { + fs.writeFileSync(filename, data); + } + catch (error) { + console.error("Error writing data file ".concat(filename, ": ").concat(error.message)); + } +} +function readFileContent(filename) { + try { + return fs.readFileSync(filename, 'utf-8'); + } + catch (error) { + console.error("Error reading file ".concat(filename, ": ").concat(error.message)); + return ''; + } +} +function moveFile(source, destination) { + try { + if (fs.existsSync(source)) { + fs.renameSync(source, destination); + console.log("Moved ".concat(source, " to ").concat(destination)); + } + else { + console.log("Source file ".concat(source, " does not exist")); + } + } + catch (error) { + console.error("Error moving file ".concat(source, ": ").concat(error.message)); + } +} +function createDirectoryIfNotExists(dir) { + if (!fs.existsSync(dir)) { + try { + fs.mkdirSync(dir); + } + catch (error) { + console.error("Error creating directory: ".concat(error.message)); + } + } +} diff --git a/analyze/fileUtils.ts b/analyze/fileUtils.ts new file mode 100644 index 00000000..5e7ce332 --- /dev/null +++ b/analyze/fileUtils.ts @@ -0,0 +1,42 @@ +import * as fs from 'fs'; +import * as path from 'path'; + +export function writeDataFile(filename: string, data: string): void { + try { + fs.writeFileSync(filename, data); + } catch (error) { + console.error(`Error writing data file ${filename}: ${(error as Error).message}`); + } +} + +export function readFileContent(filename: string): string { + try { + return fs.readFileSync(filename, 'utf-8'); + } catch (error) { + console.error(`Error reading file ${filename}: ${(error as Error).message}`); + return ''; + } +} + +export function moveFile(source: string, destination: string): void { + try { + if (fs.existsSync(source)) { + fs.renameSync(source, destination); + console.log(`Moved ${source} to ${destination}`); + } else { + console.log(`Source file ${source} does not exist`); + } + } catch (error) { + console.error(`Error moving file ${source}: ${(error as Error).message}`); + } +} + +export function createDirectoryIfNotExists(dir: string): void { + if (!fs.existsSync(dir)) { + try { + fs.mkdirSync(dir); + } catch (error) { + console.error(`Error creating directory: ${(error as Error).message}`); + } + } +} diff --git a/analyze/gnuplotGenerator.js b/analyze/gnuplotGenerator.js new file mode 100644 index 00000000..ba7296da --- /dev/null +++ b/analyze/gnuplotGenerator.js @@ -0,0 +1,14 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.generateGnuplotScript = generateGnuplotScript; +exports.writeGnuplotScript = writeGnuplotScript; +var fileUtils_1 = require("./fileUtils"); +function generateGnuplotScript(whichBench, reqSecMax, latencyMax) { + var reqSecHistogramFile = "req_sec_histogram".concat(whichBench, ".png"); + var latencyHistogramFile = "latency_histogram".concat(whichBench, ".png"); + return "\nset term pngcairo size 1280,720 enhanced font 'Courier,12'\nset output '".concat(reqSecHistogramFile, "'\nset style data histograms\nset style histogram cluster gap 1\nset style fill solid border -1\nset xtics rotate by -45\nset boxwidth 0.9\nset title 'Requests/Sec'\nset yrange [0:").concat(reqSecMax, "]\nset key outside right top\nplot '/tmp/reqSec.dat' using 2:xtic(1) title 'Req/Sec'\n\nset output '").concat(latencyHistogramFile, "'\nset title 'Latency (in ms)'\nset yrange [0:").concat(latencyMax, "]\nplot '/tmp/latency.dat' using 2:xtic(1) title 'Latency'\n"); +} +function writeGnuplotScript(script) { + var gnuplotScriptFile = '/tmp/gnuplot_script.gp'; + (0, fileUtils_1.writeDataFile)(gnuplotScriptFile, script); +} diff --git a/analyze/gnuplotGenerator.ts b/analyze/gnuplotGenerator.ts new file mode 100644 index 00000000..65a9f76e --- /dev/null +++ b/analyze/gnuplotGenerator.ts @@ -0,0 +1,30 @@ +import { writeDataFile } from './fileUtils'; + +export function generateGnuplotScript(whichBench: number, reqSecMax: number, latencyMax: number): string { + const reqSecHistogramFile = `req_sec_histogram${whichBench}.png`; + const latencyHistogramFile = `latency_histogram${whichBench}.png`; + + return ` +set term pngcairo size 1280,720 enhanced font 'Courier,12' +set output '${reqSecHistogramFile}' +set style data histograms +set style histogram cluster gap 1 +set style fill solid border -1 +set xtics rotate by -45 +set boxwidth 0.9 +set title 'Requests/Sec' +set yrange [0:${reqSecMax}] +set key outside right top +plot '/tmp/reqSec.dat' using 2:xtic(1) title 'Req/Sec' + +set output '${latencyHistogramFile}' +set title 'Latency (in ms)' +set yrange [0:${latencyMax}] +plot '/tmp/latency.dat' using 2:xtic(1) title 'Latency' +`; +} + +export function writeGnuplotScript(script: string): void { + const gnuplotScriptFile = '/tmp/gnuplot_script.gp'; + writeDataFile(gnuplotScriptFile, script); +} diff --git a/analyze/parser.js b/analyze/parser.js new file mode 100644 index 00000000..f2e571da --- /dev/null +++ b/analyze/parser.js @@ -0,0 +1,49 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.parseMetric = parseMetric; +exports.calculateAverage = calculateAverage; +exports.parseServerMetrics = parseServerMetrics; +function parseMetric(input, metric) { + var lines = input.split('\n'); + var metricLine; + if (metric === "Latency") { + metricLine = lines.find(function (line) { return line.trim().startsWith("Latency"); }); + } + else if (metric === "Requests/sec") { + metricLine = lines.find(function (line) { return line.trim().startsWith("Requests/sec"); }); + } + if (!metricLine) + return null; + var match = metricLine.match(/([\d.]+)/); + return match ? parseFloat(match[1]) : null; +} +function calculateAverage(values) { + if (values.length === 0) + return 0; + var sum = values.reduce(function (a, b) { return a + b; }, 0); + return sum / values.length; +} +function parseServerMetrics(servers, inputs) { + var serverMetrics = {}; + servers.forEach(function (server, idx) { + var startIdx = idx * 3; + var reqSecVals = []; + var latencyVals = []; + for (var j = 0; j < 3; j++) { + var inputIdx = startIdx + j; + if (inputIdx < inputs.length) { + var reqSec = parseMetric(inputs[inputIdx], "Requests/sec"); + var latency = parseMetric(inputs[inputIdx], "Latency"); + if (reqSec !== null) + reqSecVals.push(reqSec); + if (latency !== null) + latencyVals.push(latency); + } + } + serverMetrics[server] = { + reqSec: calculateAverage(reqSecVals), + latency: calculateAverage(latencyVals) + }; + }); + return serverMetrics; +} diff --git a/analyze/parser.ts b/analyze/parser.ts new file mode 100644 index 00000000..b15e47c0 --- /dev/null +++ b/analyze/parser.ts @@ -0,0 +1,48 @@ +import { ServerMetrics } from './types'; + +export function parseMetric(input: string, metric: string): number | null { + const lines = input.split('\n'); + let metricLine: string | undefined; + + if (metric === "Latency") { + metricLine = lines.find(line => line.trim().startsWith("Latency")); + } else if (metric === "Requests/sec") { + metricLine = lines.find(line => line.trim().startsWith("Requests/sec")); + } + + if (!metricLine) return null; + + const match = metricLine.match(/([\d.]+)/); + return match ? parseFloat(match[1]) : null; +} + +export function calculateAverage(values: number[]): number { + if (values.length === 0) return 0; + const sum = values.reduce((a, b) => a + b, 0); + return sum / values.length; +} + +export function parseServerMetrics(servers: string[], inputs: string[]): Record { + const serverMetrics: Record = {}; + + servers.forEach((server, idx) => { + const startIdx = idx * 3; + const reqSecVals: number[] = []; + const latencyVals: number[] = []; + for (let j = 0; j < 3; j++) { + const inputIdx = startIdx + j; + if (inputIdx < inputs.length) { + const reqSec = parseMetric(inputs[inputIdx], "Requests/sec"); + const latency = parseMetric(inputs[inputIdx], "Latency"); + if (reqSec !== null) reqSecVals.push(reqSec); + if (latency !== null) latencyVals.push(latency); + } + } + serverMetrics[server] = { + reqSec: calculateAverage(reqSecVals), + latency: calculateAverage(latencyVals) + }; + }); + + return serverMetrics; +} diff --git a/analyze/resultsFormatter.js b/analyze/resultsFormatter.js new file mode 100644 index 00000000..f35c5763 --- /dev/null +++ b/analyze/resultsFormatter.js @@ -0,0 +1,70 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.formatResults = formatResults; +exports.writeResults = writeResults; +var fs = require("fs"); +function formatResults(serverMetrics, formattedServerNames, whichBench) { + var sortedServers = Object.keys(serverMetrics).sort(function (a, b) { return serverMetrics[b].reqSec - serverMetrics[a].reqSec; }); + var lastServer = sortedServers[sortedServers.length - 1]; + var lastServerReqSecs = serverMetrics[lastServer].reqSec; + var resultsTable = ""; + if (whichBench === 1) { + resultsTable += "\n| ".concat(whichBench, " | `{ posts { id userId title user { id name email }}}` |"); + } + else if (whichBench === 2) { + resultsTable += "\n| ".concat(whichBench, " | `{ posts { title }}` |"); + } + else if (whichBench === 3) { + resultsTable += "\n| ".concat(whichBench, " | `{ greet }` |"); + } + sortedServers.forEach(function (server) { + var formattedReqSecs = serverMetrics[server].reqSec.toLocaleString(undefined, { + minimumFractionDigits: 2, + maximumFractionDigits: 2, + }); + var formattedLatencies = serverMetrics[server].latency.toLocaleString(undefined, { + minimumFractionDigits: 2, + maximumFractionDigits: 2, + }); + var relativePerformance = (serverMetrics[server].reqSec / lastServerReqSecs).toFixed(2); + resultsTable += "\n|| [".concat(formattedServerNames[server], "] | `").concat(formattedReqSecs, "` | `").concat(formattedLatencies, "` | `").concat(relativePerformance, "x` |"); + }); + return resultsTable; +} +function writeResults(resultsTable, whichBench) { + var resultsFile = "results.md"; + try { + if (!fs.existsSync(resultsFile) || fs.readFileSync(resultsFile, 'utf8').trim() === '') { + fs.writeFileSync(resultsFile, "\n\n| Query | Server | Requests/sec | Latency (ms) | Relative |\n|-------:|--------:|--------------:|--------------:|---------:|"); + } + fs.appendFileSync(resultsFile, resultsTable); + if (whichBench === 3) { + fs.appendFileSync(resultsFile, "\n\n"); + updateReadme(resultsFile); + } + } + catch (error) { + console.error("Error writing results: ".concat(error.message)); + } +} +function updateReadme(resultsFile) { + try { + var finalResults = fs + .readFileSync(resultsFile, "utf-8") + .replace(/\\/g, ''); // Remove backslashes + var readmePath = "README.md"; + var readmeContent = fs.readFileSync(readmePath, "utf-8"); + var performanceResultsRegex = /[\s\S]*/; + if (performanceResultsRegex.test(readmeContent)) { + readmeContent = readmeContent.replace(performanceResultsRegex, finalResults); + } + else { + readmeContent += "\n".concat(finalResults); + } + fs.writeFileSync(readmePath, readmeContent); + console.log("README.md updated successfully"); + } + catch (error) { + console.error("Error updating README: ".concat(error.message)); + } +} diff --git a/analyze/resultsFormatter.ts b/analyze/resultsFormatter.ts new file mode 100644 index 00000000..2169c6f9 --- /dev/null +++ b/analyze/resultsFormatter.ts @@ -0,0 +1,83 @@ +import * as fs from 'fs'; +import { FormattedServerNames, ServerMetrics } from './types'; + +export function formatResults(serverMetrics: Record, formattedServerNames: FormattedServerNames, whichBench: number): string { + const sortedServers = Object.keys(serverMetrics).sort( + (a, b) => serverMetrics[b].reqSec - serverMetrics[a].reqSec + ); + const lastServer = sortedServers[sortedServers.length - 1]; + const lastServerReqSecs = serverMetrics[lastServer].reqSec; + + let resultsTable = ""; + + if (whichBench === 1) { + resultsTable += `\n| ${whichBench} | \`{ posts { id userId title user { id name email }}}\` |`; + } else if (whichBench === 2) { + resultsTable += `\n| ${whichBench} | \`{ posts { title }}\` |`; + } else if (whichBench === 3) { + resultsTable += `\n| ${whichBench} | \`{ greet }\` |`; + } + + sortedServers.forEach((server) => { + const formattedReqSecs = serverMetrics[server].reqSec.toLocaleString(undefined, { + minimumFractionDigits: 2, + maximumFractionDigits: 2, + }); + const formattedLatencies = serverMetrics[server].latency.toLocaleString(undefined, { + minimumFractionDigits: 2, + maximumFractionDigits: 2, + }); + const relativePerformance = (serverMetrics[server].reqSec / lastServerReqSecs).toFixed(2); + + resultsTable += `\n|| [${formattedServerNames[server]}] | \`${formattedReqSecs}\` | \`${formattedLatencies}\` | \`${relativePerformance}x\` |`; + }); + + return resultsTable; +} + +export function writeResults(resultsTable: string, whichBench: number): void { + const resultsFile = "results.md"; + + try { + if (!fs.existsSync(resultsFile) || fs.readFileSync(resultsFile, 'utf8').trim() === '') { + fs.writeFileSync(resultsFile, ` + +| Query | Server | Requests/sec | Latency (ms) | Relative | +|-------:|--------:|--------------:|--------------:|---------:|`); + } + + fs.appendFileSync(resultsFile, resultsTable); + + if (whichBench === 3) { + fs.appendFileSync(resultsFile, "\n\n"); + updateReadme(resultsFile); + } + } catch (error) { + console.error(`Error writing results: ${(error as Error).message}`); + } +} + +function updateReadme(resultsFile: string): void { + try { + const finalResults = fs + .readFileSync(resultsFile, "utf-8") + .replace(/\\/g, ''); // Remove backslashes + + const readmePath = "README.md"; + let readmeContent = fs.readFileSync(readmePath, "utf-8"); + const performanceResultsRegex = + /[\s\S]*/; + if (performanceResultsRegex.test(readmeContent)) { + readmeContent = readmeContent.replace( + performanceResultsRegex, + finalResults + ); + } else { + readmeContent += `\n${finalResults}`; + } + fs.writeFileSync(readmePath, readmeContent); + console.log("README.md updated successfully"); + } catch (error) { + console.error(`Error updating README: ${(error as Error).message}`); + } +} diff --git a/analyze/types.js b/analyze/types.js new file mode 100644 index 00000000..c8ad2e54 --- /dev/null +++ b/analyze/types.js @@ -0,0 +1,2 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); diff --git a/analyze/types.ts b/analyze/types.ts new file mode 100644 index 00000000..e8672961 --- /dev/null +++ b/analyze/types.ts @@ -0,0 +1,8 @@ +export interface ServerMetrics { + reqSec: number; + latency: number; +} + +export interface FormattedServerNames { + [key: string]: string; +} diff --git a/run_analyze_script.sh b/run_analyze_script.sh index dd284135..1a84356f 100755 --- a/run_analyze_script.sh +++ b/run_analyze_script.sh @@ -12,8 +12,9 @@ services=("apollo" "caliban" "netflixdgs" "gqlgen" "tailcall" "async_graphql" "h for bench in 1 2 3; do echo "Processing files for bench${bench}:" - # Construct the command for each benchmark - cmd="node analyze.js" + tsc analyze.ts + # Construct the command for each benchmark + cmd="node analyze.js" # Loop through each service for service in "${services[@]}"; do @@ -33,4 +34,4 @@ for bench in 1 2 3; do # Execute the command echo "Executing: $cmd" eval $cmd -done \ No newline at end of file +done diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 00000000..9db8d78f --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,9 @@ +{ + "compilerOptions": { + "target": "ES2018", + "module": "CommonJS", + "rootDir": "./", + "strict": true, + "esModuleInterop": true + } +} From 2f91046d0e059f5d18e7d6445ef423821e0ac629 Mon Sep 17 00:00:00 2001 From: daveads Date: Sun, 28 Jul 2024 13:02:24 +0100 Subject: [PATCH 6/8] clean up --- analyze/config.js | 14 -------- analyze/dataFileWriter.js | 10 ------ analyze/fileUtils.js | 48 ------------------------- analyze/gnuplotGenerator.js | 14 -------- analyze/parser.js | 49 -------------------------- analyze/resultsFormatter.js | 70 ------------------------------------- analyze/types.js | 2 -- 7 files changed, 207 deletions(-) delete mode 100644 analyze/config.js delete mode 100644 analyze/dataFileWriter.js delete mode 100644 analyze/fileUtils.js delete mode 100644 analyze/gnuplotGenerator.js delete mode 100644 analyze/parser.js delete mode 100644 analyze/resultsFormatter.js delete mode 100644 analyze/types.js diff --git a/analyze/config.js b/analyze/config.js deleted file mode 100644 index 393684fa..00000000 --- a/analyze/config.js +++ /dev/null @@ -1,14 +0,0 @@ -"use strict"; -Object.defineProperty(exports, "__esModule", { value: true }); -exports.servers = exports.formattedServerNames = void 0; -exports.formattedServerNames = { - tailcall: "Tailcall", - gqlgen: "Gqlgen", - apollo: "Apollo GraphQL", - netflixdgs: "Netflix DGS", - caliban: "Caliban", - async_graphql: "async-graphql", - hasura: "Hasura", - graphql_jit: "GraphQL JIT", -}; -exports.servers = ["apollo", "caliban", "netflixdgs", "gqlgen", "tailcall", "async_graphql", "hasura", "graphql_jit"]; diff --git a/analyze/dataFileWriter.js b/analyze/dataFileWriter.js deleted file mode 100644 index 67653355..00000000 --- a/analyze/dataFileWriter.js +++ /dev/null @@ -1,10 +0,0 @@ -"use strict"; -Object.defineProperty(exports, "__esModule", { value: true }); -exports.writeMetricsDataFiles = writeMetricsDataFiles; -var fileUtils_1 = require("./fileUtils"); -function writeMetricsDataFiles(serverMetrics, servers) { - var reqSecData = "/tmp/reqSec.dat"; - var latencyData = "/tmp/latency.dat"; - (0, fileUtils_1.writeDataFile)(reqSecData, "Server Value\n" + servers.map(function (server) { return "".concat(server, " ").concat(serverMetrics[server].reqSec); }).join('\n')); - (0, fileUtils_1.writeDataFile)(latencyData, "Server Value\n" + servers.map(function (server) { return "".concat(server, " ").concat(serverMetrics[server].latency); }).join('\n')); -} diff --git a/analyze/fileUtils.js b/analyze/fileUtils.js deleted file mode 100644 index a7459d1f..00000000 --- a/analyze/fileUtils.js +++ /dev/null @@ -1,48 +0,0 @@ -"use strict"; -Object.defineProperty(exports, "__esModule", { value: true }); -exports.writeDataFile = writeDataFile; -exports.readFileContent = readFileContent; -exports.moveFile = moveFile; -exports.createDirectoryIfNotExists = createDirectoryIfNotExists; -var fs = require("fs"); -function writeDataFile(filename, data) { - try { - fs.writeFileSync(filename, data); - } - catch (error) { - console.error("Error writing data file ".concat(filename, ": ").concat(error.message)); - } -} -function readFileContent(filename) { - try { - return fs.readFileSync(filename, 'utf-8'); - } - catch (error) { - console.error("Error reading file ".concat(filename, ": ").concat(error.message)); - return ''; - } -} -function moveFile(source, destination) { - try { - if (fs.existsSync(source)) { - fs.renameSync(source, destination); - console.log("Moved ".concat(source, " to ").concat(destination)); - } - else { - console.log("Source file ".concat(source, " does not exist")); - } - } - catch (error) { - console.error("Error moving file ".concat(source, ": ").concat(error.message)); - } -} -function createDirectoryIfNotExists(dir) { - if (!fs.existsSync(dir)) { - try { - fs.mkdirSync(dir); - } - catch (error) { - console.error("Error creating directory: ".concat(error.message)); - } - } -} diff --git a/analyze/gnuplotGenerator.js b/analyze/gnuplotGenerator.js deleted file mode 100644 index ba7296da..00000000 --- a/analyze/gnuplotGenerator.js +++ /dev/null @@ -1,14 +0,0 @@ -"use strict"; -Object.defineProperty(exports, "__esModule", { value: true }); -exports.generateGnuplotScript = generateGnuplotScript; -exports.writeGnuplotScript = writeGnuplotScript; -var fileUtils_1 = require("./fileUtils"); -function generateGnuplotScript(whichBench, reqSecMax, latencyMax) { - var reqSecHistogramFile = "req_sec_histogram".concat(whichBench, ".png"); - var latencyHistogramFile = "latency_histogram".concat(whichBench, ".png"); - return "\nset term pngcairo size 1280,720 enhanced font 'Courier,12'\nset output '".concat(reqSecHistogramFile, "'\nset style data histograms\nset style histogram cluster gap 1\nset style fill solid border -1\nset xtics rotate by -45\nset boxwidth 0.9\nset title 'Requests/Sec'\nset yrange [0:").concat(reqSecMax, "]\nset key outside right top\nplot '/tmp/reqSec.dat' using 2:xtic(1) title 'Req/Sec'\n\nset output '").concat(latencyHistogramFile, "'\nset title 'Latency (in ms)'\nset yrange [0:").concat(latencyMax, "]\nplot '/tmp/latency.dat' using 2:xtic(1) title 'Latency'\n"); -} -function writeGnuplotScript(script) { - var gnuplotScriptFile = '/tmp/gnuplot_script.gp'; - (0, fileUtils_1.writeDataFile)(gnuplotScriptFile, script); -} diff --git a/analyze/parser.js b/analyze/parser.js deleted file mode 100644 index f2e571da..00000000 --- a/analyze/parser.js +++ /dev/null @@ -1,49 +0,0 @@ -"use strict"; -Object.defineProperty(exports, "__esModule", { value: true }); -exports.parseMetric = parseMetric; -exports.calculateAverage = calculateAverage; -exports.parseServerMetrics = parseServerMetrics; -function parseMetric(input, metric) { - var lines = input.split('\n'); - var metricLine; - if (metric === "Latency") { - metricLine = lines.find(function (line) { return line.trim().startsWith("Latency"); }); - } - else if (metric === "Requests/sec") { - metricLine = lines.find(function (line) { return line.trim().startsWith("Requests/sec"); }); - } - if (!metricLine) - return null; - var match = metricLine.match(/([\d.]+)/); - return match ? parseFloat(match[1]) : null; -} -function calculateAverage(values) { - if (values.length === 0) - return 0; - var sum = values.reduce(function (a, b) { return a + b; }, 0); - return sum / values.length; -} -function parseServerMetrics(servers, inputs) { - var serverMetrics = {}; - servers.forEach(function (server, idx) { - var startIdx = idx * 3; - var reqSecVals = []; - var latencyVals = []; - for (var j = 0; j < 3; j++) { - var inputIdx = startIdx + j; - if (inputIdx < inputs.length) { - var reqSec = parseMetric(inputs[inputIdx], "Requests/sec"); - var latency = parseMetric(inputs[inputIdx], "Latency"); - if (reqSec !== null) - reqSecVals.push(reqSec); - if (latency !== null) - latencyVals.push(latency); - } - } - serverMetrics[server] = { - reqSec: calculateAverage(reqSecVals), - latency: calculateAverage(latencyVals) - }; - }); - return serverMetrics; -} diff --git a/analyze/resultsFormatter.js b/analyze/resultsFormatter.js deleted file mode 100644 index f35c5763..00000000 --- a/analyze/resultsFormatter.js +++ /dev/null @@ -1,70 +0,0 @@ -"use strict"; -Object.defineProperty(exports, "__esModule", { value: true }); -exports.formatResults = formatResults; -exports.writeResults = writeResults; -var fs = require("fs"); -function formatResults(serverMetrics, formattedServerNames, whichBench) { - var sortedServers = Object.keys(serverMetrics).sort(function (a, b) { return serverMetrics[b].reqSec - serverMetrics[a].reqSec; }); - var lastServer = sortedServers[sortedServers.length - 1]; - var lastServerReqSecs = serverMetrics[lastServer].reqSec; - var resultsTable = ""; - if (whichBench === 1) { - resultsTable += "\n| ".concat(whichBench, " | `{ posts { id userId title user { id name email }}}` |"); - } - else if (whichBench === 2) { - resultsTable += "\n| ".concat(whichBench, " | `{ posts { title }}` |"); - } - else if (whichBench === 3) { - resultsTable += "\n| ".concat(whichBench, " | `{ greet }` |"); - } - sortedServers.forEach(function (server) { - var formattedReqSecs = serverMetrics[server].reqSec.toLocaleString(undefined, { - minimumFractionDigits: 2, - maximumFractionDigits: 2, - }); - var formattedLatencies = serverMetrics[server].latency.toLocaleString(undefined, { - minimumFractionDigits: 2, - maximumFractionDigits: 2, - }); - var relativePerformance = (serverMetrics[server].reqSec / lastServerReqSecs).toFixed(2); - resultsTable += "\n|| [".concat(formattedServerNames[server], "] | `").concat(formattedReqSecs, "` | `").concat(formattedLatencies, "` | `").concat(relativePerformance, "x` |"); - }); - return resultsTable; -} -function writeResults(resultsTable, whichBench) { - var resultsFile = "results.md"; - try { - if (!fs.existsSync(resultsFile) || fs.readFileSync(resultsFile, 'utf8').trim() === '') { - fs.writeFileSync(resultsFile, "\n\n| Query | Server | Requests/sec | Latency (ms) | Relative |\n|-------:|--------:|--------------:|--------------:|---------:|"); - } - fs.appendFileSync(resultsFile, resultsTable); - if (whichBench === 3) { - fs.appendFileSync(resultsFile, "\n\n"); - updateReadme(resultsFile); - } - } - catch (error) { - console.error("Error writing results: ".concat(error.message)); - } -} -function updateReadme(resultsFile) { - try { - var finalResults = fs - .readFileSync(resultsFile, "utf-8") - .replace(/\\/g, ''); // Remove backslashes - var readmePath = "README.md"; - var readmeContent = fs.readFileSync(readmePath, "utf-8"); - var performanceResultsRegex = /[\s\S]*/; - if (performanceResultsRegex.test(readmeContent)) { - readmeContent = readmeContent.replace(performanceResultsRegex, finalResults); - } - else { - readmeContent += "\n".concat(finalResults); - } - fs.writeFileSync(readmePath, readmeContent); - console.log("README.md updated successfully"); - } - catch (error) { - console.error("Error updating README: ".concat(error.message)); - } -} diff --git a/analyze/types.js b/analyze/types.js deleted file mode 100644 index c8ad2e54..00000000 --- a/analyze/types.js +++ /dev/null @@ -1,2 +0,0 @@ -"use strict"; -Object.defineProperty(exports, "__esModule", { value: true }); From 8b08d3cd356aafc66cfe07f3df344bebf5a451ab Mon Sep 17 00:00:00 2001 From: daveads Date: Sun, 28 Jul 2024 15:33:49 +0100 Subject: [PATCH 7/8] done --- package.json | 5 +++++ run_analyze_script.sh | 1 + 2 files changed, 6 insertions(+) create mode 100644 package.json diff --git a/package.json b/package.json new file mode 100644 index 00000000..85d4a4e3 --- /dev/null +++ b/package.json @@ -0,0 +1,5 @@ +{ + "devDependencies": { + "@types/node": "^22.0.0" + } +} diff --git a/run_analyze_script.sh b/run_analyze_script.sh index 1a84356f..a081e1cb 100755 --- a/run_analyze_script.sh +++ b/run_analyze_script.sh @@ -2,6 +2,7 @@ # Update and install gnuplot sudo apt-get update && sudo apt-get install -y gnuplot +npm install # Remove existing results file rm -f results.md From f06104859799c77151e4fcc0533156dce07e8fa6 Mon Sep 17 00:00:00 2001 From: daveads Date: Tue, 30 Jul 2024 04:58:41 +0100 Subject: [PATCH 8/8] js files --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 1ecfb42c..25ffbb84 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,6 @@ graphql/gqlgen/main tailcall-src metals.* + +analyze/* +analyze.js