From f12abde0668aa657c1cf50bd74cb74d1416d9f9d Mon Sep 17 00:00:00 2001 From: daveads Date: Thu, 18 Jul 2024 23:09:20 +0100 Subject: [PATCH] Port analyze.sh to analyze.js - Converted Bash script to JavaScript for Node.js environment - Maintained core functionality of metric extraction and analysis - Utilized child_process to execute shell commands for file parsing - Implemented Gnuplot execution via Node.js - Ensured compatibility with existing benchmark result files --- analyze.js | 198 ++++++++++++++++++++++++++++++++++++++++++++++ analyze.sh | 164 -------------------------------------- run_benchmarks.sh | 6 +- 3 files changed, 201 insertions(+), 167 deletions(-) create mode 100644 analyze.js delete mode 100755 analyze.sh diff --git a/analyze.js b/analyze.js new file mode 100644 index 00000000..7bc07c96 --- /dev/null +++ b/analyze.js @@ -0,0 +1,198 @@ +#!/usr/bin/env node + +const { execSync } = require('child_process'); +const fs = require('fs'); +const path = require('path'); + +function extractMetric(file, metric) { + try { + const command = `grep "${metric}" "${file}" | awk '{print $2}' | sed 's/ms//'`; + const result = execSync(command, { encoding: 'utf-8' }).trim(); + return result; + } catch (error) { + console.error(`Error extracting metric from ${file}: ${error.message}`); + return null; + } +} + +function average(values) { + const sum = values.reduce((a, b) => parseFloat(a) + parseFloat(b), 0); + return sum / values.length; +} + +const formattedServerNames = { + tailcall: "Tailcall", + gqlgen: "Gqlgen", + apollo: "Apollo GraphQL", + netflixdgs: "Netflix DGS", + caliban: "Caliban", + async_graphql: "async-graphql", + hasura: "Hasura", + graphql_jit: "GraphQL JIT", +}; + +const servers = ["apollo", "caliban", "netflixdgs", "gqlgen", "tailcall", "async_graphql", "hasura", "graphql_jit"]; +const resultFiles = process.argv.slice(2); +const avgReqSecs = {}; +const avgLatencies = {}; + +servers.forEach((server, idx) => { + const startIdx = idx * 3; + const reqSecVals = []; + const latencyVals = []; + for (let j = 0; j < 3; j++) { + const fileIdx = startIdx + j; + const reqSec = extractMetric(resultFiles[fileIdx], "Requests/sec"); + const latency = extractMetric(resultFiles[fileIdx], "Latency"); + if (reqSec !== null) reqSecVals.push(reqSec); + if (latency !== null) latencyVals.push(latency); + } + avgReqSecs[server] = average(reqSecVals); + avgLatencies[server] = average(latencyVals); +}); + +const reqSecData = "/tmp/reqSec.dat"; +const latencyData = "/tmp/latency.dat"; + +fs.writeFileSync(reqSecData, "Server Value\n" + servers.map(server => `${server} ${avgReqSecs[server]}`).join('\n')); +fs.writeFileSync(latencyData, "Server Value\n" + servers.map(server => `${server} ${avgLatencies[server]}`).join('\n')); + +let whichBench = 1; +if (resultFiles[0].startsWith("bench2")) { + whichBench = 2; +} else if (resultFiles[0].startsWith("bench3")) { + whichBench = 3; +} + +const reqSecHistogramFile = `req_sec_histogram${whichBench}.png`; +const latencyHistogramFile = `latency_histogram${whichBench}.png`; + +function getMaxValue(data) { + return Math.max(...data.split('\n').slice(1).map(line => parseFloat(line.split(' ')[1]))); +} + +const reqSecMax = getMaxValue(fs.readFileSync(reqSecData, 'utf-8')) * 1.2; +const latencyMax = getMaxValue(fs.readFileSync(latencyData, 'utf-8')) * 1.2; + +const gnuplotScript = ` +set term pngcairo size 1280,720 enhanced font 'Courier,12' +set output '${reqSecHistogramFile}' +set style data histograms +set style histogram cluster gap 1 +set style fill solid border -1 +set xtics rotate by -45 +set boxwidth 0.9 +set title 'Requests/Sec' +set yrange [0:${reqSecMax}] +set key outside right top +plot '${reqSecData}' using 2:xtic(1) title 'Req/Sec' + +set output '${latencyHistogramFile}' +set title 'Latency (in ms)' +set yrange [0:${latencyMax}] +plot '${latencyData}' using 2:xtic(1) title 'Latency' +`; + +const gnuplotScriptFile = '/tmp/gnuplot_script.gp'; +fs.writeFileSync(gnuplotScriptFile, gnuplotScript); + +try { + execSync(`gnuplot ${gnuplotScriptFile}`, { stdio: 'inherit' }); + console.log('Gnuplot executed successfully'); +} catch (error) { + console.error('Error executing gnuplot:', error.message); + process.exit(1); +} + +const assetsDir = path.join(__dirname, "assets"); +if (!fs.existsSync(assetsDir)) { + fs.mkdirSync(assetsDir); +} + +function moveFile(source, destination) { + try { + if (fs.existsSync(source)) { + fs.renameSync(source, destination); + console.log(`Moved ${source} to ${destination}`); + } else { + console.log(`Source file ${source} does not exist`); + } + } catch (error) { + console.error(`Error moving file ${source}: ${error.message}`); + } +} + +moveFile(reqSecHistogramFile, path.join(assetsDir, reqSecHistogramFile)); +moveFile(latencyHistogramFile, path.join(assetsDir, latencyHistogramFile)); + +const serverRPS = {}; +servers.forEach((server) => { + serverRPS[server] = avgReqSecs[server]; +}); + +const sortedServers = Object.keys(serverRPS).sort( + (a, b) => serverRPS[b] - serverRPS[a] +); +const lastServer = sortedServers[sortedServers.length - 1]; +const lastServerReqSecs = avgReqSecs[lastServer]; + +const resultsFile = "results.md"; + +if (!fs.existsSync(resultsFile) || fs.readFileSync(resultsFile, 'utf8').trim() === '') { + fs.writeFileSync(resultsFile, ` + +| Query | Server | Requests/sec | Latency (ms) | Relative | +|-------:|--------:|--------------:|--------------:|---------:|`); +} + +let resultsTable = ""; + +if (whichBench === 1) { + resultsTable += `\n| ${whichBench} | \`{ posts { id userId title user { id name email }}}\` |`; +} else if (whichBench === 2) { + resultsTable += `\n| ${whichBench} | \`{ posts { title }}\` |`; +} else if (whichBench === 3) { + resultsTable += `\n| ${whichBench} | \`{ greet }\` |`; +} + +sortedServers.forEach((server) => { + const formattedReqSecs = avgReqSecs[server].toLocaleString(undefined, { + minimumFractionDigits: 2, + maximumFractionDigits: 2, + }); + const formattedLatencies = avgLatencies[server].toLocaleString(undefined, { + minimumFractionDigits: 2, + maximumFractionDigits: 2, + }); + const relativePerformance = (avgReqSecs[server] / lastServerReqSecs).toFixed(2); + + resultsTable += `\n|| [${formattedServerNames[server]}] | \`${formattedReqSecs}\` | \`${formattedLatencies}\` | \`${relativePerformance}x\` |`; +}); + +fs.appendFileSync(resultsFile, resultsTable + "\n"); + +if (whichBench === 3) { + fs.appendFileSync(resultsFile, "\n"); + + const finalResults = fs + .readFileSync(resultsFile, "utf-8") + .replace(/(\r\n|\n|\r)/gm, "\\n"); + + const readmePath = "README.md"; + let readmeContent = fs.readFileSync(readmePath, "utf-8"); + const performanceResultsRegex = + /[\s\S]*/; + if (performanceResultsRegex.test(readmeContent)) { + readmeContent = readmeContent.replace( + performanceResultsRegex, + finalResults + ); + } else { + readmeContent += `\n${finalResults}`; + } + fs.writeFileSync(readmePath, readmeContent); +} + +resultFiles.forEach((file) => { + fs.unlinkSync(file); +}); diff --git a/analyze.sh b/analyze.sh deleted file mode 100755 index 17bbbce1..00000000 --- a/analyze.sh +++ /dev/null @@ -1,164 +0,0 @@ -#!/bin/bash - -function extractMetric() { - local file="$1" - local metric="$2" - grep "$metric" "$file" | awk '{print $2}' | sed 's/ms//' -} - -function average() { - echo "$@" | awk '{for(i=1;i<=NF;i++) s+=$i; print s/NF}' -} - -declare -A formattedServerNames -formattedServerNames=( - ["tailcall"]="Tailcall" - ["gqlgen"]="Gqlgen" - ["apollo"]="Apollo GraphQL" - ["netflixdgs"]="Netflix DGS" - ["caliban"]="Caliban" - ["async_graphql"]="async-graphql" - ["hasura"]="Hasura" - ["graphql_jit"]="GraphQL JIT" -) - -servers=("apollo" "caliban" "netflixdgs" "gqlgen" "tailcall" "async_graphql" "hasura" "graphql_jit") -resultFiles=("$@") -declare -A avgReqSecs -declare -A avgLatencies - -# Extract metrics and calculate averages -for idx in "${!servers[@]}"; do - startIdx=$((idx * 3)) - reqSecVals=() - latencyVals=() - for j in 0 1 2; do - fileIdx=$((startIdx + j)) - reqSecVals+=($(extractMetric "${resultFiles[$fileIdx]}" "Requests/sec")) - latencyVals+=($(extractMetric "${resultFiles[$fileIdx]}" "Latency")) - done - avgReqSecs[${servers[$idx]}]=$(average "${reqSecVals[@]}") - avgLatencies[${servers[$idx]}]=$(average "${latencyVals[@]}") -done - -# Generating data files for gnuplot -reqSecData="/tmp/reqSec.dat" -latencyData="/tmp/latency.dat" - -echo "Server Value" >"$reqSecData" -for server in "${servers[@]}"; do - echo "$server ${avgReqSecs[$server]}" >>"$reqSecData" -done - -echo "Server Value" >"$latencyData" -for server in "${servers[@]}"; do - echo "$server ${avgLatencies[$server]}" >>"$latencyData" -done - -whichBench=1 -if [[ $1 == bench2* ]]; then - whichBench=2 -elif [[ $1 == bench3* ]]; then - whichBench=3 -fi - -reqSecHistogramFile="req_sec_histogram${whichBench}.png" -latencyHistogramFile="latency_histogram${whichBench}.png" - -# Plotting using gnuplot -gnuplot <<-EOF - set term pngcairo size 1280,720 enhanced font "Courier,12" - set output "$reqSecHistogramFile" - set style data histograms - set style histogram cluster gap 1 - set style fill solid border -1 - set xtics rotate by -45 - set boxwidth 0.9 - set title "Requests/Sec" - stats "$reqSecData" using 2 nooutput - set yrange [0:STATS_max*1.2] - set key outside right top - plot "$reqSecData" using 2:xtic(1) title "Req/Sec" - - set output "$latencyHistogramFile" - set title "Latency (in ms)" - stats "$latencyData" using 2 nooutput - set yrange [0:STATS_max*1.2] - plot "$latencyData" using 2:xtic(1) title "Latency" -EOF - -# Move PNGs to assets -mkdir -p assets -mv $reqSecHistogramFile assets/ -mv $latencyHistogramFile assets/ - -# Declare an associative array for server RPS -declare -A serverRPS - -# Populate the serverRPS array -for server in "${servers[@]}"; do - serverRPS[$server]=${avgReqSecs[$server]} -done - -# Get the servers sorted by RPS in descending order -IFS=$'\n' sortedServers=($(for server in "${!serverRPS[@]}"; do echo "$server ${serverRPS[$server]}"; done | sort -rn -k2 | cut -d' ' -f1)) - -echo "Sorted servers: ${sortedServers[@]}" -lastServer="${sortedServers[-1]}" -lastServerReqSecs=${avgReqSecs[$lastServer]} - -# Start building the resultsTable -if [[ $whichBench == 1 ]]; then - resultsTable="\n\n| Query | Server | Requests/sec | Latency (ms) | Relative |\n|-------:|--------:|--------------:|--------------:|---------:|\n| $whichBench | \`{ posts { id userId title user { id name email }}}\` |" -elif [[ $whichBench == 2 ]]; then - resultsTable="| $whichBench | \`{ posts { title }}\` |" -elif [[ $whichBench == 3 ]]; then - resultsTable="| $whichBench | \`{ greet }\` |" -fi - -# Build the resultsTable with sorted servers and formatted numbers -for server in "${sortedServers[@]}"; do - formattedReqSecs=$(printf "%.2f" ${avgReqSecs[$server]} | perl -pe 's/(?<=\d)(?=(\d{3})+(\.\d*)?$)/,/g') - formattedLatencies=$(printf "%.2f" ${avgLatencies[$server]} | perl -pe 's/(?<=\d)(?=(\d{3})+(\.\d*)?$)/,/g') - # Calculate the relative performance - relativePerformance=$(echo "${avgReqSecs[$server]} $lastServerReqSecs" | awk '{printf "%.2f", $1 / $2}') - - resultsTable+="\n|| [${formattedServerNames[$server]}] | \`${formattedReqSecs}\` | \`${formattedLatencies}\` | \`${relativePerformance}x\` |" -done - -if [[ $whichBench == 3 ]]; then - resultsTable+="\n\n" -fi - -echo "resultsTable: $resultsTable" - -# Print the results table in a new file -resultsFile="results.md" -echo -e $resultsTable >> $resultsFile - - -if [[ $whichBench == 3 ]]; then - finalResults=$(printf '%s\n' "$(cat $resultsFile)" | sed 's/$/\\n/'| tr -d '\n') - # Remove the last newline character - finalResults=${finalResults::-2} - - # Print the results as a table in the terminal - echo -e $finalResults | sed "s///;s///" - # Check if the markers are present - if grep -q "PERFORMANCE_RESULTS_START" README.md; then - # Replace the old results with the new results - sed -i "/PERFORMANCE_RESULTS_START/,/PERFORMANCE_RESULTS_END/c\\$finalResults" README.md - else - # Append the results at the end of the README.md file - echo -e "\n$finalResults" >> README.md - fi -fi - -# Move the generated images to the assets folder -mv $reqSecHistogramFile assets/ -mv $latencyHistogramFile assets/ - -# Delete the result TXT files -for file in "${resultFiles[@]}"; do - rm "$file" -done diff --git a/run_benchmarks.sh b/run_benchmarks.sh index 263e6f69..09d625e3 100755 --- a/run_benchmarks.sh +++ b/run_benchmarks.sh @@ -83,6 +83,6 @@ for service in "apollo_server" "caliban" "netflix_dgs" "gqlgen" "tailcall" "asyn fi done -bash analyze.sh "${bench1Results[@]}" -bash analyze.sh "${bench2Results[@]}" -bash analyze.sh "${bench3Results[@]}" +node analyze.js "${bench1Results[@]}" +node analyze.js "${bench2Results[@]}" +node analyze.js "${bench3Results[@]}"