diff --git a/analyze.js b/analyze.js deleted file mode 100644 index 78a40454..00000000 --- a/analyze.js +++ /dev/null @@ -1,242 +0,0 @@ -#!/usr/bin/env node -const { execSync } = require('child_process'); -const fs = require('fs'); -const path = require('path'); - -function extractMetric(file, metric) { - try { - const command = `grep "${metric}" "${file}" | awk '{print $2}' | sed 's/ms//'`; - const result = execSync(command, { encoding: 'utf-8' }).trim(); - return result; - } catch (error) { - console.error(`Error extracting metric from ${file}: ${error.message}`); - return null; - } -} - -function average(values) { - if (values.length === 0) return 0; - const sum = values.reduce((a, b) => parseFloat(a) + parseFloat(b), 0); - return sum / values.length; -} - -const formattedServerNames = { - tailcall: "Tailcall", - gqlgen: "Gqlgen", - apollo: "Apollo GraphQL", - netflixdgs: "Netflix DGS", - caliban: "Caliban", - async_graphql: "async-graphql", - hasura: "Hasura", - graphql_jit: "GraphQL JIT", -}; - -const servers = ["apollo", "caliban", "netflixdgs", "gqlgen", "tailcall", "async_graphql", "hasura", "graphql_jit"]; -const resultFiles = process.argv.slice(2); -const avgReqSecs = {}; -const avgLatencies = {}; - -servers.forEach((server, idx) => { - const startIdx = idx * 3; - const reqSecVals = []; - const latencyVals = []; - for (let j = 0; j < 3; j++) { - const fileIdx = startIdx + j; - if (fileIdx < resultFiles.length) { - const reqSec = extractMetric(resultFiles[fileIdx], "Requests/sec"); - const latency = extractMetric(resultFiles[fileIdx], "Latency"); - if (reqSec !== null) reqSecVals.push(reqSec); - if (latency !== null) latencyVals.push(latency); - } - } - avgReqSecs[server] = average(reqSecVals); - avgLatencies[server] = average(latencyVals); -}); - -const reqSecData = "/tmp/reqSec.dat"; -const latencyData = "/tmp/latency.dat"; - -try { - fs.writeFileSync(reqSecData, "Server Value\n" + servers.map(server => `${server} ${avgReqSecs[server]}`).join('\n')); - fs.writeFileSync(latencyData, "Server Value\n" + servers.map(server => `${server} ${avgLatencies[server]}`).join('\n')); -} catch (error) { - console.error(`Error writing data files: ${error.message}`); -} - -let whichBench = 1; -if (resultFiles.length > 0) { - if (resultFiles[0].startsWith("bench2")) { - whichBench = 2; - } else if (resultFiles[0].startsWith("bench3")) { - whichBench = 3; - } -} - -const reqSecHistogramFile = `req_sec_histogram${whichBench}.png`; -const latencyHistogramFile = `latency_histogram${whichBench}.png`; - -function getMaxValue(data) { - try { - return Math.max(...data.split('\n').slice(1).map(line => parseFloat(line.split(' ')[1]))); - } catch (error) { - console.error(`Error getting max value: ${error.message}`); - return 0; - } -} - -let reqSecMax, latencyMax; -try { - reqSecMax = getMaxValue(fs.readFileSync(reqSecData, 'utf-8')) * 1.2; - latencyMax = getMaxValue(fs.readFileSync(latencyData, 'utf-8')) * 1.2; -} catch (error) { - console.error(`Error reading data files: ${error.message}`); - reqSecMax = 0; - latencyMax = 0; -} - -const gnuplotScript = ` -set term pngcairo size 1280,720 enhanced font 'Courier,12' -set output '${reqSecHistogramFile}' -set style data histograms -set style histogram cluster gap 1 -set style fill solid border -1 -set xtics rotate by -45 -set boxwidth 0.9 -set title 'Requests/Sec' -set yrange [0:${reqSecMax}] -set key outside right top -plot '${reqSecData}' using 2:xtic(1) title 'Req/Sec' - -set output '${latencyHistogramFile}' -set title 'Latency (in ms)' -set yrange [0:${latencyMax}] -plot '${latencyData}' using 2:xtic(1) title 'Latency' -`; - -const gnuplotScriptFile = '/tmp/gnuplot_script.gp'; -try { - fs.writeFileSync(gnuplotScriptFile, gnuplotScript); -} catch (error) { - console.error(`Error writing gnuplot script: ${error.message}`); -} - -try { - execSync(`gnuplot ${gnuplotScriptFile}`, { stdio: 'inherit' }); - console.log('Gnuplot executed successfully'); -} catch (error) { - console.error('Error executing gnuplot:', error.message); -} - -const assetsDir = path.join(__dirname, "assets"); -if (!fs.existsSync(assetsDir)) { - try { - fs.mkdirSync(assetsDir); - } catch (error) { - console.error(`Error creating assets directory: ${error.message}`); - } -} - -function moveFile(source, destination) { - try { - if (fs.existsSync(source)) { - fs.renameSync(source, destination); - console.log(`Moved ${source} to ${destination}`); - } else { - console.log(`Source file ${source} does not exist`); - } - } catch (error) { - console.error(`Error moving file ${source}: ${error.message}`); - } -} - -moveFile(reqSecHistogramFile, path.join(assetsDir, reqSecHistogramFile)); -moveFile(latencyHistogramFile, path.join(assetsDir, latencyHistogramFile)); - -const serverRPS = {}; -servers.forEach((server) => { - serverRPS[server] = avgReqSecs[server]; -}); - -const sortedServers = Object.keys(serverRPS).sort( - (a, b) => serverRPS[b] - serverRPS[a] -); -const lastServer = sortedServers[sortedServers.length - 1]; -const lastServerReqSecs = avgReqSecs[lastServer]; - -const resultsFile = "results.md"; - -try { - if (!fs.existsSync(resultsFile) || fs.readFileSync(resultsFile, 'utf8').trim() === '') { - fs.writeFileSync(resultsFile, ` - -| Query | Server | Requests/sec | Latency (ms) | Relative | -|-------:|--------:|--------------:|--------------:|---------:|`); - } -} catch (error) { - console.error(`Error initializing results file: ${error.message}`); -} - -let resultsTable = ""; - -if (whichBench === 1) { - resultsTable += `\n| ${whichBench} | \`{ posts { id userId title user { id name email }}}\` |`; -} else if (whichBench === 2) { - resultsTable += `\n| ${whichBench} | \`{ posts { title }}\` |`; -} else if (whichBench === 3) { - resultsTable += `\n| ${whichBench} | \`{ greet }\` |`; -} - -sortedServers.forEach((server) => { - const formattedReqSecs = avgReqSecs[server].toLocaleString(undefined, { - minimumFractionDigits: 2, - maximumFractionDigits: 2, - }); - const formattedLatencies = avgLatencies[server].toLocaleString(undefined, { - minimumFractionDigits: 2, - maximumFractionDigits: 2, - }); - const relativePerformance = (avgReqSecs[server] / lastServerReqSecs).toFixed(2); - - resultsTable += `\n|| [${formattedServerNames[server]}] | \`${formattedReqSecs}\` | \`${formattedLatencies}\` | \`${relativePerformance}x\` |`; -}); - -try { - fs.appendFileSync(resultsFile, resultsTable); -} catch (error) { - console.error(`Error appending to results file: ${error.message}`); -} - -if (whichBench === 3) { - try { - fs.appendFileSync(resultsFile, "\n\n"); - - const finalResults = fs - .readFileSync(resultsFile, "utf-8") - .replace(/\\/g, ''); // Remove backslashes - - const readmePath = "README.md"; - let readmeContent = fs.readFileSync(readmePath, "utf-8"); - const performanceResultsRegex = - /[\s\S]*/; - if (performanceResultsRegex.test(readmeContent)) { - readmeContent = readmeContent.replace( - performanceResultsRegex, - finalResults - ); - } else { - readmeContent += `\n${finalResults}`; - } - fs.writeFileSync(readmePath, readmeContent); - console.log("README.md updated successfully"); - } catch (error) { - console.error(`Error updating README: ${error.message}`); - } -} - -resultFiles.forEach((file) => { - try { - fs.unlinkSync(file); - } catch (error) { - console.error(`Error deleting file ${file}: ${error.message}`); - } -}); \ No newline at end of file diff --git a/analyze.ts b/analyze.ts new file mode 100644 index 00000000..518719f9 --- /dev/null +++ b/analyze.ts @@ -0,0 +1,76 @@ +#!/usr/bin/env node +import { execSync } from 'child_process'; +import * as path from 'path'; +import * as fs from 'fs'; +import { servers, formattedServerNames } from './analyze/config'; +import { createDirectoryIfNotExists, moveFile, readFileContent } from './analyze/fileUtils'; +import { parseServerMetrics } from './analyze/parser'; +import { writeMetricsDataFiles } from './analyze/dataFileWriter'; +import { generateGnuplotScript, writeGnuplotScript } from './analyze/gnuplotGenerator'; +import { formatResults, writeResults } from './analyze/resultsFormatter'; + +const resultFiles: string[] = process.argv.slice(2); + +// Read content of result files +const resultContents: string[] = resultFiles.map(file => readFileContent(file)); + +const serverMetrics = parseServerMetrics(servers, resultContents); + +writeMetricsDataFiles(serverMetrics, servers); + +let whichBench = 1; +if (resultFiles.length > 0) { + if (resultFiles[0].startsWith("bench2")) { + whichBench = 2; + } else if (resultFiles[0].startsWith("bench3")) { + whichBench = 3; + } +} + +function getMaxValue(data: string): number { + try { + return Math.max(...data.split('\n') + .slice(1) + .map(line => { + const [, valueStr] = line.split(' '); + const value = parseFloat(valueStr); + if (isNaN(value)) { + throw new Error(`Invalid number in data: ${valueStr}`); + } + return value; + })); + } catch (error) { + console.error(`Error getting max value: ${(error as Error).message}`); + return 0; + } +} + +const reqSecMax = getMaxValue(readFileContent('/tmp/reqSec.dat')) * 1.2; +const latencyMax = getMaxValue(readFileContent('/tmp/latency.dat')) * 1.2; + +const gnuplotScript = generateGnuplotScript(whichBench, reqSecMax, latencyMax); +writeGnuplotScript(gnuplotScript); + +try { + execSync(`gnuplot /tmp/gnuplot_script.gp`, { stdio: 'inherit' }); + console.log('Gnuplot executed successfully'); +} catch (error) { + console.error('Error executing gnuplot:', (error as Error).message); +} + +const assetsDir = path.join(__dirname, "assets"); +createDirectoryIfNotExists(assetsDir); + +moveFile(`req_sec_histogram${whichBench}.png`, path.join(assetsDir, `req_sec_histogram${whichBench}.png`)); +moveFile(`latency_histogram${whichBench}.png`, path.join(assetsDir, `latency_histogram${whichBench}.png`)); + +const resultsTable = formatResults(serverMetrics, formattedServerNames, whichBench); +writeResults(resultsTable, whichBench); + +resultFiles.forEach((file) => { + try { + fs.unlinkSync(file); + } catch (error) { + console.error(`Error deleting file ${file}: ${(error as Error).message}`); + } +}); diff --git a/analyze/config.js b/analyze/config.js new file mode 100644 index 00000000..393684fa --- /dev/null +++ b/analyze/config.js @@ -0,0 +1,14 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.servers = exports.formattedServerNames = void 0; +exports.formattedServerNames = { + tailcall: "Tailcall", + gqlgen: "Gqlgen", + apollo: "Apollo GraphQL", + netflixdgs: "Netflix DGS", + caliban: "Caliban", + async_graphql: "async-graphql", + hasura: "Hasura", + graphql_jit: "GraphQL JIT", +}; +exports.servers = ["apollo", "caliban", "netflixdgs", "gqlgen", "tailcall", "async_graphql", "hasura", "graphql_jit"]; diff --git a/analyze/config.ts b/analyze/config.ts new file mode 100644 index 00000000..0b84b675 --- /dev/null +++ b/analyze/config.ts @@ -0,0 +1,14 @@ +import { FormattedServerNames } from './types'; + +export const formattedServerNames: FormattedServerNames = { + tailcall: "Tailcall", + gqlgen: "Gqlgen", + apollo: "Apollo GraphQL", + netflixdgs: "Netflix DGS", + caliban: "Caliban", + async_graphql: "async-graphql", + hasura: "Hasura", + graphql_jit: "GraphQL JIT", +}; + +export const servers: string[] = ["apollo", "caliban", "netflixdgs", "gqlgen", "tailcall", "async_graphql", "hasura", "graphql_jit"]; diff --git a/analyze/dataFileWriter.js b/analyze/dataFileWriter.js new file mode 100644 index 00000000..67653355 --- /dev/null +++ b/analyze/dataFileWriter.js @@ -0,0 +1,10 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.writeMetricsDataFiles = writeMetricsDataFiles; +var fileUtils_1 = require("./fileUtils"); +function writeMetricsDataFiles(serverMetrics, servers) { + var reqSecData = "/tmp/reqSec.dat"; + var latencyData = "/tmp/latency.dat"; + (0, fileUtils_1.writeDataFile)(reqSecData, "Server Value\n" + servers.map(function (server) { return "".concat(server, " ").concat(serverMetrics[server].reqSec); }).join('\n')); + (0, fileUtils_1.writeDataFile)(latencyData, "Server Value\n" + servers.map(function (server) { return "".concat(server, " ").concat(serverMetrics[server].latency); }).join('\n')); +} diff --git a/analyze/dataFileWriter.ts b/analyze/dataFileWriter.ts new file mode 100644 index 00000000..3812d5d0 --- /dev/null +++ b/analyze/dataFileWriter.ts @@ -0,0 +1,10 @@ +import { writeDataFile } from './fileUtils'; +import { ServerMetrics } from './types'; + +export function writeMetricsDataFiles(serverMetrics: Record, servers: string[]): void { + const reqSecData = "/tmp/reqSec.dat"; + const latencyData = "/tmp/latency.dat"; + + writeDataFile(reqSecData, "Server Value\n" + servers.map(server => `${server} ${serverMetrics[server].reqSec}`).join('\n')); + writeDataFile(latencyData, "Server Value\n" + servers.map(server => `${server} ${serverMetrics[server].latency}`).join('\n')); +} diff --git a/analyze/fileUtils.js b/analyze/fileUtils.js new file mode 100644 index 00000000..a7459d1f --- /dev/null +++ b/analyze/fileUtils.js @@ -0,0 +1,48 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.writeDataFile = writeDataFile; +exports.readFileContent = readFileContent; +exports.moveFile = moveFile; +exports.createDirectoryIfNotExists = createDirectoryIfNotExists; +var fs = require("fs"); +function writeDataFile(filename, data) { + try { + fs.writeFileSync(filename, data); + } + catch (error) { + console.error("Error writing data file ".concat(filename, ": ").concat(error.message)); + } +} +function readFileContent(filename) { + try { + return fs.readFileSync(filename, 'utf-8'); + } + catch (error) { + console.error("Error reading file ".concat(filename, ": ").concat(error.message)); + return ''; + } +} +function moveFile(source, destination) { + try { + if (fs.existsSync(source)) { + fs.renameSync(source, destination); + console.log("Moved ".concat(source, " to ").concat(destination)); + } + else { + console.log("Source file ".concat(source, " does not exist")); + } + } + catch (error) { + console.error("Error moving file ".concat(source, ": ").concat(error.message)); + } +} +function createDirectoryIfNotExists(dir) { + if (!fs.existsSync(dir)) { + try { + fs.mkdirSync(dir); + } + catch (error) { + console.error("Error creating directory: ".concat(error.message)); + } + } +} diff --git a/analyze/fileUtils.ts b/analyze/fileUtils.ts new file mode 100644 index 00000000..5e7ce332 --- /dev/null +++ b/analyze/fileUtils.ts @@ -0,0 +1,42 @@ +import * as fs from 'fs'; +import * as path from 'path'; + +export function writeDataFile(filename: string, data: string): void { + try { + fs.writeFileSync(filename, data); + } catch (error) { + console.error(`Error writing data file ${filename}: ${(error as Error).message}`); + } +} + +export function readFileContent(filename: string): string { + try { + return fs.readFileSync(filename, 'utf-8'); + } catch (error) { + console.error(`Error reading file ${filename}: ${(error as Error).message}`); + return ''; + } +} + +export function moveFile(source: string, destination: string): void { + try { + if (fs.existsSync(source)) { + fs.renameSync(source, destination); + console.log(`Moved ${source} to ${destination}`); + } else { + console.log(`Source file ${source} does not exist`); + } + } catch (error) { + console.error(`Error moving file ${source}: ${(error as Error).message}`); + } +} + +export function createDirectoryIfNotExists(dir: string): void { + if (!fs.existsSync(dir)) { + try { + fs.mkdirSync(dir); + } catch (error) { + console.error(`Error creating directory: ${(error as Error).message}`); + } + } +} diff --git a/analyze/gnuplotGenerator.js b/analyze/gnuplotGenerator.js new file mode 100644 index 00000000..ba7296da --- /dev/null +++ b/analyze/gnuplotGenerator.js @@ -0,0 +1,14 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.generateGnuplotScript = generateGnuplotScript; +exports.writeGnuplotScript = writeGnuplotScript; +var fileUtils_1 = require("./fileUtils"); +function generateGnuplotScript(whichBench, reqSecMax, latencyMax) { + var reqSecHistogramFile = "req_sec_histogram".concat(whichBench, ".png"); + var latencyHistogramFile = "latency_histogram".concat(whichBench, ".png"); + return "\nset term pngcairo size 1280,720 enhanced font 'Courier,12'\nset output '".concat(reqSecHistogramFile, "'\nset style data histograms\nset style histogram cluster gap 1\nset style fill solid border -1\nset xtics rotate by -45\nset boxwidth 0.9\nset title 'Requests/Sec'\nset yrange [0:").concat(reqSecMax, "]\nset key outside right top\nplot '/tmp/reqSec.dat' using 2:xtic(1) title 'Req/Sec'\n\nset output '").concat(latencyHistogramFile, "'\nset title 'Latency (in ms)'\nset yrange [0:").concat(latencyMax, "]\nplot '/tmp/latency.dat' using 2:xtic(1) title 'Latency'\n"); +} +function writeGnuplotScript(script) { + var gnuplotScriptFile = '/tmp/gnuplot_script.gp'; + (0, fileUtils_1.writeDataFile)(gnuplotScriptFile, script); +} diff --git a/analyze/gnuplotGenerator.ts b/analyze/gnuplotGenerator.ts new file mode 100644 index 00000000..65a9f76e --- /dev/null +++ b/analyze/gnuplotGenerator.ts @@ -0,0 +1,30 @@ +import { writeDataFile } from './fileUtils'; + +export function generateGnuplotScript(whichBench: number, reqSecMax: number, latencyMax: number): string { + const reqSecHistogramFile = `req_sec_histogram${whichBench}.png`; + const latencyHistogramFile = `latency_histogram${whichBench}.png`; + + return ` +set term pngcairo size 1280,720 enhanced font 'Courier,12' +set output '${reqSecHistogramFile}' +set style data histograms +set style histogram cluster gap 1 +set style fill solid border -1 +set xtics rotate by -45 +set boxwidth 0.9 +set title 'Requests/Sec' +set yrange [0:${reqSecMax}] +set key outside right top +plot '/tmp/reqSec.dat' using 2:xtic(1) title 'Req/Sec' + +set output '${latencyHistogramFile}' +set title 'Latency (in ms)' +set yrange [0:${latencyMax}] +plot '/tmp/latency.dat' using 2:xtic(1) title 'Latency' +`; +} + +export function writeGnuplotScript(script: string): void { + const gnuplotScriptFile = '/tmp/gnuplot_script.gp'; + writeDataFile(gnuplotScriptFile, script); +} diff --git a/analyze/parser.js b/analyze/parser.js new file mode 100644 index 00000000..f2e571da --- /dev/null +++ b/analyze/parser.js @@ -0,0 +1,49 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.parseMetric = parseMetric; +exports.calculateAverage = calculateAverage; +exports.parseServerMetrics = parseServerMetrics; +function parseMetric(input, metric) { + var lines = input.split('\n'); + var metricLine; + if (metric === "Latency") { + metricLine = lines.find(function (line) { return line.trim().startsWith("Latency"); }); + } + else if (metric === "Requests/sec") { + metricLine = lines.find(function (line) { return line.trim().startsWith("Requests/sec"); }); + } + if (!metricLine) + return null; + var match = metricLine.match(/([\d.]+)/); + return match ? parseFloat(match[1]) : null; +} +function calculateAverage(values) { + if (values.length === 0) + return 0; + var sum = values.reduce(function (a, b) { return a + b; }, 0); + return sum / values.length; +} +function parseServerMetrics(servers, inputs) { + var serverMetrics = {}; + servers.forEach(function (server, idx) { + var startIdx = idx * 3; + var reqSecVals = []; + var latencyVals = []; + for (var j = 0; j < 3; j++) { + var inputIdx = startIdx + j; + if (inputIdx < inputs.length) { + var reqSec = parseMetric(inputs[inputIdx], "Requests/sec"); + var latency = parseMetric(inputs[inputIdx], "Latency"); + if (reqSec !== null) + reqSecVals.push(reqSec); + if (latency !== null) + latencyVals.push(latency); + } + } + serverMetrics[server] = { + reqSec: calculateAverage(reqSecVals), + latency: calculateAverage(latencyVals) + }; + }); + return serverMetrics; +} diff --git a/analyze/parser.ts b/analyze/parser.ts new file mode 100644 index 00000000..b15e47c0 --- /dev/null +++ b/analyze/parser.ts @@ -0,0 +1,48 @@ +import { ServerMetrics } from './types'; + +export function parseMetric(input: string, metric: string): number | null { + const lines = input.split('\n'); + let metricLine: string | undefined; + + if (metric === "Latency") { + metricLine = lines.find(line => line.trim().startsWith("Latency")); + } else if (metric === "Requests/sec") { + metricLine = lines.find(line => line.trim().startsWith("Requests/sec")); + } + + if (!metricLine) return null; + + const match = metricLine.match(/([\d.]+)/); + return match ? parseFloat(match[1]) : null; +} + +export function calculateAverage(values: number[]): number { + if (values.length === 0) return 0; + const sum = values.reduce((a, b) => a + b, 0); + return sum / values.length; +} + +export function parseServerMetrics(servers: string[], inputs: string[]): Record { + const serverMetrics: Record = {}; + + servers.forEach((server, idx) => { + const startIdx = idx * 3; + const reqSecVals: number[] = []; + const latencyVals: number[] = []; + for (let j = 0; j < 3; j++) { + const inputIdx = startIdx + j; + if (inputIdx < inputs.length) { + const reqSec = parseMetric(inputs[inputIdx], "Requests/sec"); + const latency = parseMetric(inputs[inputIdx], "Latency"); + if (reqSec !== null) reqSecVals.push(reqSec); + if (latency !== null) latencyVals.push(latency); + } + } + serverMetrics[server] = { + reqSec: calculateAverage(reqSecVals), + latency: calculateAverage(latencyVals) + }; + }); + + return serverMetrics; +} diff --git a/analyze/resultsFormatter.js b/analyze/resultsFormatter.js new file mode 100644 index 00000000..f35c5763 --- /dev/null +++ b/analyze/resultsFormatter.js @@ -0,0 +1,70 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.formatResults = formatResults; +exports.writeResults = writeResults; +var fs = require("fs"); +function formatResults(serverMetrics, formattedServerNames, whichBench) { + var sortedServers = Object.keys(serverMetrics).sort(function (a, b) { return serverMetrics[b].reqSec - serverMetrics[a].reqSec; }); + var lastServer = sortedServers[sortedServers.length - 1]; + var lastServerReqSecs = serverMetrics[lastServer].reqSec; + var resultsTable = ""; + if (whichBench === 1) { + resultsTable += "\n| ".concat(whichBench, " | `{ posts { id userId title user { id name email }}}` |"); + } + else if (whichBench === 2) { + resultsTable += "\n| ".concat(whichBench, " | `{ posts { title }}` |"); + } + else if (whichBench === 3) { + resultsTable += "\n| ".concat(whichBench, " | `{ greet }` |"); + } + sortedServers.forEach(function (server) { + var formattedReqSecs = serverMetrics[server].reqSec.toLocaleString(undefined, { + minimumFractionDigits: 2, + maximumFractionDigits: 2, + }); + var formattedLatencies = serverMetrics[server].latency.toLocaleString(undefined, { + minimumFractionDigits: 2, + maximumFractionDigits: 2, + }); + var relativePerformance = (serverMetrics[server].reqSec / lastServerReqSecs).toFixed(2); + resultsTable += "\n|| [".concat(formattedServerNames[server], "] | `").concat(formattedReqSecs, "` | `").concat(formattedLatencies, "` | `").concat(relativePerformance, "x` |"); + }); + return resultsTable; +} +function writeResults(resultsTable, whichBench) { + var resultsFile = "results.md"; + try { + if (!fs.existsSync(resultsFile) || fs.readFileSync(resultsFile, 'utf8').trim() === '') { + fs.writeFileSync(resultsFile, "\n\n| Query | Server | Requests/sec | Latency (ms) | Relative |\n|-------:|--------:|--------------:|--------------:|---------:|"); + } + fs.appendFileSync(resultsFile, resultsTable); + if (whichBench === 3) { + fs.appendFileSync(resultsFile, "\n\n"); + updateReadme(resultsFile); + } + } + catch (error) { + console.error("Error writing results: ".concat(error.message)); + } +} +function updateReadme(resultsFile) { + try { + var finalResults = fs + .readFileSync(resultsFile, "utf-8") + .replace(/\\/g, ''); // Remove backslashes + var readmePath = "README.md"; + var readmeContent = fs.readFileSync(readmePath, "utf-8"); + var performanceResultsRegex = /[\s\S]*/; + if (performanceResultsRegex.test(readmeContent)) { + readmeContent = readmeContent.replace(performanceResultsRegex, finalResults); + } + else { + readmeContent += "\n".concat(finalResults); + } + fs.writeFileSync(readmePath, readmeContent); + console.log("README.md updated successfully"); + } + catch (error) { + console.error("Error updating README: ".concat(error.message)); + } +} diff --git a/analyze/resultsFormatter.ts b/analyze/resultsFormatter.ts new file mode 100644 index 00000000..2169c6f9 --- /dev/null +++ b/analyze/resultsFormatter.ts @@ -0,0 +1,83 @@ +import * as fs from 'fs'; +import { FormattedServerNames, ServerMetrics } from './types'; + +export function formatResults(serverMetrics: Record, formattedServerNames: FormattedServerNames, whichBench: number): string { + const sortedServers = Object.keys(serverMetrics).sort( + (a, b) => serverMetrics[b].reqSec - serverMetrics[a].reqSec + ); + const lastServer = sortedServers[sortedServers.length - 1]; + const lastServerReqSecs = serverMetrics[lastServer].reqSec; + + let resultsTable = ""; + + if (whichBench === 1) { + resultsTable += `\n| ${whichBench} | \`{ posts { id userId title user { id name email }}}\` |`; + } else if (whichBench === 2) { + resultsTable += `\n| ${whichBench} | \`{ posts { title }}\` |`; + } else if (whichBench === 3) { + resultsTable += `\n| ${whichBench} | \`{ greet }\` |`; + } + + sortedServers.forEach((server) => { + const formattedReqSecs = serverMetrics[server].reqSec.toLocaleString(undefined, { + minimumFractionDigits: 2, + maximumFractionDigits: 2, + }); + const formattedLatencies = serverMetrics[server].latency.toLocaleString(undefined, { + minimumFractionDigits: 2, + maximumFractionDigits: 2, + }); + const relativePerformance = (serverMetrics[server].reqSec / lastServerReqSecs).toFixed(2); + + resultsTable += `\n|| [${formattedServerNames[server]}] | \`${formattedReqSecs}\` | \`${formattedLatencies}\` | \`${relativePerformance}x\` |`; + }); + + return resultsTable; +} + +export function writeResults(resultsTable: string, whichBench: number): void { + const resultsFile = "results.md"; + + try { + if (!fs.existsSync(resultsFile) || fs.readFileSync(resultsFile, 'utf8').trim() === '') { + fs.writeFileSync(resultsFile, ` + +| Query | Server | Requests/sec | Latency (ms) | Relative | +|-------:|--------:|--------------:|--------------:|---------:|`); + } + + fs.appendFileSync(resultsFile, resultsTable); + + if (whichBench === 3) { + fs.appendFileSync(resultsFile, "\n\n"); + updateReadme(resultsFile); + } + } catch (error) { + console.error(`Error writing results: ${(error as Error).message}`); + } +} + +function updateReadme(resultsFile: string): void { + try { + const finalResults = fs + .readFileSync(resultsFile, "utf-8") + .replace(/\\/g, ''); // Remove backslashes + + const readmePath = "README.md"; + let readmeContent = fs.readFileSync(readmePath, "utf-8"); + const performanceResultsRegex = + /[\s\S]*/; + if (performanceResultsRegex.test(readmeContent)) { + readmeContent = readmeContent.replace( + performanceResultsRegex, + finalResults + ); + } else { + readmeContent += `\n${finalResults}`; + } + fs.writeFileSync(readmePath, readmeContent); + console.log("README.md updated successfully"); + } catch (error) { + console.error(`Error updating README: ${(error as Error).message}`); + } +} diff --git a/analyze/types.js b/analyze/types.js new file mode 100644 index 00000000..c8ad2e54 --- /dev/null +++ b/analyze/types.js @@ -0,0 +1,2 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); diff --git a/analyze/types.ts b/analyze/types.ts new file mode 100644 index 00000000..e8672961 --- /dev/null +++ b/analyze/types.ts @@ -0,0 +1,8 @@ +export interface ServerMetrics { + reqSec: number; + latency: number; +} + +export interface FormattedServerNames { + [key: string]: string; +} diff --git a/run_analyze_script.sh b/run_analyze_script.sh index dd284135..1a84356f 100755 --- a/run_analyze_script.sh +++ b/run_analyze_script.sh @@ -12,8 +12,9 @@ services=("apollo" "caliban" "netflixdgs" "gqlgen" "tailcall" "async_graphql" "h for bench in 1 2 3; do echo "Processing files for bench${bench}:" - # Construct the command for each benchmark - cmd="node analyze.js" + tsc analyze.ts + # Construct the command for each benchmark + cmd="node analyze.js" # Loop through each service for service in "${services[@]}"; do @@ -33,4 +34,4 @@ for bench in 1 2 3; do # Execute the command echo "Executing: $cmd" eval $cmd -done \ No newline at end of file +done diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 00000000..9db8d78f --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,9 @@ +{ + "compilerOptions": { + "target": "ES2018", + "module": "CommonJS", + "rootDir": "./", + "strict": true, + "esModuleInterop": true + } +}