feat(272): Port analyze.sh to analyze.js

.gitignore

@@ -24,3 +24,6 @@ graphql/gqlgen/main
 tailcall-src
 
 metals.*
+
+analyze/*
+analyze.js

analyze.ts

@@ -0,0 +1,76 @@
+#!/usr/bin/env node
+import { execSync } from 'child_process';
+import * as path from 'path';
+import * as fs from 'fs';
+import { servers, formattedServerNames } from './analyze/config';
+import { createDirectoryIfNotExists, moveFile, readFileContent } from './analyze/fileUtils';
+import { parseServerMetrics } from './analyze/parser';
+import { writeMetricsDataFiles } from './analyze/dataFileWriter';
+import { generateGnuplotScript, writeGnuplotScript } from './analyze/gnuplotGenerator';
+import { formatResults, writeResults } from './analyze/resultsFormatter';
+
+const resultFiles: string[] = process.argv.slice(2);
+
+// Read content of result files
+const resultContents: string[] = resultFiles.map(file => readFileContent(file));
+
+const serverMetrics = parseServerMetrics(servers, resultContents);
+
+writeMetricsDataFiles(serverMetrics, servers);
+
+let whichBench = 1;
+if (resultFiles.length > 0) {
+  if (resultFiles[0].startsWith("bench2")) {
+    whichBench = 2;
+  } else if (resultFiles[0].startsWith("bench3")) {
+    whichBench = 3;
+  }
+}
+
+function getMaxValue(data: string): number {
+  try {
+    return Math.max(...data.split('\n')
+      .slice(1)
+      .map(line => {
+        const [, valueStr] = line.split(' ');
+        const value = parseFloat(valueStr);
+        if (isNaN(value)) {
+          throw new Error(`Invalid number in data: ${valueStr}`);
+        }
+        return value;
+      }));
+  } catch (error) {
+    console.error(`Error getting max value: ${(error as Error).message}`);
+    return 0;
+  }
+}
+
+const reqSecMax = getMaxValue(readFileContent('/tmp/reqSec.dat')) * 1.2;
+const latencyMax = getMaxValue(readFileContent('/tmp/latency.dat')) * 1.2;
+
+const gnuplotScript = generateGnuplotScript(whichBench, reqSecMax, latencyMax);
+writeGnuplotScript(gnuplotScript);
+
+try {
+  execSync(`gnuplot /tmp/gnuplot_script.gp`, { stdio: 'inherit' });
+  console.log('Gnuplot executed successfully');
+} catch (error) {
+  console.error('Error executing gnuplot:', (error as Error).message);
+}
+
+const assetsDir = path.join(__dirname, "assets");
+createDirectoryIfNotExists(assetsDir);
+
+moveFile(`req_sec_histogram${whichBench}.png`, path.join(assetsDir, `req_sec_histogram${whichBench}.png`));
+moveFile(`latency_histogram${whichBench}.png`, path.join(assetsDir, `latency_histogram${whichBench}.png`));
+
+const resultsTable = formatResults(serverMetrics, formattedServerNames, whichBench);
+writeResults(resultsTable, whichBench);
+
+resultFiles.forEach((file) => {
+  try {
+    fs.unlinkSync(file);
+  } catch (error) {
+    console.error(`Error deleting file ${file}: ${(error as Error).message}`);
+  }
+});

analyze/config.ts

@@ -0,0 +1,14 @@
+import { FormattedServerNames } from './types';
+
+export const formattedServerNames: FormattedServerNames = {
+  tailcall: "Tailcall",
+  gqlgen: "Gqlgen",
+  apollo: "Apollo GraphQL",
+  netflixdgs: "Netflix DGS",
+  caliban: "Caliban",
+  async_graphql: "async-graphql",
+  hasura: "Hasura",
+  graphql_jit: "GraphQL JIT",
+};
+
+export const servers: string[] = ["apollo", "caliban", "netflixdgs", "gqlgen", "tailcall", "async_graphql", "hasura", "graphql_jit"];

analyze/dataFileWriter.ts

@@ -0,0 +1,10 @@
+import { writeDataFile } from './fileUtils';
+import { ServerMetrics } from './types';
+
+export function writeMetricsDataFiles(serverMetrics: Record<string, ServerMetrics>, servers: string[]): void {
+  const reqSecData = "/tmp/reqSec.dat";
+  const latencyData = "/tmp/latency.dat";
+
+  writeDataFile(reqSecData, "Server Value\n" + servers.map(server => `${server} ${serverMetrics[server].reqSec}`).join('\n'));
+  writeDataFile(latencyData, "Server Value\n" + servers.map(server => `${server} ${serverMetrics[server].latency}`).join('\n'));
+}

analyze/fileUtils.ts

@@ -0,0 +1,42 @@
+import * as fs from 'fs';
+import * as path from 'path';
+
+export function writeDataFile(filename: string, data: string): void {
+  try {
+    fs.writeFileSync(filename, data);
+  } catch (error) {
+    console.error(`Error writing data file ${filename}: ${(error as Error).message}`);
+  }
+}
+
+export function readFileContent(filename: string): string {
+  try {
+    return fs.readFileSync(filename, 'utf-8');
+  } catch (error) {
+    console.error(`Error reading file ${filename}: ${(error as Error).message}`);
+    return '';
+  }
+}
+
+export function moveFile(source: string, destination: string): void {
+  try {
+    if (fs.existsSync(source)) {
+      fs.renameSync(source, destination);
+      console.log(`Moved ${source} to ${destination}`);
+    } else {
+      console.log(`Source file ${source} does not exist`);
+    }
+  } catch (error) {
+    console.error(`Error moving file ${source}: ${(error as Error).message}`);
+  }
+}
+
+export function createDirectoryIfNotExists(dir: string): void {
+  if (!fs.existsSync(dir)) {
+    try {
+      fs.mkdirSync(dir);
+    } catch (error) {
+      console.error(`Error creating directory: ${(error as Error).message}`);
+    }
+  }
+}

analyze/gnuplotGenerator.ts

@@ -0,0 +1,30 @@
+import { writeDataFile } from './fileUtils';
+
+export function generateGnuplotScript(whichBench: number, reqSecMax: number, latencyMax: number): string {
+  const reqSecHistogramFile = `req_sec_histogram${whichBench}.png`;
+  const latencyHistogramFile = `latency_histogram${whichBench}.png`;
+
+  return `
+set term pngcairo size 1280,720 enhanced font 'Courier,12'
+set output '${reqSecHistogramFile}'
+set style data histograms
+set style histogram cluster gap 1
+set style fill solid border -1
+set xtics rotate by -45
+set boxwidth 0.9
+set title 'Requests/Sec'
+set yrange [0:${reqSecMax}]
+set key outside right top
+plot '/tmp/reqSec.dat' using 2:xtic(1) title 'Req/Sec'
+
+set output '${latencyHistogramFile}'
+set title 'Latency (in ms)'
+set yrange [0:${latencyMax}]
+plot '/tmp/latency.dat' using 2:xtic(1) title 'Latency'
+`;
+}
+
+export function writeGnuplotScript(script: string): void {
+  const gnuplotScriptFile = '/tmp/gnuplot_script.gp';
+  writeDataFile(gnuplotScriptFile, script);
+}

analyze/parser.ts

@@ -0,0 +1,48 @@
+import { ServerMetrics } from './types';
+
+export function parseMetric(input: string, metric: string): number | null {
+  const lines = input.split('\n');
+  let metricLine: string | undefined;
+
+  if (metric === "Latency") {
+    metricLine = lines.find(line => line.trim().startsWith("Latency"));
+  } else if (metric === "Requests/sec") {
+    metricLine = lines.find(line => line.trim().startsWith("Requests/sec"));
+  }
+
+  if (!metricLine) return null;
+
+  const match = metricLine.match(/([\d.]+)/);
+  return match ? parseFloat(match[1]) : null;
+}
+
+export function calculateAverage(values: number[]): number {
+  if (values.length === 0) return 0;
+  const sum = values.reduce((a, b) => a + b, 0);
+  return sum / values.length;
+}
+
+export function parseServerMetrics(servers: string[], inputs: string[]): Record<string, ServerMetrics> {
+  const serverMetrics: Record<string, ServerMetrics> = {};
+
+  servers.forEach((server, idx) => {
+    const startIdx = idx * 3;
+    const reqSecVals: number[] = [];
+    const latencyVals: number[] = [];
+    for (let j = 0; j < 3; j++) {
+      const inputIdx = startIdx + j;
+      if (inputIdx < inputs.length) {
+        const reqSec = parseMetric(inputs[inputIdx], "Requests/sec");
+        const latency = parseMetric(inputs[inputIdx], "Latency");
+        if (reqSec !== null) reqSecVals.push(reqSec);
+        if (latency !== null) latencyVals.push(latency);
+      }
+    }
+    serverMetrics[server] = {
+      reqSec: calculateAverage(reqSecVals),
+      latency: calculateAverage(latencyVals)
+    };
+  });
+
+  return serverMetrics;
+}

analyze/resultsFormatter.ts

@@ -0,0 +1,83 @@
+import * as fs from 'fs';
+import { FormattedServerNames, ServerMetrics } from './types';
+
+export function formatResults(serverMetrics: Record<string, ServerMetrics>, formattedServerNames: FormattedServerNames, whichBench: number): string {
+  const sortedServers = Object.keys(serverMetrics).sort(
+    (a, b) => serverMetrics[b].reqSec - serverMetrics[a].reqSec
+  );
+  const lastServer = sortedServers[sortedServers.length - 1];
+  const lastServerReqSecs = serverMetrics[lastServer].reqSec;
+
+  let resultsTable = "";
+
+  if (whichBench === 1) {
+    resultsTable += `\n| ${whichBench} | \`{ posts { id userId title user { id name email }}}\` |`;
+  } else if (whichBench === 2) {
+    resultsTable += `\n| ${whichBench} | \`{ posts { title }}\` |`;
+  } else if (whichBench === 3) {
+    resultsTable += `\n| ${whichBench} | \`{ greet }\` |`;
+  }
+
+  sortedServers.forEach((server) => {
+    const formattedReqSecs = serverMetrics[server].reqSec.toLocaleString(undefined, {
+      minimumFractionDigits: 2,
+      maximumFractionDigits: 2,
+    });
+    const formattedLatencies = serverMetrics[server].latency.toLocaleString(undefined, {
+      minimumFractionDigits: 2,
+      maximumFractionDigits: 2,
+    });
+    const relativePerformance = (serverMetrics[server].reqSec / lastServerReqSecs).toFixed(2);
+
+    resultsTable += `\n|| [${formattedServerNames[server]}] | \`${formattedReqSecs}\` | \`${formattedLatencies}\` | \`${relativePerformance}x\` |`;
+  });
+
+  return resultsTable;
+}
+
+export function writeResults(resultsTable: string, whichBench: number): void {
+  const resultsFile = "results.md";
+
+  try {
+    if (!fs.existsSync(resultsFile) || fs.readFileSync(resultsFile, 'utf8').trim() === '') {
+      fs.writeFileSync(resultsFile, `<!-- PERFORMANCE_RESULTS_START -->
+
+| Query | Server | Requests/sec | Latency (ms) | Relative |
+|-------:|--------:|--------------:|--------------:|---------:|`);
+    }
+
+    fs.appendFileSync(resultsFile, resultsTable);
+
+    if (whichBench === 3) {
+      fs.appendFileSync(resultsFile, "\n\n<!-- PERFORMANCE_RESULTS_END -->");
+      updateReadme(resultsFile);
+    }
+  } catch (error) {
+    console.error(`Error writing results: ${(error as Error).message}`);
+  }
+}
+
+function updateReadme(resultsFile: string): void {
+  try {
+    const finalResults = fs
+      .readFileSync(resultsFile, "utf-8")
+      .replace(/\\/g, '');  // Remove backslashes
+
+    const readmePath = "README.md";
+    let readmeContent = fs.readFileSync(readmePath, "utf-8");
+    const performanceResultsRegex =
+      /<!-- PERFORMANCE_RESULTS_START -->[\s\S]*<!-- PERFORMANCE_RESULTS_END -->/;
+    if (performanceResultsRegex.test(readmeContent)) {
+      readmeContent = readmeContent.replace(
+        performanceResultsRegex,
+        finalResults
+      );
+    } else {
+      readmeContent += `\n${finalResults}`;
+    }
+    fs.writeFileSync(readmePath, readmeContent);
+    console.log("README.md updated successfully");
+  } catch (error) {
+    console.error(`Error updating README: ${(error as Error).message}`);
+  }
+}

analyze/types.ts

@@ -0,0 +1,8 @@
+export interface ServerMetrics {
+  reqSec: number;
+  latency: number;
+}
+
+export interface FormattedServerNames {
+  [key: string]: string;
+}

package.json

@@ -0,0 +1,5 @@
+{
+  "devDependencies": {
+    "@types/node": "^22.0.0"
+  }
+}

run_analyze_script.sh

@@ -2,6 +2,7 @@
 
 # Update and install gnuplot
 sudo apt-get update && sudo apt-get install -y gnuplot
+npm install
 
 # Remove existing results file
 rm -f results.md
@@ -12,8 +13,9 @@ services=("apollo" "caliban" "netflixdgs" "gqlgen" "tailcall" "async_graphql" "h
 for bench in 1 2 3; do
     echo "Processing files for bench${bench}:"
 
-    # Construct the command for each benchmark
-    cmd="bash analyze.sh"
+    tsc analyze.ts
+	# Construct the command for each benchmark
+	cmd="node analyze.js"
 
     # Loop through each service 
     for service in "${services[@]}"; do
@@ -33,4 +35,4 @@ for bench in 1 2 3; do
     # Execute the command
     echo "Executing: $cmd"
     eval $cmd
-done
+done

tsconfig.json

@@ -0,0 +1,9 @@
+{
+  "compilerOptions": {
+    "target": "ES2018",
+    "module": "CommonJS",
+    "rootDir": "./",
+    "strict": true,
+    "esModuleInterop": true
+  }
+}