HTML report for data visualization

Part of #296
symflower · Jul 30, 2024 · b515b6f · b515b6f
1 parent e5c1bcc
commit b515b6f
Show file tree

Hide file tree

Showing 5 changed files with 221 additions and 0 deletions.
diff --git a/evaluate/report/charts/costs.csv b/evaluate/report/charts/costs.csv
@@ -0,0 +1,8 @@
+model-id,model-cost,score
+modelA,1000,1142
+modelB,2000,716
+modelC,3000,220
+modelD,4000,870
+modelE,1000,2673
+modelF,2500,12
+modelG,1250,982
diff --git a/evaluate/report/charts/evaluation.csv b/evaluate/report/charts/evaluation.csv
@@ -0,0 +1,15 @@
+model-id,language,repository,task,score,coverage,files-executed,files-executed-maximum-reachable,generate-tests-for-file-character-count,processing-time,response-character-count,response-no-error,response-no-excess,response-with-code,tests-passing
+openrouter/meta-llama/llama-3.1-8b-instruct,golang,golang/plain,write-tests,3,0,0,1,611,2451,621,1,1,1,0
+openrouter/meta-llama/llama-3.1-8b-instruct,golang,golang/plain,write-tests-symflower-fix,3,0,0,1,611,2451,621,1,1,1,0
+openrouter/meta-llama/llama-3.1-8b-instruct,java,java/plain,write-tests,14,10,1,1,597,2011,609,1,1,1,0
+openrouter/meta-llama/llama-3.1-8b-instruct,java,java/plain,write-tests-symflower-fix,14,10,1,1,597,2011,609,1,1,1,0
+openrouter/meta-llama/llama-3.1-8b-instruct,golang,golang/light,write-tests,486,410,11,23,24954,697098,25170,23,21,21,0
+openrouter/meta-llama/llama-3.1-8b-instruct,golang,golang/light,write-tests-symflower-fix,567,490,12,23,24954,697628,25170,23,21,21,0
+openrouter/meta-llama/llama-3.1-8b-instruct,golang,golang/mistakes,code-repair,53,40,2,5,527,5470,568,5,3,3,0
+openrouter/meta-llama/llama-3.1-8b-instruct,golang,golang/transpile,transpile,228,0,4,5,1389,10908,1905,5,4,5,210
+openrouter/meta-llama/llama-3.1-8b-instruct,golang,golang/transpile,transpile-symflower-fix,289,0,5,5,1389,11142,1905,5,4,5,270
+openrouter/meta-llama/llama-3.1-8b-instruct,java,java/light,write-tests,1524,1450,12,23,34748,99019,35166,23,19,20,0
+openrouter/meta-llama/llama-3.1-8b-instruct,java,java/light,write-tests-symflower-fix,1524,1450,12,23,34748,99019,35166,23,19,20,0
+openrouter/meta-llama/llama-3.1-8b-instruct,java,java/mistakes,code-repair,222,210,3,5,718,6528,1393,5,1,3,0
+openrouter/meta-llama/llama-3.1-8b-instruct,java,java/transpile,transpile,197,0,4,5,1244,10499,1295,5,4,4,180
+openrouter/meta-llama/llama-3.1-8b-instruct,java,java/transpile,transpile-symflower-fix,197,0,4,5,1244,10499,1295,5,4,4,180
diff --git a/evaluate/report/charts/index.html b/evaluate/report/charts/index.html
@@ -0,0 +1,47 @@
+<!DOCTYPE html>
+<html lang="en">
+
+<head>
+	<meta charset="utf-8">
+	<meta name="viewport" content="width=device-width, initial-scale=1">
+	<title>EvalDevQuality</title>
+	<link href="https://stackpath.bootstrapcdn.com/bootstrap/4.5.2/css/bootstrap.min.css" rel="stylesheet">
+	<link href="https://cdn.datatables.net/1.11.5/css/jquery.dataTables.min.css" rel="stylesheet">
+	<link rel="stylesheet" href="styles.css">
+</head>
+
+<body>
+	<div class="container-fluid h-100">
+		<h1 class="text-center my-4">EvalDevQuality</h1>
+		<ul class="nav nav-tabs" id="chartTabs" role="tablist">
+			<li class="nav-item">
+				<a class="nav-link active" id="evaluation-tab" data-toggle="tab" href="#evaluation" role="tab"
+					aria-controls="evaluation" aria-selected="false">Evaluation</a>
+			</li>
+			<li class="nav-item">
+				<a class="nav-link" id="scatter-tab" data-toggle="tab" href="#scatter" role="tab"
+					aria-controls="scatter" aria-selected="false">Costs vs. Score</a>
+			</li>
+		</ul>
+		<div class="tab-content h-100 mx-5 my-5" id="chartTabsContent">
+			<div class="tab-pane fade show active" id="evaluation" role="tabpanel" aria-labelledby="evaluation-tab">
+				<div class="table-container">
+					<table class="table table-striped table-bordered" id="evaluation-table"></table>
+				</div>
+			</div>
+			<div class="tab-pane fade" id="scatter" role="tabpanel" aria-labelledby="scatter-tab">
+				<div class="container-fluid d-flex justify-content-center align-items-center">
+					<div id="evaluation-scatter"></div>
+				</div>
+			</div>
+		</div>
+	</div>
+
+	<script src="https://code.jquery.com/jquery-3.5.1.min.js"></script>
+	<script src="https://cdn.datatables.net/1.11.5/js/jquery.dataTables.min.js"></script>
+	<script src="https://d3js.org/d3.v6.min.js"></script>
+	<script src="https://stackpath.bootstrapcdn.com/bootstrap/4.5.2/js/bootstrap.min.js"></script>
+	<script src="script.js"></script>
+</body>
+
+</html>
diff --git a/evaluate/report/charts/script.js b/evaluate/report/charts/script.js
@@ -0,0 +1,141 @@
+// evaluationTable creates a table given an evaluation CSV file.
+function evaluationTable(csvFilePath) {
+  d3.csv(csvFilePath).then((data) => {
+    const table = $("#evaluation-table");
+    const columns = Object.keys(data[0]);
+
+    // Create the header.
+    let thead = "<thead><tr>";
+    columns.forEach((col) => {
+      thead += `<th>${col}</th>`;
+    });
+    thead += "</tr></thead>";
+    table.append(thead);
+
+    // Create the body.
+    let tbody = "<tbody>";
+    data.forEach((row) => {
+      tbody += "<tr>";
+      columns.forEach((col) => {
+        tbody += `<td>${row[col]}</td>`;
+      });
+      tbody += "</tr>";
+    });
+    tbody += "</tbody>";
+    table.append(tbody);
+
+    // Initialize the DataTable.
+    table.DataTable({
+      order: [[0, "asc"]],
+      dom: "lfrtip",
+    });
+  });
+}
+
+// evaluationScatterPlot creates a scatter plot comparing models costs and scores.
+function evaluationScatterPlot(csvFilePath) {
+  // Set dimensions and margins for the scatter plot.
+  const margin = { top: 20, right: 30, bottom: 50, left: 70 },
+    width = 800,
+    height = 600;
+
+  // Append the SVG element to the body.
+  const svg = d3
+    .select("#evaluation-scatter")
+    .append("svg")
+    .attr("width", width + margin.left + margin.right)
+    .attr("height", height + margin.top + margin.bottom)
+    .append("g")
+    .attr("transform", `translate(${margin.left},${margin.top})`);
+
+  // Load the CSV data
+  d3.csv(csvFilePath).then((data) => {
+    // Convert strings to numbers.
+    data.forEach((d) => {
+      d["model-cost"] = +d["model-cost"];
+      d.score = +d.score;
+    });
+
+    // Set up the x and y scales.
+    const x = d3
+      .scaleLinear()
+      .domain([0, d3.max(data, (d) => d["model-cost"]) * 1.1])
+      .range([0, width]);
+
+    const y = d3
+      .scaleLinear()
+      .domain([
+        d3.min(data, (d) => d.score),
+        d3.max(data, (d) => d.score * 1.1),
+      ])
+      .range([height, 0]);
+
+    // Add the grid lines for the x-axis.
+    const xAxisGrid = d3.axisBottom(x).tickSize(-height).tickFormat("");
+    svg
+      .append("g")
+      .attr("class", "grid")
+      .attr("transform", `translate(0, ${height})`)
+      .call(xAxisGrid);
+
+    // Add the grid lines for the y-axis.
+    const yAxisGrid = d3.axisLeft(y).tickSize(-width).tickFormat("");
+    svg.append("g").attr("class", "grid").call(yAxisGrid);
+
+    // Add the x-axis.
+    svg
+      .append("g")
+      .attr("transform", `translate(0,${height})`)
+      .call(d3.axisBottom(x))
+      .append("text")
+      .attr("x", width / 2)
+      .attr("y", margin.bottom - 10)
+      .attr("fill", "black")
+      .attr("text-anchor", "middle")
+      .style("font-size", "16px")
+      .text("Costs per token");
+
+    // Add the y-axis.
+    svg
+      .append("g")
+      .call(d3.axisLeft(y))
+      .append("text")
+      .attr("transform", "rotate(-90)")
+      .attr("x", -height / 2)
+      .attr("y", -margin.left + 15)
+      .attr("fill", "black")
+      .attr("text-anchor", "middle")
+      .style("font-size", "16px")
+      .text("Score");
+
+    // Add points.
+    svg
+      .append("g")
+      .selectAll("circle")
+      .data(data)
+      .enter()
+      .append("circle")
+      .attr("cx", (d) => x(d["model-cost"]))
+      .attr("cy", (d) => y(d.score))
+      .attr("r", 5)
+      .attr("fill", "green");
+
+    // Add labels.
+    svg
+      .append("g")
+      .selectAll("text")
+      .data(data)
+      .enter()
+      .append("text")
+      .attr("x", (d) => x(d["model-cost"]))
+      .attr("y", (d) => y(d.score))
+      .attr("dy", -10)
+      .attr("text-anchor", "middle")
+      .text((d) => d["model-id"])
+      .attr("font-size", "16px")
+      .attr("fill", "black");
+  });
+}
+
+evaluationTable("evaluation.csv");
+evaluationScatterPlot("costs.csv");
diff --git a/evaluate/report/charts/styles.css b/evaluate/report/charts/styles.css
@@ -0,0 +1,10 @@
+html,
+body {
+	height: 100%;
+	margin: 0;
+}
+
+.grid line {
+	stroke: #e0e0e0;
+	stroke-width: 1px;
+}