HTML report for data visualization

Part of #296

evaluate/report/charts/charts.js

@@ -0,0 +1,124 @@
+// evaluationTable creates a table given an evaluation CSV file.
+function evaluationTable(data) {
+    const table = $("#evaluation-table");
+    const columns = data.columns;
+
+    // Create the header.
+    let thead = "<thead><tr>";
+    columns.forEach((col) => {
+        thead += `<th>${col}</th>`;
+    });
+    thead += "</tr></thead>";
+    table.append(thead);
+
+    // Create the body.
+    let tbody = "<tbody>";
+    data.forEach((row) => {
+        tbody += "<tr>";
+        columns.forEach((col) => {
+            tbody += `<td>${row[col]}</td>`;
+        });
+        tbody += "</tr>";
+    });
+    tbody += "</tbody>";
+    table.append(tbody);
+
+    // Initialize the DataTable.
+    table.DataTable({
+        order: [[0, "asc"]],
+        dom: "lfrtip",
+    });
+}
+
+// evaluationScatterPlot creates a scatter plot comparing models costs and scores.
+function evaluationScatterPlot(data) {
+    // Set dimensions and margins for the scatter plot.
+    const margin = { top: 20, right: 30, bottom: 50, left: 70 },
+        width = 800,
+        height = 600;
+
+    // Append the SVG element to the body.
+    const svg = d3
+        .select("#evaluation-scatter")
+        .append("svg")
+        .attr("width", width + margin.left + margin.right)
+        .attr("height", height + margin.top + margin.bottom)
+        .append("g")
+        .attr("transform", `translate(${margin.left},${margin.top})`);
+
+    // Set up the x and y scales.
+    const x = d3
+        .scaleLinear()
+        .domain([0, d3.max(data, (d) => d["model-costs"]) * 1.1])
+        .range([0, width]);
+
+    const y = d3
+        .scaleLinear()
+        .domain([
+            d3.min(data, (d) => d.score),
+            d3.max(data, (d) => d.score * 1.1),
+        ])
+        .range([height, 0]);
+
+    // Add the grid lines for the x-axis.
+    const xAxisGrid = d3.axisBottom(x).tickSize(-height).tickFormat("");
+    svg.append("g")
+        .attr("class", "grid")
+        .attr("transform", `translate(0, ${height})`)
+        .call(xAxisGrid);
+
+    // Add the grid lines for the y-axis.
+    const yAxisGrid = d3.axisLeft(y).tickSize(-width).tickFormat("");
+    svg.append("g").attr("class", "grid").call(yAxisGrid);
+
+    // Add the x-axis.
+    svg.append("g")
+        .attr("transform", `translate(0,${height})`)
+        .call(d3.axisBottom(x))
+        .append("text")
+        .attr("x", width / 2)
+        .attr("y", margin.bottom - 10)
+        .attr("fill", "black")
+        .attr("text-anchor", "middle")
+        .style("font-size", "16px")
+        .text("Costs per token");
+
+    // Add the y-axis.
+    svg.append("g")
+        .call(d3.axisLeft(y))
+        .append("text")
+        .attr("transform", "rotate(-90)")
+        .attr("x", -height / 2)
+        .attr("y", -margin.left + 15)
+        .attr("fill", "black")
+        .attr("text-anchor", "middle")
+        .style("font-size", "16px")
+        .text("Score");
+
+    // Add points.
+    svg.append("g")
+        .selectAll("circle")
+        .data(data)
+        .enter()
+        .append("circle")
+        .attr("cx", (d) => x(d["model-costs"]))
+        .attr("cy", (d) => y(d.score))
+        .attr("r", 5)
+        .attr("fill", "green");
+
+    // Add labels.
+    svg.append("g")
+        .selectAll("text")
+        .data(data)
+        .enter()
+        .append("text")
+        .attr("x", (d) => x(d["model-costs"]))
+        .attr("y", (d) => y(d.score))
+        .attr("dy", -10)
+        .attr("text-anchor", "middle")
+        .text((d) => d["model-name"])
+        .attr("font-size", "16px")
+        .attr("fill", "black");
+}
+
+export { evaluationTable, evaluationScatterPlot };

evaluate/report/charts/data.js

@@ -0,0 +1,51 @@
+// loadCSVFile loads a CSV file and returns the list of raw records.
+async function loadCSVFile(filepath) {
+    try {
+        const records = d3.csv(filepath);
+
+        return records;
+    } catch (error) {
+        console.error("error loading CSV file: ", error);
+    }
+}
+
+// modelsWithCostsAndScores returns an array of objects with model names, costs and scores.
+function modelsWithCostsAndScores(evaluationRecords, metaInformationRecords) {
+    // Group models and sum scores.
+    const modelsWithSummedScores = {};
+    evaluationRecords.forEach((record) => {
+        const modelId = record["model-id"];
+        const score = +record["score"];
+
+        if (!modelsWithSummedScores[modelId]) {
+            modelsWithSummedScores[modelId] = score;
+        } else {
+            modelsWithSummedScores[modelId] += score;
+        }
+    });
+
+    // Add costs and human-readable name.
+    const models = {};
+    for (const modelId in modelsWithSummedScores) {
+        models[modelId] = {
+            "model-name": modelId,
+            "model-costs": 0.0,
+            score: modelsWithSummedScores[modelId],
+        };
+    }
+    metaInformationRecords.forEach((record) => {
+        const modelId = record["model-id"];
+        if (models[modelId]) {
+            models[modelId]["model-name"] = record["model-name"];
+            models[modelId]["model-costs"] =
+                +record.completion +
+                +record.image +
+                +record.prompt +
+                +record.request;
+        }
+    });
+
+    return Object.values(models);
+}
+
+export { loadCSVFile, modelsWithCostsAndScores };

evaluate/report/charts/evaluation.csv

@@ -0,0 +1,7 @@
+model-id,language,repository,task,score,coverage,files-executed,files-executed-maximum-reachable,generate-tests-for-file-character-count,processing-time,response-character-count,response-no-error,response-no-excess,response-with-code,tests-passing
+provider/modelA,java,java/plain,write-tests,10,0,0,0,0,0,0,0,0,0,0
+provider/modelA,golang,golang/plain,write-tests,10,0,0,0,0,0,0,0,0,0,0
+provider/modelB,java,java/plain,write-tests,20,0,0,0,0,0,0,0,0,0,0
+provider/modelB,golang,golang/plain,write-tests,30,0,0,0,0,0,0,0,0,0,0
+provider/modelB,java,java/light,write-tests,40,0,0,0,0,0,0,0,0,0,0
+provider/modelC,java,java/plain,write-tests,10,0,0,0,0,0,0,0,0,0,0

evaluate/report/charts/index.html

@@ -0,0 +1,47 @@
+<!DOCTYPE html>
+<html lang="en">
+
+<head>
+	<meta charset="utf-8">
+	<meta name="viewport" content="width=device-width, initial-scale=1">
+	<title>EvalDevQuality</title>
+	<link href="https://stackpath.bootstrapcdn.com/bootstrap/4.5.2/css/bootstrap.min.css" rel="stylesheet">
+	<link href="https://cdn.datatables.net/1.11.5/css/jquery.dataTables.min.css" rel="stylesheet">
+	<link rel="stylesheet" href="styles.css">
+</head>
+
+<body>
+	<div class="container-fluid h-100">
+		<h1 class="text-center my-4">EvalDevQuality</h1>
+		<ul class="nav nav-tabs" id="chartTabs" role="tablist">
+			<li class="nav-item">
+				<a class="nav-link active" id="evaluation-tab" data-toggle="tab" href="#evaluation" role="tab"
+					aria-controls="evaluation" aria-selected="false">Evaluation</a>
+			</li>
+			<li class="nav-item">
+				<a class="nav-link" id="scatter-tab" data-toggle="tab" href="#scatter" role="tab"
+					aria-controls="scatter" aria-selected="false">Costs vs. Score</a>
+			</li>
+		</ul>
+		<div class="tab-content h-100 mx-5 my-5" id="chartTabsContent">
+			<div class="tab-pane fade show active" id="evaluation" role="tabpanel" aria-labelledby="evaluation-tab">
+				<div class="table-container">
+					<table class="table table-striped table-bordered" id="evaluation-table"></table>
+				</div>
+			</div>
+			<div class="tab-pane fade" id="scatter" role="tabpanel" aria-labelledby="scatter-tab">
+				<div class="container-fluid d-flex justify-content-center align-items-center">
+					<div id="evaluation-scatter"></div>
+				</div>
+			</div>
+		</div>
+	</div>
+
+	<script src="https://code.jquery.com/jquery-3.5.1.min.js"></script>
+	<script src="https://cdn.datatables.net/1.11.5/js/jquery.dataTables.min.js"></script>
+	<script src="https://d3js.org/d3.v6.min.js"></script>
+	<script src="https://stackpath.bootstrapcdn.com/bootstrap/4.5.2/js/bootstrap.min.js"></script>
+	<script type="module" src="script.js"></script>
+</body>
+
+</html>

evaluate/report/charts/meta.csv

@@ -0,0 +1,4 @@
+model-id,model-name,completion,image,prompt,request
+provider/modelA,modelA,0.1,0.2,0.3,0.4
+provider/modelB,modelB,0.01,0.02,0.03,0.04
+provider/modelZ,modelZ,0.001,0.002,0.003,0.004

evaluate/report/charts/script.js

@@ -0,0 +1,13 @@
+import { loadCSVFile, modelsWithCostsAndScores } from "./data.js";
+import { evaluationTable, evaluationScatterPlot } from "./charts.js";
+
+const evaluationRecords = await loadCSVFile("evaluation.csv");
+const metaInformationRecords = await loadCSVFile("meta.csv");
+
+const modelsWithCostsAndScoresRecords = modelsWithCostsAndScores(
+    evaluationRecords,
+    metaInformationRecords
+);
+
+evaluationTable(evaluationRecords);
+evaluationScatterPlot(modelsWithCostsAndScoresRecords);

evaluate/report/charts/styles.css

@@ -0,0 +1,10 @@
+html,
+body {
+	height: 100%;
+	margin: 0;
+}
+
+.grid line {
+	stroke: #e0e0e0;
+	stroke-width: 1px;
+}