From c477647c6be97f4f1c547b7c48cc65128ea9f77b Mon Sep 17 00:00:00 2001 From: Rui Azevedo Date: Tue, 30 Jul 2024 08:44:31 +0100 Subject: [PATCH] HTML report for data visualization Part of #296 --- evaluate/report/charts/charts.js | 124 ++++++++++++++++++++++++++ evaluate/report/charts/data.js | 51 +++++++++++ evaluate/report/charts/evaluation.csv | 7 ++ evaluate/report/charts/index.html | 47 ++++++++++ evaluate/report/charts/meta.csv | 4 + evaluate/report/charts/script.js | 13 +++ evaluate/report/charts/styles.css | 10 +++ 7 files changed, 256 insertions(+) create mode 100644 evaluate/report/charts/charts.js create mode 100644 evaluate/report/charts/data.js create mode 100644 evaluate/report/charts/evaluation.csv create mode 100644 evaluate/report/charts/index.html create mode 100644 evaluate/report/charts/meta.csv create mode 100644 evaluate/report/charts/script.js create mode 100644 evaluate/report/charts/styles.css diff --git a/evaluate/report/charts/charts.js b/evaluate/report/charts/charts.js new file mode 100644 index 00000000..d05b4b75 --- /dev/null +++ b/evaluate/report/charts/charts.js @@ -0,0 +1,124 @@ +// evaluationTable creates a table given an evaluation CSV file. +function evaluationTable(data) { + const table = $("#evaluation-table"); + const columns = data.columns; + + // Create the header. + let thead = ""; + columns.forEach((col) => { + thead += `${col}`; + }); + thead += ""; + table.append(thead); + + // Create the body. + let tbody = ""; + data.forEach((row) => { + tbody += ""; + columns.forEach((col) => { + tbody += `${row[col]}`; + }); + tbody += ""; + }); + tbody += ""; + table.append(tbody); + + // Initialize the DataTable. + table.DataTable({ + order: [[0, "asc"]], + dom: "lfrtip", + }); +} + +// evaluationScatterPlot creates a scatter plot comparing models costs and scores. +function evaluationScatterPlot(data) { + // Set dimensions and margins for the scatter plot. + const margin = { top: 20, right: 30, bottom: 50, left: 70 }, + width = 800, + height = 600; + + // Append the SVG element to the body. + const svg = d3 + .select("#evaluation-scatter") + .append("svg") + .attr("width", width + margin.left + margin.right) + .attr("height", height + margin.top + margin.bottom) + .append("g") + .attr("transform", `translate(${margin.left},${margin.top})`); + + // Set up the x and y scales. + const x = d3 + .scaleLinear() + .domain([0, d3.max(data, (d) => d["model-costs"]) * 1.1]) + .range([0, width]); + + const y = d3 + .scaleLinear() + .domain([ + d3.min(data, (d) => d.score), + d3.max(data, (d) => d.score * 1.1), + ]) + .range([height, 0]); + + // Add the grid lines for the x-axis. + const xAxisGrid = d3.axisBottom(x).tickSize(-height).tickFormat(""); + svg.append("g") + .attr("class", "grid") + .attr("transform", `translate(0, ${height})`) + .call(xAxisGrid); + + // Add the grid lines for the y-axis. + const yAxisGrid = d3.axisLeft(y).tickSize(-width).tickFormat(""); + svg.append("g").attr("class", "grid").call(yAxisGrid); + + // Add the x-axis. + svg.append("g") + .attr("transform", `translate(0,${height})`) + .call(d3.axisBottom(x)) + .append("text") + .attr("x", width / 2) + .attr("y", margin.bottom - 10) + .attr("fill", "black") + .attr("text-anchor", "middle") + .style("font-size", "16px") + .text("Costs per token"); + + // Add the y-axis. + svg.append("g") + .call(d3.axisLeft(y)) + .append("text") + .attr("transform", "rotate(-90)") + .attr("x", -height / 2) + .attr("y", -margin.left + 15) + .attr("fill", "black") + .attr("text-anchor", "middle") + .style("font-size", "16px") + .text("Score"); + + // Add points. + svg.append("g") + .selectAll("circle") + .data(data) + .enter() + .append("circle") + .attr("cx", (d) => x(d["model-costs"])) + .attr("cy", (d) => y(d.score)) + .attr("r", 5) + .attr("fill", "green"); + + // Add labels. + svg.append("g") + .selectAll("text") + .data(data) + .enter() + .append("text") + .attr("x", (d) => x(d["model-costs"])) + .attr("y", (d) => y(d.score)) + .attr("dy", -10) + .attr("text-anchor", "middle") + .text((d) => d["model-name"]) + .attr("font-size", "16px") + .attr("fill", "black"); +} + +export { evaluationTable, evaluationScatterPlot }; diff --git a/evaluate/report/charts/data.js b/evaluate/report/charts/data.js new file mode 100644 index 00000000..57c46dc7 --- /dev/null +++ b/evaluate/report/charts/data.js @@ -0,0 +1,51 @@ +// loadCSVFile loads a CSV file and returns the list of raw records. +async function loadCSVFile(filepath) { + try { + const records = d3.csv(filepath); + + return records; + } catch (error) { + console.error("error loading CSV file: ", error); + } +} + +// modelsWithCostsAndScores returns an array of objects with model names, costs and scores. +function modelsWithCostsAndScores(evaluationRecords, metaInformationRecords) { + // Group models and sum scores. + const modelsWithSummedScores = {}; + evaluationRecords.forEach((record) => { + const modelId = record["model-id"]; + const score = +record["score"]; + + if (!modelsWithSummedScores[modelId]) { + modelsWithSummedScores[modelId] = score; + } else { + modelsWithSummedScores[modelId] += score; + } + }); + + // Add costs and human-readable name. + const models = {}; + for (const modelId in modelsWithSummedScores) { + models[modelId] = { + "model-name": modelId, + "model-costs": 0.0, + score: modelsWithSummedScores[modelId], + }; + } + metaInformationRecords.forEach((record) => { + const modelId = record["model-id"]; + if (models[modelId]) { + models[modelId]["model-name"] = record["model-name"]; + models[modelId]["model-costs"] = + +record.completion + + +record.image + + +record.prompt + + +record.request; + } + }); + + return Object.values(models); +} + +export { loadCSVFile, modelsWithCostsAndScores }; diff --git a/evaluate/report/charts/evaluation.csv b/evaluate/report/charts/evaluation.csv new file mode 100644 index 00000000..f4f06600 --- /dev/null +++ b/evaluate/report/charts/evaluation.csv @@ -0,0 +1,7 @@ +model-id,language,repository,task,score,coverage,files-executed,files-executed-maximum-reachable,generate-tests-for-file-character-count,processing-time,response-character-count,response-no-error,response-no-excess,response-with-code,tests-passing +provider/modelA,java,java/plain,write-tests,10,0,0,0,0,0,0,0,0,0,0 +provider/modelA,golang,golang/plain,write-tests,10,0,0,0,0,0,0,0,0,0,0 +provider/modelB,java,java/plain,write-tests,20,0,0,0,0,0,0,0,0,0,0 +provider/modelB,golang,golang/plain,write-tests,30,0,0,0,0,0,0,0,0,0,0 +provider/modelB,java,java/light,write-tests,40,0,0,0,0,0,0,0,0,0,0 +provider/modelC,java,java/plain,write-tests,10,0,0,0,0,0,0,0,0,0,0 diff --git a/evaluate/report/charts/index.html b/evaluate/report/charts/index.html new file mode 100644 index 00000000..1f4e4307 --- /dev/null +++ b/evaluate/report/charts/index.html @@ -0,0 +1,47 @@ + + + + + + + EvalDevQuality + + + + + + +
+

EvalDevQuality

+ +
+
+
+
+
+
+
+
+
+
+
+
+
+ + + + + + + + + diff --git a/evaluate/report/charts/meta.csv b/evaluate/report/charts/meta.csv new file mode 100644 index 00000000..b1add641 --- /dev/null +++ b/evaluate/report/charts/meta.csv @@ -0,0 +1,4 @@ +model-id,model-name,completion,image,prompt,request +provider/modelA,modelA,0.1,0.2,0.3,0.4 +provider/modelB,modelB,0.01,0.02,0.03,0.04 +provider/modelZ,modelZ,0.001,0.002,0.003,0.004 diff --git a/evaluate/report/charts/script.js b/evaluate/report/charts/script.js new file mode 100644 index 00000000..c3c8cbeb --- /dev/null +++ b/evaluate/report/charts/script.js @@ -0,0 +1,13 @@ +import { loadCSVFile, modelsWithCostsAndScores } from "./data.js"; +import { evaluationTable, evaluationScatterPlot } from "./charts.js"; + +const evaluationRecords = await loadCSVFile("evaluation.csv"); +const metaInformationRecords = await loadCSVFile("meta.csv"); + +const modelsWithCostsAndScoresRecords = modelsWithCostsAndScores( + evaluationRecords, + metaInformationRecords +); + +evaluationTable(evaluationRecords); +evaluationScatterPlot(modelsWithCostsAndScoresRecords); diff --git a/evaluate/report/charts/styles.css b/evaluate/report/charts/styles.css new file mode 100644 index 00000000..c1b5ad34 --- /dev/null +++ b/evaluate/report/charts/styles.css @@ -0,0 +1,10 @@ +html, +body { + height: 100%; + margin: 0; +} + +.grid line { + stroke: #e0e0e0; + stroke-width: 1px; +}