From 496418c0385292144f2e10bcc538eeae7b476410 Mon Sep 17 00:00:00 2001 From: Rui Azevedo Date: Tue, 9 Jul 2024 16:29:53 +0100 Subject: [PATCH] Store the task identifier and the repository name when reading the evaluation CSV file, since they are part of an evaluation record Part of #237 --- evaluate/report/csv.go | 11 +++++++++++ evaluate/report/csv_test.go | 32 +++++++++++++++++++------------- 2 files changed, 30 insertions(+), 13 deletions(-) diff --git a/evaluate/report/csv.go b/evaluate/report/csv.go index 17e44615..7e3fcc2f 100644 --- a/evaluate/report/csv.go +++ b/evaluate/report/csv.go @@ -85,6 +85,11 @@ type EvaluationRecord struct { // LanguageID holds the language id. LanguageID string + // RepositoryName holds the name of a repository . + RepositoryName string + // Task holds the task identifier. + Task string + // Assessments holds the assessments of an entry. Assessments metrics.Assessments } @@ -225,6 +230,9 @@ func convertRawRecordToEvaluationRecord(raw []string) (record *EvaluationRecord, languageID := raw[3] + repositoryName := raw[4] + task := raw[5] + rawMetrics := raw[7:] for i, assessementKey := range metrics.AllAssessmentKeysStrings { metric, err := strconv.ParseUint(rawMetrics[i], 10, 64) @@ -242,6 +250,9 @@ func convertRawRecordToEvaluationRecord(raw []string) (record *EvaluationRecord, LanguageID: languageID, + RepositoryName: repositoryName, + Task: task, + Assessments: assessments, }, nil } diff --git a/evaluate/report/csv_test.go b/evaluate/report/csv_test.go index b8b6873f..02bf5099 100644 --- a/evaluate/report/csv_test.go +++ b/evaluate/report/csv_test.go @@ -257,10 +257,12 @@ func TestLoadEvaluationRecords(t *testing.T) { ExpectedEvaluationRecords: EvaluationRecords{ &EvaluationRecord{ - ModelID: "openrouter/anthropic/claude-1.2", - ModelName: "Claude 1.2", - ModelCost: 0.0001, - LanguageID: "golang", + ModelID: "openrouter/anthropic/claude-1.2", + ModelName: "Claude 1.2", + ModelCost: 0.0001, + LanguageID: "golang", + RepositoryName: "golang/light", + Task: "write-tests", Assessments: metrics.Assessments{ metrics.AssessmentKeyCoverage: 750, metrics.AssessmentKeyFilesExecuted: 18, @@ -281,17 +283,19 @@ func TestLoadEvaluationRecords(t *testing.T) { fileContent := bytesutil.StringTrimIndentations(` model-id,model-name,cost,language,repository,task,score,coverage,files-executed,generate-tests-for-file-character-count,processing-time,response-character-count,response-no-error,response-no-excess,response-with-code openrouter/anthropic/claude-1.2,Claude 1.2,0.0001,golang,golang/light,write-tests,982,750,18,70179,720571,71195,115,49,50 - openrouter/anthropic/claude-1.2,Claude 1.2,0.0002,golang,golang/plain,write-tests,37,20,2,441,11042,523,5,5,5 + openrouter/anthropic/claude-1.2,Claude 1.2,0.0002,golang,golang/plain,transpile,37,20,2,441,11042,523,5,5,5 `) require.NoError(t, os.WriteFile(filepath.Join(resultPath, "evaluation.csv"), []byte(fileContent), 0644)) }, ExpectedEvaluationRecords: EvaluationRecords{ &EvaluationRecord{ - ModelID: "openrouter/anthropic/claude-1.2", - ModelName: "Claude 1.2", - ModelCost: 0.0001, - LanguageID: "golang", + ModelID: "openrouter/anthropic/claude-1.2", + ModelName: "Claude 1.2", + ModelCost: 0.0001, + LanguageID: "golang", + RepositoryName: "golang/light", + Task: "write-tests", Assessments: metrics.Assessments{ metrics.AssessmentKeyCoverage: 750, metrics.AssessmentKeyFilesExecuted: 18, @@ -304,10 +308,12 @@ func TestLoadEvaluationRecords(t *testing.T) { }, }, &EvaluationRecord{ - ModelID: "openrouter/anthropic/claude-1.2", - ModelName: "Claude 1.2", - ModelCost: 0.0002, - LanguageID: "golang", + ModelID: "openrouter/anthropic/claude-1.2", + ModelName: "Claude 1.2", + ModelCost: 0.0002, + LanguageID: "golang", + RepositoryName: "golang/plain", + Task: "transpile", Assessments: metrics.Assessments{ metrics.AssessmentKeyCoverage: 20, metrics.AssessmentKeyFilesExecuted: 2,