Store the task identifier and the repository name when reading the ev…

…aluation CSV file, since they are part of an evaluation record Part of #237
symflower · Jul 9, 2024 · 3f091f9 · 3f091f9
1 parent c07ed10
commit 3f091f9
Show file tree

Hide file tree

Showing 2 changed files with 30 additions and 13 deletions.
diff --git a/evaluate/report/csv.go b/evaluate/report/csv.go
@@ -85,6 +85,11 @@ type EvaluationRecord struct {
 	// LanguageID holds the language id.
 	LanguageID string
 
+	// RepositoryName holds the name of a repository .
+	RepositoryName string
+	// Task holds the task identifier.
+	Task string
+
 	// Assessments holds the assessments of an entry.
 	Assessments metrics.Assessments
 }
@@ -225,6 +230,9 @@ func convertRawRecordToEvaluationRecord(raw []string) (record *EvaluationRecord,
 
 	languageID := raw[3]
 
+	repositoryName := raw[4]
+	task := raw[5]
+
 	rawMetrics := raw[7:]
 	for i, assessementKey := range metrics.AllAssessmentKeysStrings {
 		metric, err := strconv.ParseUint(rawMetrics[i], 10, 64)
@@ -242,6 +250,9 @@ func convertRawRecordToEvaluationRecord(raw []string) (record *EvaluationRecord,
 
 		LanguageID: languageID,
 
+		RepositoryName: repositoryName,
+		Task:           task,
+
 		Assessments: assessments,
 	}, nil
 }

diff --git a/evaluate/report/csv_test.go b/evaluate/report/csv_test.go
@@ -257,10 +257,12 @@ func TestLoadEvaluationRecords(t *testing.T) {
 
 		ExpectedEvaluationRecords: EvaluationRecords{
 			&EvaluationRecord{
-				ModelID:    "openrouter/anthropic/claude-1.2",
-				ModelName:  "Claude 1.2",
-				ModelCost:  0.0001,
-				LanguageID: "golang",
+				ModelID:        "openrouter/anthropic/claude-1.2",
+				ModelName:      "Claude 1.2",
+				ModelCost:      0.0001,
+				LanguageID:     "golang",
+				RepositoryName: "golang/light",
+				Task:           "write-tests",
 				Assessments: metrics.Assessments{
 					metrics.AssessmentKeyCoverage:                           750,
 					metrics.AssessmentKeyFilesExecuted:                      18,
@@ -281,17 +283,19 @@ func TestLoadEvaluationRecords(t *testing.T) {
 			fileContent := bytesutil.StringTrimIndentations(`
 					model-id,model-name,cost,language,repository,task,score,coverage,files-executed,generate-tests-for-file-character-count,processing-time,response-character-count,response-no-error,response-no-excess,response-with-code
 					openrouter/anthropic/claude-1.2,Claude 1.2,0.0001,golang,golang/light,write-tests,982,750,18,70179,720571,71195,115,49,50
-					openrouter/anthropic/claude-1.2,Claude 1.2,0.0002,golang,golang/plain,write-tests,37,20,2,441,11042,523,5,5,5
+					openrouter/anthropic/claude-1.2,Claude 1.2,0.0002,golang,golang/plain,transpile,37,20,2,441,11042,523,5,5,5
 				`)
 			require.NoError(t, os.WriteFile(filepath.Join(resultPath, "evaluation.csv"), []byte(fileContent), 0644))
 		},
 
 		ExpectedEvaluationRecords: EvaluationRecords{
 			&EvaluationRecord{
-				ModelID:    "openrouter/anthropic/claude-1.2",
-				ModelName:  "Claude 1.2",
-				ModelCost:  0.0001,
-				LanguageID: "golang",
+				ModelID:        "openrouter/anthropic/claude-1.2",
+				ModelName:      "Claude 1.2",
+				ModelCost:      0.0001,
+				LanguageID:     "golang",
+				RepositoryName: "golang/light",
+				Task:           "write-tests",
 				Assessments: metrics.Assessments{
 					metrics.AssessmentKeyCoverage:                           750,
 					metrics.AssessmentKeyFilesExecuted:                      18,
@@ -304,10 +308,12 @@ func TestLoadEvaluationRecords(t *testing.T) {
 				},
 			},
 			&EvaluationRecord{
-				ModelID:    "openrouter/anthropic/claude-1.2",
-				ModelName:  "Claude 1.2",
-				ModelCost:  0.0002,
-				LanguageID: "golang",
+				ModelID:        "openrouter/anthropic/claude-1.2",
+				ModelName:      "Claude 1.2",
+				ModelCost:      0.0002,
+				LanguageID:     "golang",
+				RepositoryName: "golang/plain",
+				Task:           "transpile",
 				Assessments: metrics.Assessments{
 					metrics.AssessmentKeyCoverage:                           20,
 					metrics.AssessmentKeyFilesExecuted:                      2,