refactor, Specific result type for test execution to make adding new …

…data easier

evaluate/task/symflower-fix.go

@@ -42,18 +42,18 @@ func ExecuteWithSymflowerFix(ctx evaltask.Context, logger *log.Logger, packagePa
 		return nil, problems, pkgerrors.WithStack(err)
 	}
 
-	coverage, ps, err := ctx.Language.Execute(logger, packagePath)
+	testResult, ps, err := ctx.Language.ExecuteTests(logger, packagePath)
 	problems = append(problems, ps...)
 	if err != nil {
 		return nil, problems, pkgerrors.WithMessage(err, "symflower fix")
 	}
-	logger.Printf("with symflower repair: Executes tests with %d coverage objects", coverage)
+	logger.Printf("with symflower repair: Executes tests with %d coverage objects", testResult.Coverage)
 
 	// Symflower was able to fix a failure so now update the assessment with the improved results.
 	assessments = metrics.NewAssessments()
 	assessments[metrics.AssessmentKeyProcessingTime] = duration
 	assessments.Award(metrics.AssessmentKeyFilesExecuted)
-	assessments.AwardPoints(metrics.AssessmentKeyCoverage, coverage)
+	assessments.AwardPoints(metrics.AssessmentKeyCoverage, testResult.Coverage)
 
 	return assessments, problems, nil
 }

evaluate/task/task-code-repair.go

@@ -94,16 +94,16 @@ func (t *TaskCodeRepair) Run(ctx evaltask.Context) (repositoryAssessment map[eva
 		modelAssessment.Add(assessments)
 		modelAssessment.Award(metrics.AssessmentKeyResponseNoError)
 
-		coverage, ps, err := ctx.Language.Execute(taskLogger.Logger, packagePath)
+		testResult, ps, err := ctx.Language.ExecuteTests(taskLogger.Logger, packagePath)
 		problems = append(problems, ps...)
 		if err != nil {
 			problems = append(problems, pkgerrors.WithMessage(err, sourceFile))
 
 			continue
 		}
-		taskLogger.Printf("Executes tests with %d coverage objects", coverage)
+		taskLogger.Printf("Executes tests with %d coverage objects", testResult.Coverage)
 		modelAssessment.Award(metrics.AssessmentKeyFilesExecuted)
-		modelAssessment.AwardPoints(metrics.AssessmentKeyCoverage, coverage)
+		modelAssessment.AwardPoints(metrics.AssessmentKeyCoverage, testResult.Coverage)
 	}
 
 	repositoryAssessment = map[evaltask.Identifier]metrics.Assessments{

evaluate/task/task-write-test.go

@@ -81,7 +81,7 @@ func (t *TaskWriteTests) Run(ctx evaltask.Context) (repositoryAssessment map[eva
 		modelAssessmentForFile.Add(assessments)
 		modelAssessmentForFile.Award(metrics.AssessmentKeyResponseNoError)
 
-		coverage, ps, err := ctx.Language.Execute(taskLogger.Logger, dataPath)
+		testResult, ps, err := ctx.Language.ExecuteTests(taskLogger.Logger, dataPath)
 		problems = append(problems, ps...)
 		if err != nil {
 			problems = append(problems, pkgerrors.WithMessage(err, filePath))
@@ -110,9 +110,9 @@ func (t *TaskWriteTests) Run(ctx evaltask.Context) (repositoryAssessment map[eva
 				}
 			}
 		} else {
-			taskLogger.Printf("Executes tests with %d coverage objects", coverage)
+			taskLogger.Printf("Executes tests with %d coverage objects", testResult.Coverage)
 			modelAssessmentForFile.Award(metrics.AssessmentKeyFilesExecuted)
-			modelAssessmentForFile.AwardPoints(metrics.AssessmentKeyCoverage, coverage)
+			modelAssessmentForFile.AwardPoints(metrics.AssessmentKeyCoverage, testResult.Coverage)
 		}
 
 		modelAssessment.Add(modelAssessmentForFile)

evaluate/task/task-write-test_test.go

@@ -207,7 +207,7 @@ func TestTaskWriteTestsRun(t *testing.T) {
 
 				languageMock := languagetesting.NewMockLanguageNamed(t, "golang")
 				languageMock.On("Files", mock.Anything, mock.Anything).Return([]string{filepath.Join("golang", "plain")}, nil).Once()
-				languageMock.On("Execute", mock.Anything, mock.Anything).Return(uint64(0), nil, context.DeadlineExceeded).Once()
+				languageMock.On("ExecuteTests", mock.Anything, mock.Anything).Return(nil, nil, context.DeadlineExceeded).Once()
 
 				validateGo(t, "Execution timeout", languageMock, "", expectedAssessments, expectedProblems, false)
 			}

language/golang/language.go

@@ -87,8 +87,8 @@ func (l *Language) DefaultTestFileSuffix() string {
 
 var languageGoTestsErrorMatch = regexp.MustCompile(`DONE (\d+) tests, (\d+) failure`)
 
-// Execute invokes the language specific testing on the given repository.
-func (l *Language) Execute(logger *log.Logger, repositoryPath string) (coverage uint64, problems []error, err error) {
+// ExecuteTests invokes the language specific testing on the given repository.
+func (l *Language) ExecuteTests(logger *log.Logger, repositoryPath string) (testResult *language.TestResult, problems []error, err error) {
 	commandOutput, err := util.CommandWithResult(context.Background(), logger, &util.Command{
 		Command: []string{
 			"go",
@@ -99,7 +99,7 @@ func (l *Language) Execute(logger *log.Logger, repositoryPath string) (coverage
 		Directory: repositoryPath,
 	})
 	if err != nil {
-		return 0, problems, pkgerrors.WithMessage(pkgerrors.WithStack(err), commandOutput)
+		return nil, nil, pkgerrors.WithMessage(pkgerrors.WithStack(err), commandOutput)
 	}
 
 	ctx, cancel := context.WithTimeout(context.Background(), language.DefaultExecutionTimeout)
@@ -119,22 +119,22 @@ func (l *Language) Execute(logger *log.Logger, repositoryPath string) (coverage
 		testSummary := languageGoTestsErrorMatch.FindStringSubmatch(commandOutput)
 		if len(testSummary) > 0 {
 			if failureCount, e := strconv.Atoi(testSummary[2]); e != nil {
-				return 0, problems, pkgerrors.WithStack(e)
+				return nil, nil, pkgerrors.WithStack(e)
 			} else if failureCount > 0 {
 				// If there are test failures, then this is just a soft error since we still are able to receive coverage data.
 				problems = append(problems, pkgerrors.WithMessage(pkgerrors.WithStack(err), commandOutput))
 			}
 		} else {
-			return 0, problems, pkgerrors.WithMessage(pkgerrors.WithStack(err), commandOutput)
+			return nil, nil, pkgerrors.WithMessage(pkgerrors.WithStack(err), commandOutput)
 		}
 	}
 
-	coverage, err = language.CoverageObjectCountOfFile(logger, coverageFilePath)
+	testResult.Coverage, err = language.CoverageObjectCountOfFile(logger, coverageFilePath)
 	if err != nil {
-		return 0, problems, pkgerrors.WithMessage(pkgerrors.WithStack(err), commandOutput)
+		return testResult, problems, pkgerrors.WithMessage(pkgerrors.WithStack(err), commandOutput)
 	}
 
-	return coverage, problems, nil
+	return testResult, problems, nil
 }
 
 // Mistakes builds a Go repository and returns the list of mistakes found.

language/golang/language_test.go

@@ -10,6 +10,7 @@ import (
 	"github.com/zimmski/osutil"
 	"github.com/zimmski/osutil/bytesutil"
 
+	"github.com/symflower/eval-dev-quality/language"
 	languagetesting "github.com/symflower/eval-dev-quality/language/testing"
 	"github.com/symflower/eval-dev-quality/log"
 )
@@ -57,25 +58,27 @@ func TestLanguageFiles(t *testing.T) {
 }
 
 func TestLanguageExecute(t *testing.T) {
-	validate := func(t *testing.T, tc *languagetesting.TestCaseExecute) {
+	validate := func(t *testing.T, tc *languagetesting.TestCaseExecuteTests) {
 		if tc.Language == nil {
 			tc.Language = &Language{}
 		}
 
 		tc.Validate(t)
 	}
 
-	validate(t, &languagetesting.TestCaseExecute{
+	validate(t, &languagetesting.TestCaseExecuteTests{
 		Name: "No test files",
 
 		RepositoryPath: filepath.Join("..", "..", "testdata", "golang", "plain"),
 
-		ExpectedCoverage:  0,
+		ExpectedTestResult: &language.TestResult{
+			Coverage: 0,
+		},
 		ExpectedErrorText: "exit status 1",
 	})
 
 	t.Run("With test file", func(t *testing.T) {
-		validate(t, &languagetesting.TestCaseExecute{
+		validate(t, &languagetesting.TestCaseExecuteTests{
 			Name: "Valid",
 
 			RepositoryPath: filepath.Join("..", "..", "testdata", "golang", "plain"),
@@ -93,10 +96,12 @@ func TestLanguageExecute(t *testing.T) {
 				`)), 0660))
 			},
 
-			ExpectedCoverage: 1,
+			ExpectedTestResult: &language.TestResult{
+				Coverage: 1,
+			},
 		})
 
-		validate(t, &languagetesting.TestCaseExecute{
+		validate(t, &languagetesting.TestCaseExecuteTests{
 			Name: "Failing tests",
 
 			RepositoryPath: filepath.Join("..", "..", "testdata", "golang", "light"),
@@ -115,13 +120,15 @@ func TestLanguageExecute(t *testing.T) {
 				`)), 0660))
 			},
 
-			ExpectedCoverage: 1,
+			ExpectedTestResult: &language.TestResult{
+				Coverage: 1,
+			},
 			ExpectedProblemTexts: []string{
 				"exit status 1", // Test execution fails.
 			},
 		})
 
-		validate(t, &languagetesting.TestCaseExecute{
+		validate(t, &languagetesting.TestCaseExecuteTests{
 			Name: "Syntax error",
 
 			RepositoryPath: filepath.Join("..", "..", "testdata", "golang", "plain"),

language/java/language.go

@@ -100,8 +100,8 @@ func (l *Language) DefaultTestFileSuffix() string {
 
 var languageJavaCoverageMatch = regexp.MustCompile(`Total coverage (.+?)%`)
 
-// Execute invokes the language specific testing on the given repository.
-func (l *Language) Execute(logger *log.Logger, repositoryPath string) (coverage uint64, problems []error, err error) {
+// ExecuteTests invokes the language specific testing on the given repository.
+func (l *Language) ExecuteTests(logger *log.Logger, repositoryPath string) (testResult *language.TestResult, problems []error, err error) {
 	ctx, cancel := context.WithTimeout(context.Background(), language.DefaultExecutionTimeout)
 	defer cancel()
 	coverageFilePath := filepath.Join(repositoryPath, "coverage.json")
@@ -116,15 +116,16 @@ func (l *Language) Execute(logger *log.Logger, repositoryPath string) (coverage
 		Directory: repositoryPath,
 	})
 	if err != nil {
-		return 0, nil, pkgerrors.WithMessage(pkgerrors.WithStack(err), commandOutput)
+		return nil, nil, pkgerrors.WithMessage(pkgerrors.WithStack(err), commandOutput)
 	}
 
-	coverage, err = language.CoverageObjectCountOfFile(logger, coverageFilePath)
+	testResult = &language.TestResult{}
+	testResult.Coverage, err = language.CoverageObjectCountOfFile(logger, coverageFilePath)
 	if err != nil {
-		return 0, nil, pkgerrors.WithMessage(pkgerrors.WithStack(err), commandOutput)
+		return nil, nil, pkgerrors.WithMessage(pkgerrors.WithStack(err), commandOutput)
 	}
 
-	return coverage, nil, nil
+	return testResult, nil, nil
 }
 
 // Mistakes builds a Java repository and returns the list of mistakes found.

language/java/language_test.go

@@ -10,6 +10,7 @@ import (
 	"github.com/zimmski/osutil"
 	"github.com/zimmski/osutil/bytesutil"
 
+	"github.com/symflower/eval-dev-quality/language"
 	languagetesting "github.com/symflower/eval-dev-quality/language/testing"
 	"github.com/symflower/eval-dev-quality/log"
 )
@@ -133,25 +134,27 @@ func TestLanguageTestFilePath(t *testing.T) {
 }
 
 func TestLanguageExecute(t *testing.T) {
-	validate := func(t *testing.T, tc *languagetesting.TestCaseExecute) {
+	validate := func(t *testing.T, tc *languagetesting.TestCaseExecuteTests) {
 		if tc.Language == nil {
 			tc.Language = &Language{}
 		}
 
 		tc.Validate(t)
 	}
 
-	validate(t, &languagetesting.TestCaseExecute{
+	validate(t, &languagetesting.TestCaseExecuteTests{
 		Name: "No test files",
 
 		RepositoryPath: filepath.Join("..", "..", "testdata", "java", "plain"),
 
-		ExpectedCoverage:  0,
+		ExpectedTestResult: &language.TestResult{
+			Coverage: 0,
+		},
 		ExpectedErrorText: "exit status 1",
 	})
 
 	t.Run("With test file", func(t *testing.T) {
-		validate(t, &languagetesting.TestCaseExecute{
+		validate(t, &languagetesting.TestCaseExecuteTests{
 			Name: "Valid",
 
 			RepositoryPath: filepath.Join("..", "..", "testdata", "java", "plain"),
@@ -172,10 +175,12 @@ func TestLanguageExecute(t *testing.T) {
 				`)), 0660))
 			},
 
-			ExpectedCoverage: 1,
+			ExpectedTestResult: &language.TestResult{
+				Coverage: 1,
+			},
 		})
 
-		validate(t, &languagetesting.TestCaseExecute{
+		validate(t, &languagetesting.TestCaseExecuteTests{
 			Name: "Failing tests",
 
 			RepositoryPath: filepath.Join("..", "..", "testdata", "java", "light"),
@@ -197,10 +202,12 @@ func TestLanguageExecute(t *testing.T) {
 				`)), 0660))
 			},
 
-			ExpectedCoverage: 3,
+			ExpectedTestResult: &language.TestResult{
+				Coverage: 3,
+			},
 		})
 
-		validate(t, &languagetesting.TestCaseExecute{
+		validate(t, &languagetesting.TestCaseExecuteTests{
 			Name: "Syntax error",
 
 			RepositoryPath: filepath.Join("..", "..", "testdata", "java", "plain"),

language/language.go

@@ -36,8 +36,8 @@ type Language interface {
 	// DefaultTestFileSuffix returns the default test file suffix of the implemented language.
 	DefaultTestFileSuffix() string
 
-	// Execute invokes the language specific testing on the given repository.
-	Execute(logger *log.Logger, repositoryPath string) (coverage uint64, problems []error, err error)
+	// ExecuteTests invokes the language specific testing on the given repository.
+	ExecuteTests(logger *log.Logger, repositoryPath string) (testResult *TestResult, problems []error, err error)
 	// Mistakes builds a repository and returns the list of mistakes found.
 	Mistakes(logger *log.Logger, repositoryPath string) (mistakes []string, err error)
 }
@@ -74,3 +74,8 @@ func RepositoriesForLanguage(language Language, testdataPath string) (relativeRe
 
 	return relativeRepositoryPaths, nil
 }
+
+// TestResult holds the result of running tests.
+type TestResult struct {
+	Coverage uint64
+}