From 02fed50f3ac25111a18a89091b379e416809d9c1 Mon Sep 17 00:00:00 2001 From: Andreas Humenberger Date: Wed, 17 Jul 2024 08:42:15 +0200 Subject: [PATCH] refactor, Specific result type for test execution to make adding new data easier --- evaluate/task/symflower-fix.go | 6 +++--- evaluate/task/task-code-repair.go | 6 +++--- evaluate/task/task-write-test.go | 6 +++--- evaluate/task/task-write-test_test.go | 2 +- language/golang/language.go | 16 ++++++++-------- language/golang/language_test.go | 23 +++++++++++++++-------- language/java/language.go | 13 +++++++------ language/java/language_test.go | 23 +++++++++++++++-------- language/language.go | 9 +++++++-- language/testing/Language_mock_gen.go | 20 ++++++++++++-------- language/testing/language.go | 10 +++++----- model/symflower/symflower_test.go | 4 ++-- 12 files changed, 81 insertions(+), 57 deletions(-) diff --git a/evaluate/task/symflower-fix.go b/evaluate/task/symflower-fix.go index 01003b1c..a63c2412 100644 --- a/evaluate/task/symflower-fix.go +++ b/evaluate/task/symflower-fix.go @@ -42,18 +42,18 @@ func ExecuteWithSymflowerFix(ctx evaltask.Context, logger *log.Logger, packagePa return nil, problems, pkgerrors.WithStack(err) } - coverage, ps, err := ctx.Language.Execute(logger, packagePath) + testResult, ps, err := ctx.Language.ExecuteTests(logger, packagePath) problems = append(problems, ps...) if err != nil { return nil, problems, pkgerrors.WithMessage(err, "symflower fix") } - logger.Printf("with symflower repair: Executes tests with %d coverage objects", coverage) + logger.Printf("with symflower repair: Executes tests with %d coverage objects", testResult.Coverage) // Symflower was able to fix a failure so now update the assessment with the improved results. assessments = metrics.NewAssessments() assessments[metrics.AssessmentKeyProcessingTime] = duration assessments.Award(metrics.AssessmentKeyFilesExecuted) - assessments.AwardPoints(metrics.AssessmentKeyCoverage, coverage) + assessments.AwardPoints(metrics.AssessmentKeyCoverage, testResult.Coverage) return assessments, problems, nil } diff --git a/evaluate/task/task-code-repair.go b/evaluate/task/task-code-repair.go index 103113f5..59dd9ae9 100644 --- a/evaluate/task/task-code-repair.go +++ b/evaluate/task/task-code-repair.go @@ -94,16 +94,16 @@ func (t *TaskCodeRepair) Run(ctx evaltask.Context) (repositoryAssessment map[eva modelAssessment.Add(assessments) modelAssessment.Award(metrics.AssessmentKeyResponseNoError) - coverage, ps, err := ctx.Language.Execute(taskLogger.Logger, packagePath) + testResult, ps, err := ctx.Language.ExecuteTests(taskLogger.Logger, packagePath) problems = append(problems, ps...) if err != nil { problems = append(problems, pkgerrors.WithMessage(err, sourceFile)) continue } - taskLogger.Printf("Executes tests with %d coverage objects", coverage) + taskLogger.Printf("Executes tests with %d coverage objects", testResult.Coverage) modelAssessment.Award(metrics.AssessmentKeyFilesExecuted) - modelAssessment.AwardPoints(metrics.AssessmentKeyCoverage, coverage) + modelAssessment.AwardPoints(metrics.AssessmentKeyCoverage, testResult.Coverage) } repositoryAssessment = map[evaltask.Identifier]metrics.Assessments{ diff --git a/evaluate/task/task-write-test.go b/evaluate/task/task-write-test.go index e63d6fe3..1215a72f 100644 --- a/evaluate/task/task-write-test.go +++ b/evaluate/task/task-write-test.go @@ -81,7 +81,7 @@ func (t *TaskWriteTests) Run(ctx evaltask.Context) (repositoryAssessment map[eva modelAssessmentForFile.Add(assessments) modelAssessmentForFile.Award(metrics.AssessmentKeyResponseNoError) - coverage, ps, err := ctx.Language.Execute(taskLogger.Logger, dataPath) + testResult, ps, err := ctx.Language.ExecuteTests(taskLogger.Logger, dataPath) problems = append(problems, ps...) if err != nil { problems = append(problems, pkgerrors.WithMessage(err, filePath)) @@ -110,9 +110,9 @@ func (t *TaskWriteTests) Run(ctx evaltask.Context) (repositoryAssessment map[eva } } } else { - taskLogger.Printf("Executes tests with %d coverage objects", coverage) + taskLogger.Printf("Executes tests with %d coverage objects", testResult.Coverage) modelAssessmentForFile.Award(metrics.AssessmentKeyFilesExecuted) - modelAssessmentForFile.AwardPoints(metrics.AssessmentKeyCoverage, coverage) + modelAssessmentForFile.AwardPoints(metrics.AssessmentKeyCoverage, testResult.Coverage) } modelAssessment.Add(modelAssessmentForFile) diff --git a/evaluate/task/task-write-test_test.go b/evaluate/task/task-write-test_test.go index 419d9ea5..2ea864b4 100644 --- a/evaluate/task/task-write-test_test.go +++ b/evaluate/task/task-write-test_test.go @@ -207,7 +207,7 @@ func TestTaskWriteTestsRun(t *testing.T) { languageMock := languagetesting.NewMockLanguageNamed(t, "golang") languageMock.On("Files", mock.Anything, mock.Anything).Return([]string{filepath.Join("golang", "plain")}, nil).Once() - languageMock.On("Execute", mock.Anything, mock.Anything).Return(uint64(0), nil, context.DeadlineExceeded).Once() + languageMock.On("ExecuteTests", mock.Anything, mock.Anything).Return(nil, nil, context.DeadlineExceeded).Once() validateGo(t, "Execution timeout", languageMock, "", expectedAssessments, expectedProblems, false) } diff --git a/language/golang/language.go b/language/golang/language.go index 6033de08..56779313 100644 --- a/language/golang/language.go +++ b/language/golang/language.go @@ -87,8 +87,8 @@ func (l *Language) DefaultTestFileSuffix() string { var languageGoTestsErrorMatch = regexp.MustCompile(`DONE (\d+) tests, (\d+) failure`) -// Execute invokes the language specific testing on the given repository. -func (l *Language) Execute(logger *log.Logger, repositoryPath string) (coverage uint64, problems []error, err error) { +// ExecuteTests invokes the language specific testing on the given repository. +func (l *Language) ExecuteTests(logger *log.Logger, repositoryPath string) (testResult *language.TestResult, problems []error, err error) { commandOutput, err := util.CommandWithResult(context.Background(), logger, &util.Command{ Command: []string{ "go", @@ -99,7 +99,7 @@ func (l *Language) Execute(logger *log.Logger, repositoryPath string) (coverage Directory: repositoryPath, }) if err != nil { - return 0, problems, pkgerrors.WithMessage(pkgerrors.WithStack(err), commandOutput) + return nil, nil, pkgerrors.WithMessage(pkgerrors.WithStack(err), commandOutput) } ctx, cancel := context.WithTimeout(context.Background(), language.DefaultExecutionTimeout) @@ -119,22 +119,22 @@ func (l *Language) Execute(logger *log.Logger, repositoryPath string) (coverage testSummary := languageGoTestsErrorMatch.FindStringSubmatch(commandOutput) if len(testSummary) > 0 { if failureCount, e := strconv.Atoi(testSummary[2]); e != nil { - return 0, problems, pkgerrors.WithStack(e) + return nil, nil, pkgerrors.WithStack(e) } else if failureCount > 0 { // If there are test failures, then this is just a soft error since we still are able to receive coverage data. problems = append(problems, pkgerrors.WithMessage(pkgerrors.WithStack(err), commandOutput)) } } else { - return 0, problems, pkgerrors.WithMessage(pkgerrors.WithStack(err), commandOutput) + return nil, nil, pkgerrors.WithMessage(pkgerrors.WithStack(err), commandOutput) } } - coverage, err = language.CoverageObjectCountOfFile(logger, coverageFilePath) + testResult.Coverage, err = language.CoverageObjectCountOfFile(logger, coverageFilePath) if err != nil { - return 0, problems, pkgerrors.WithMessage(pkgerrors.WithStack(err), commandOutput) + return testResult, problems, pkgerrors.WithMessage(pkgerrors.WithStack(err), commandOutput) } - return coverage, problems, nil + return testResult, problems, nil } // Mistakes builds a Go repository and returns the list of mistakes found. diff --git a/language/golang/language_test.go b/language/golang/language_test.go index 7b28b39c..39665a43 100644 --- a/language/golang/language_test.go +++ b/language/golang/language_test.go @@ -10,6 +10,7 @@ import ( "github.com/zimmski/osutil" "github.com/zimmski/osutil/bytesutil" + "github.com/symflower/eval-dev-quality/language" languagetesting "github.com/symflower/eval-dev-quality/language/testing" "github.com/symflower/eval-dev-quality/log" ) @@ -57,7 +58,7 @@ func TestLanguageFiles(t *testing.T) { } func TestLanguageExecute(t *testing.T) { - validate := func(t *testing.T, tc *languagetesting.TestCaseExecute) { + validate := func(t *testing.T, tc *languagetesting.TestCaseExecuteTests) { if tc.Language == nil { tc.Language = &Language{} } @@ -65,17 +66,19 @@ func TestLanguageExecute(t *testing.T) { tc.Validate(t) } - validate(t, &languagetesting.TestCaseExecute{ + validate(t, &languagetesting.TestCaseExecuteTests{ Name: "No test files", RepositoryPath: filepath.Join("..", "..", "testdata", "golang", "plain"), - ExpectedCoverage: 0, + ExpectedTestResult: &language.TestResult{ + Coverage: 0, + }, ExpectedErrorText: "exit status 1", }) t.Run("With test file", func(t *testing.T) { - validate(t, &languagetesting.TestCaseExecute{ + validate(t, &languagetesting.TestCaseExecuteTests{ Name: "Valid", RepositoryPath: filepath.Join("..", "..", "testdata", "golang", "plain"), @@ -93,10 +96,12 @@ func TestLanguageExecute(t *testing.T) { `)), 0660)) }, - ExpectedCoverage: 1, + ExpectedTestResult: &language.TestResult{ + Coverage: 1, + }, }) - validate(t, &languagetesting.TestCaseExecute{ + validate(t, &languagetesting.TestCaseExecuteTests{ Name: "Failing tests", RepositoryPath: filepath.Join("..", "..", "testdata", "golang", "light"), @@ -115,13 +120,15 @@ func TestLanguageExecute(t *testing.T) { `)), 0660)) }, - ExpectedCoverage: 1, + ExpectedTestResult: &language.TestResult{ + Coverage: 1, + }, ExpectedProblemTexts: []string{ "exit status 1", // Test execution fails. }, }) - validate(t, &languagetesting.TestCaseExecute{ + validate(t, &languagetesting.TestCaseExecuteTests{ Name: "Syntax error", RepositoryPath: filepath.Join("..", "..", "testdata", "golang", "plain"), diff --git a/language/java/language.go b/language/java/language.go index 323a617b..f8744c12 100644 --- a/language/java/language.go +++ b/language/java/language.go @@ -100,8 +100,8 @@ func (l *Language) DefaultTestFileSuffix() string { var languageJavaCoverageMatch = regexp.MustCompile(`Total coverage (.+?)%`) -// Execute invokes the language specific testing on the given repository. -func (l *Language) Execute(logger *log.Logger, repositoryPath string) (coverage uint64, problems []error, err error) { +// ExecuteTests invokes the language specific testing on the given repository. +func (l *Language) ExecuteTests(logger *log.Logger, repositoryPath string) (testResult *language.TestResult, problems []error, err error) { ctx, cancel := context.WithTimeout(context.Background(), language.DefaultExecutionTimeout) defer cancel() coverageFilePath := filepath.Join(repositoryPath, "coverage.json") @@ -116,15 +116,16 @@ func (l *Language) Execute(logger *log.Logger, repositoryPath string) (coverage Directory: repositoryPath, }) if err != nil { - return 0, nil, pkgerrors.WithMessage(pkgerrors.WithStack(err), commandOutput) + return nil, nil, pkgerrors.WithMessage(pkgerrors.WithStack(err), commandOutput) } - coverage, err = language.CoverageObjectCountOfFile(logger, coverageFilePath) + testResult = &language.TestResult{} + testResult.Coverage, err = language.CoverageObjectCountOfFile(logger, coverageFilePath) if err != nil { - return 0, nil, pkgerrors.WithMessage(pkgerrors.WithStack(err), commandOutput) + return nil, nil, pkgerrors.WithMessage(pkgerrors.WithStack(err), commandOutput) } - return coverage, nil, nil + return testResult, nil, nil } // Mistakes builds a Java repository and returns the list of mistakes found. diff --git a/language/java/language_test.go b/language/java/language_test.go index 8e2fcad3..ed99179d 100644 --- a/language/java/language_test.go +++ b/language/java/language_test.go @@ -10,6 +10,7 @@ import ( "github.com/zimmski/osutil" "github.com/zimmski/osutil/bytesutil" + "github.com/symflower/eval-dev-quality/language" languagetesting "github.com/symflower/eval-dev-quality/language/testing" "github.com/symflower/eval-dev-quality/log" ) @@ -133,7 +134,7 @@ func TestLanguageTestFilePath(t *testing.T) { } func TestLanguageExecute(t *testing.T) { - validate := func(t *testing.T, tc *languagetesting.TestCaseExecute) { + validate := func(t *testing.T, tc *languagetesting.TestCaseExecuteTests) { if tc.Language == nil { tc.Language = &Language{} } @@ -141,17 +142,19 @@ func TestLanguageExecute(t *testing.T) { tc.Validate(t) } - validate(t, &languagetesting.TestCaseExecute{ + validate(t, &languagetesting.TestCaseExecuteTests{ Name: "No test files", RepositoryPath: filepath.Join("..", "..", "testdata", "java", "plain"), - ExpectedCoverage: 0, + ExpectedTestResult: &language.TestResult{ + Coverage: 0, + }, ExpectedErrorText: "exit status 1", }) t.Run("With test file", func(t *testing.T) { - validate(t, &languagetesting.TestCaseExecute{ + validate(t, &languagetesting.TestCaseExecuteTests{ Name: "Valid", RepositoryPath: filepath.Join("..", "..", "testdata", "java", "plain"), @@ -172,10 +175,12 @@ func TestLanguageExecute(t *testing.T) { `)), 0660)) }, - ExpectedCoverage: 1, + ExpectedTestResult: &language.TestResult{ + Coverage: 1, + }, }) - validate(t, &languagetesting.TestCaseExecute{ + validate(t, &languagetesting.TestCaseExecuteTests{ Name: "Failing tests", RepositoryPath: filepath.Join("..", "..", "testdata", "java", "light"), @@ -197,10 +202,12 @@ func TestLanguageExecute(t *testing.T) { `)), 0660)) }, - ExpectedCoverage: 3, + ExpectedTestResult: &language.TestResult{ + Coverage: 3, + }, }) - validate(t, &languagetesting.TestCaseExecute{ + validate(t, &languagetesting.TestCaseExecuteTests{ Name: "Syntax error", RepositoryPath: filepath.Join("..", "..", "testdata", "java", "plain"), diff --git a/language/language.go b/language/language.go index f4a1601d..f2afda25 100644 --- a/language/language.go +++ b/language/language.go @@ -36,8 +36,8 @@ type Language interface { // DefaultTestFileSuffix returns the default test file suffix of the implemented language. DefaultTestFileSuffix() string - // Execute invokes the language specific testing on the given repository. - Execute(logger *log.Logger, repositoryPath string) (coverage uint64, problems []error, err error) + // ExecuteTests invokes the language specific testing on the given repository. + ExecuteTests(logger *log.Logger, repositoryPath string) (testResult *TestResult, problems []error, err error) // Mistakes builds a repository and returns the list of mistakes found. Mistakes(logger *log.Logger, repositoryPath string) (mistakes []string, err error) } @@ -74,3 +74,8 @@ func RepositoriesForLanguage(language Language, testdataPath string) (relativeRe return relativeRepositoryPaths, nil } + +// TestResult holds the result of running tests. +type TestResult struct { + Coverage uint64 +} diff --git a/language/testing/Language_mock_gen.go b/language/testing/Language_mock_gen.go index 5da4445d..cb541b2f 100644 --- a/language/testing/Language_mock_gen.go +++ b/language/testing/Language_mock_gen.go @@ -3,8 +3,10 @@ package languagetesting import ( - mock "github.com/stretchr/testify/mock" + language "github.com/symflower/eval-dev-quality/language" log "github.com/symflower/eval-dev-quality/log" + + mock "github.com/stretchr/testify/mock" ) // MockLanguage is an autogenerated mock type for the Language type @@ -48,24 +50,26 @@ func (_m *MockLanguage) DefaultTestFileSuffix() string { return r0 } -// Execute provides a mock function with given fields: logger, repositoryPath -func (_m *MockLanguage) Execute(logger *log.Logger, repositoryPath string) (uint64, []error, error) { +// ExecuteTests provides a mock function with given fields: logger, repositoryPath +func (_m *MockLanguage) ExecuteTests(logger *log.Logger, repositoryPath string) (*language.TestResult, []error, error) { ret := _m.Called(logger, repositoryPath) if len(ret) == 0 { - panic("no return value specified for Execute") + panic("no return value specified for ExecuteTests") } - var r0 uint64 + var r0 *language.TestResult var r1 []error var r2 error - if rf, ok := ret.Get(0).(func(*log.Logger, string) (uint64, []error, error)); ok { + if rf, ok := ret.Get(0).(func(*log.Logger, string) (*language.TestResult, []error, error)); ok { return rf(logger, repositoryPath) } - if rf, ok := ret.Get(0).(func(*log.Logger, string) uint64); ok { + if rf, ok := ret.Get(0).(func(*log.Logger, string) *language.TestResult); ok { r0 = rf(logger, repositoryPath) } else { - r0 = ret.Get(0).(uint64) + if ret.Get(0) != nil { + r0 = ret.Get(0).(*language.TestResult) + } } if rf, ok := ret.Get(1).(func(*log.Logger, string) []error); ok { diff --git a/language/testing/language.go b/language/testing/language.go index 6c7e5859..cf219220 100644 --- a/language/testing/language.go +++ b/language/testing/language.go @@ -11,7 +11,7 @@ import ( "github.com/zimmski/osutil" ) -type TestCaseExecute struct { +type TestCaseExecuteTests struct { Name string Language language.Language @@ -19,13 +19,13 @@ type TestCaseExecute struct { RepositoryPath string RepositoryChange func(t *testing.T, repositoryPath string) - ExpectedCoverage uint64 + ExpectedTestResult *language.TestResult ExpectedProblemTexts []string ExpectedError error ExpectedErrorText string } -func (tc *TestCaseExecute) Validate(t *testing.T) { +func (tc *TestCaseExecuteTests) Validate(t *testing.T) { t.Run(tc.Name, func(t *testing.T) { logOutput, logger := log.Buffer() defer func() { @@ -42,7 +42,7 @@ func (tc *TestCaseExecute) Validate(t *testing.T) { tc.RepositoryChange(t, repositoryPath) } - actualCoverage, actualProblems, actualError := tc.Language.Execute(logger, repositoryPath) + actualTestResult, actualProblems, actualError := tc.Language.ExecuteTests(logger, repositoryPath) require.Equal(t, len(tc.ExpectedProblemTexts), len(actualProblems), "the number of expected problems need to match the number of actual problems") for i, expectedProblemText := range tc.ExpectedProblemTexts { @@ -55,7 +55,7 @@ func (tc *TestCaseExecute) Validate(t *testing.T) { assert.ErrorContains(t, actualError, tc.ExpectedErrorText) } else { assert.NoError(t, actualError) - assert.Equal(t, tc.ExpectedCoverage, actualCoverage) + assert.Equal(t, tc.ExpectedTestResult, actualTestResult) } }) } diff --git a/model/symflower/symflower_test.go b/model/symflower/symflower_test.go index a54133f3..857fa1eb 100644 --- a/model/symflower/symflower_test.go +++ b/model/symflower/symflower_test.go @@ -80,10 +80,10 @@ func TestModelGenerateTestsForFile(t *testing.T) { metricstesting.AssertAssessmentsEqual(t, tc.ExpectedAssessment, actualAssessment) - actualCoverage, actualProblems, err := tc.Language.Execute(logger, repositoryPath) + actualTestResult, actualProblems, err := tc.Language.ExecuteTests(logger, repositoryPath) require.NoError(t, err) require.Empty(t, actualProblems) - assert.Equal(t, tc.ExpectedCoverage, actualCoverage) + assert.Equal(t, tc.ExpectedCoverage, actualTestResult.Coverage) } }) }