Skip to content

Commit

Permalink
Parse number of total and passing tests from "symflower test" output
Browse files Browse the repository at this point in the history
  • Loading branch information
@ahumenberger
ahumenberger committed Jul 23, 2024
1 parent 02fed50 commit 241a3c2
Show file tree
Hide file tree
Showing 5 changed files with 426 additions and 11 deletions.
46 changes: 36 additions & 10 deletions language/golang/language.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,8 +85,6 @@ func (l *Language) DefaultTestFileSuffix() string {
return "_test.go"
}

var languageGoTestsErrorMatch = regexp.MustCompile(`DONE (\d+) tests, (\d+) failure`)

// ExecuteTests invokes the language specific testing on the given repository.
func (l *Language) ExecuteTests(logger *log.Logger, repositoryPath string) (testResult *language.TestResult, problems []error, err error) {
commandOutput, err := util.CommandWithResult(context.Background(), logger, &util.Command{
Expand Down Expand Up @@ -115,20 +113,24 @@ func (l *Language) ExecuteTests(logger *log.Logger, repositoryPath string) (test

Directory: repositoryPath,
})

testsTotal, testsPass, e := parseSymflowerTestOutput(commandOutput)
if e != nil {
problems = append(problems, pkgerrors.WithMessage(pkgerrors.WithStack(e), commandOutput))
}
// If there are test failures, then this is just a soft error since we still are able to receive coverage data.
if err != nil {
testSummary := languageGoTestsErrorMatch.FindStringSubmatch(commandOutput)
if len(testSummary) > 0 {
if failureCount, e := strconv.Atoi(testSummary[2]); e != nil {
return nil, nil, pkgerrors.WithStack(e)
} else if failureCount > 0 {
// If there are test failures, then this is just a soft error since we still are able to receive coverage data.
problems = append(problems, pkgerrors.WithMessage(pkgerrors.WithStack(err), commandOutput))
}
if testsTotal-testsPass > 0 {
problems = append(problems, pkgerrors.WithMessage(pkgerrors.WithStack(err), commandOutput))
} else {
return nil, nil, pkgerrors.WithMessage(pkgerrors.WithStack(err), commandOutput)
}
}

testResult = &language.TestResult{
TestsTotal: uint(testsTotal),
TestsPass: uint(testsPass),
}
testResult.Coverage, err = language.CoverageObjectCountOfFile(logger, coverageFilePath)
if err != nil {
return testResult, problems, pkgerrors.WithMessage(pkgerrors.WithStack(err), commandOutput)
Expand All @@ -137,6 +139,30 @@ func (l *Language) ExecuteTests(logger *log.Logger, repositoryPath string) (test
return testResult, problems, nil
}

var languageGoTestSummaryRE = regexp.MustCompile(`DONE (\d+) tests(?:, (\d+) failure)?`)

func parseSymflowerTestOutput(data string) (testsTotal int, testsPass int, err error) {
testSummary := languageGoTestSummaryRE.FindStringSubmatch(data)
if len(testSummary) == 0 {
return 0, 0, pkgerrors.WithMessage(pkgerrors.WithStack(language.ErrCannotParseTestSummary), data)
}

testsTotal, err = strconv.Atoi(testSummary[1])
if err != nil {
return 0, 0, pkgerrors.WithStack(err)
}

var testsFail int
if len(testSummary[2]) > 0 {
if testsFail, err = strconv.Atoi(testSummary[2]); err != nil {
return 0, 0, pkgerrors.WithStack(err)
}
}
testsPass = testsTotal - testsFail

return testsTotal, testsPass, nil
}

// Mistakes builds a Go repository and returns the list of mistakes found.
func (l *Language) Mistakes(logger *log.Logger, repositoryPath string) (mistakes []string, err error) {
output, err := util.CommandWithResult(context.Background(), logger, &util.Command{
Expand Down
99 changes: 99 additions & 0 deletions language/golang/language_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,9 @@ func TestLanguageExecute(t *testing.T) {
},

ExpectedTestResult: &language.TestResult{
TestsTotal: 1,
TestsPass: 1,

Coverage: 1,
},
})
Expand All @@ -121,6 +124,9 @@ func TestLanguageExecute(t *testing.T) {
},

ExpectedTestResult: &language.TestResult{
TestsTotal: 1,
TestsPass: 0,

Coverage: 1,
},
ExpectedProblemTexts: []string{
Expand Down Expand Up @@ -232,3 +238,96 @@ func TestExtractMistakes(t *testing.T) {
},
})
}

func TestParseSymflowerTestOutput(t *testing.T) {
type testCase struct {
Name string

Data string

ExpectedTestsTotal int
ExpectedTestsPass int
ExpectedErr error
}

validate := func(t *testing.T, tc *testCase) {
t.Run(tc.Name, func(t *testing.T) {
actualTestsTotal, actualTestsPass, actualErr := parseSymflowerTestOutput(bytesutil.StringTrimIndentations(tc.Data))

assert.Equal(t, tc.ExpectedTestsTotal, actualTestsTotal)
assert.Equal(t, tc.ExpectedTestsPass, actualTestsPass)
assert.Equal(t, tc.ExpectedErr, actualErr)
})
}

validate(t, &testCase{
Name: "Passing tests only",

Data: `
=== RUN TestSymflowerIsSorted
=== RUN TestSymflowerIsSorted/#00
--- PASS: TestSymflowerIsSorted/#00 (0.00s)
=== RUN TestSymflowerIsSorted/#01
--- PASS: TestSymflowerIsSorted/#01 (0.00s)
=== RUN TestSymflowerIsSorted/#02
--- PASS: TestSymflowerIsSorted/#02 (0.00s)
=== RUN TestSymflowerIsSorted/#03
--- PASS: TestSymflowerIsSorted/#03 (0.00s)
=== RUN TestSymflowerIsSorted/#04
--- PASS: TestSymflowerIsSorted/#04 (0.00s)
--- PASS: TestSymflowerIsSorted (0.00s)
PASS
coverage: 100.0% of statements
ok isSorted 0.003s
DONE 6 tests in 0.281s
`,

ExpectedTestsTotal: 6,
ExpectedTestsPass: 6,
})
validate(t, &testCase{
Name: "Failing tests",

Data: `
=== RUN TestSymflowerIsSorted
=== RUN TestSymflowerIsSorted/#00
isSorted_test.go:22:
Error Trace: /home/andreas/repos/eval-dev-quality/testdata/golang/transpile/isSorted/isSorted_test.go:22
Error: Not equal:
expected: true
actual : false
Test: TestSymflowerIsSorted/#00
--- FAIL: TestSymflowerIsSorted/#00 (0.00s)
=== RUN TestSymflowerIsSorted/#01
--- PASS: TestSymflowerIsSorted/#01 (0.00s)
=== RUN TestSymflowerIsSorted/#02
--- PASS: TestSymflowerIsSorted/#02 (0.00s)
=== RUN TestSymflowerIsSorted/#03
--- PASS: TestSymflowerIsSorted/#03 (0.00s)
=== RUN TestSymflowerIsSorted/#04
--- PASS: TestSymflowerIsSorted/#04 (0.00s)
--- FAIL: TestSymflowerIsSorted (0.00s)
FAIL
coverage: 100.0% of statements
exit status 1
FAIL isSorted 0.002s
=== Failed
=== FAIL: . TestSymflowerIsSorted/#00 (0.00s)
isSorted_test.go:22:
Error Trace: /home/andreas/repos/eval-dev-quality/testdata/golang/transpile/isSorted/isSorted_test.go:22
Error: Not equal:
expected: true
actual : false
Test: TestSymflowerIsSorted/#00
=== FAIL: . TestSymflowerIsSorted (0.00s)
DONE 6 tests, 2 failures in 0.288s
`,

ExpectedTestsTotal: 6,
ExpectedTestsPass: 4,
})
}
39 changes: 38 additions & 1 deletion language/java/language.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"path/filepath"
"regexp"
"sort"
"strconv"
"strings"

pkgerrors "github.com/pkg/errors"
Expand Down Expand Up @@ -119,7 +120,16 @@ func (l *Language) ExecuteTests(logger *log.Logger, repositoryPath string) (test
return nil, nil, pkgerrors.WithMessage(pkgerrors.WithStack(err), commandOutput)
}

testResult = &language.TestResult{}
testsTotal, testsPass, err := parseSymflowerTestOutput(commandOutput)
if err != nil {
return nil, nil, err
}

testResult = &language.TestResult{
TestsTotal: uint(testsTotal),
TestsPass: uint(testsPass),
}

testResult.Coverage, err = language.CoverageObjectCountOfFile(logger, coverageFilePath)
if err != nil {
return nil, nil, pkgerrors.WithMessage(pkgerrors.WithStack(err), commandOutput)
Expand All @@ -128,6 +138,33 @@ func (l *Language) ExecuteTests(logger *log.Logger, repositoryPath string) (test
return testResult, nil, nil
}

var languageMavenTestSummaryRE = regexp.MustCompile(`Tests run: (\d+), Failures: (\d+), Errors: (\d+)`)

func parseSymflowerTestOutput(data string) (testsTotal int, testsPass int, err error) {
testSummaries := languageMavenTestSummaryRE.FindAllStringSubmatch(data, -1)
if len(testSummaries) == 0 {
return 0, 0, pkgerrors.WithMessage(pkgerrors.WithStack(language.ErrCannotParseTestSummary), data)
}

testSummary := testSummaries[len(testSummaries)-1]
testsTotal, err = strconv.Atoi(testSummary[1])
if err != nil {
return 0, 0, pkgerrors.WithStack(err)
}
testsFail, err := strconv.Atoi(testSummary[2])
if err != nil {
return 0, 0, pkgerrors.WithStack(err)
}
testsError, err := strconv.Atoi(testSummary[3])
if err != nil {
return 0, 0, pkgerrors.WithStack(err)
}

testsPass = testsTotal - testsFail - testsError

return testsTotal, testsPass, nil
}

// Mistakes builds a Java repository and returns the list of mistakes found.
func (l *Language) Mistakes(logger *log.Logger, repositoryPath string) (mistakes []string, err error) {
output, err := util.CommandWithResult(context.Background(), logger, &util.Command{
Expand Down
Loading

0 comments on commit 241a3c2

Please sign in to comment.