diff --git a/language/golang/language.go b/language/golang/language.go index 56779313..450a51e5 100644 --- a/language/golang/language.go +++ b/language/golang/language.go @@ -85,8 +85,6 @@ func (l *Language) DefaultTestFileSuffix() string { return "_test.go" } -var languageGoTestsErrorMatch = regexp.MustCompile(`DONE (\d+) tests, (\d+) failure`) - // ExecuteTests invokes the language specific testing on the given repository. func (l *Language) ExecuteTests(logger *log.Logger, repositoryPath string) (testResult *language.TestResult, problems []error, err error) { commandOutput, err := util.CommandWithResult(context.Background(), logger, &util.Command{ @@ -115,20 +113,24 @@ func (l *Language) ExecuteTests(logger *log.Logger, repositoryPath string) (test Directory: repositoryPath, }) + + testsTotal, testsPass, e := parseSymflowerTestOutput(commandOutput) + if e != nil { + problems = append(problems, pkgerrors.WithMessage(pkgerrors.WithStack(e), commandOutput)) + } + // If there are test failures, then this is just a soft error since we still are able to receive coverage data. if err != nil { - testSummary := languageGoTestsErrorMatch.FindStringSubmatch(commandOutput) - if len(testSummary) > 0 { - if failureCount, e := strconv.Atoi(testSummary[2]); e != nil { - return nil, nil, pkgerrors.WithStack(e) - } else if failureCount > 0 { - // If there are test failures, then this is just a soft error since we still are able to receive coverage data. - problems = append(problems, pkgerrors.WithMessage(pkgerrors.WithStack(err), commandOutput)) - } + if testsTotal-testsPass > 0 { + problems = append(problems, pkgerrors.WithMessage(pkgerrors.WithStack(err), commandOutput)) } else { return nil, nil, pkgerrors.WithMessage(pkgerrors.WithStack(err), commandOutput) } } + testResult = &language.TestResult{ + TestsTotal: uint(testsTotal), + TestsPass: uint(testsPass), + } testResult.Coverage, err = language.CoverageObjectCountOfFile(logger, coverageFilePath) if err != nil { return testResult, problems, pkgerrors.WithMessage(pkgerrors.WithStack(err), commandOutput) @@ -137,6 +139,30 @@ func (l *Language) ExecuteTests(logger *log.Logger, repositoryPath string) (test return testResult, problems, nil } +var languageGoTestSummaryRE = regexp.MustCompile(`DONE (\d+) tests(?:, (\d+) failure)?`) + +func parseSymflowerTestOutput(data string) (testsTotal int, testsPass int, err error) { + testSummary := languageGoTestSummaryRE.FindStringSubmatch(data) + if len(testSummary) == 0 { + return 0, 0, pkgerrors.WithMessage(pkgerrors.WithStack(language.ErrCannotParseTestSummary), data) + } + + testsTotal, err = strconv.Atoi(testSummary[1]) + if err != nil { + return 0, 0, pkgerrors.WithStack(err) + } + + var testsFail int + if len(testSummary[2]) > 0 { + if testsFail, err = strconv.Atoi(testSummary[2]); err != nil { + return 0, 0, pkgerrors.WithStack(err) + } + } + testsPass = testsTotal - testsFail + + return testsTotal, testsPass, nil +} + // Mistakes builds a Go repository and returns the list of mistakes found. func (l *Language) Mistakes(logger *log.Logger, repositoryPath string) (mistakes []string, err error) { output, err := util.CommandWithResult(context.Background(), logger, &util.Command{ diff --git a/language/golang/language_test.go b/language/golang/language_test.go index 39665a43..f88e06b0 100644 --- a/language/golang/language_test.go +++ b/language/golang/language_test.go @@ -97,6 +97,9 @@ func TestLanguageExecute(t *testing.T) { }, ExpectedTestResult: &language.TestResult{ + TestsTotal: 1, + TestsPass: 1, + Coverage: 1, }, }) @@ -121,6 +124,9 @@ func TestLanguageExecute(t *testing.T) { }, ExpectedTestResult: &language.TestResult{ + TestsTotal: 1, + TestsPass: 0, + Coverage: 1, }, ExpectedProblemTexts: []string{ @@ -232,3 +238,96 @@ func TestExtractMistakes(t *testing.T) { }, }) } + +func TestParseSymflowerTestOutput(t *testing.T) { + type testCase struct { + Name string + + Data string + + ExpectedTestsTotal int + ExpectedTestsPass int + ExpectedErr error + } + + validate := func(t *testing.T, tc *testCase) { + t.Run(tc.Name, func(t *testing.T) { + actualTestsTotal, actualTestsPass, actualErr := parseSymflowerTestOutput(bytesutil.StringTrimIndentations(tc.Data)) + + assert.Equal(t, tc.ExpectedTestsTotal, actualTestsTotal) + assert.Equal(t, tc.ExpectedTestsPass, actualTestsPass) + assert.Equal(t, tc.ExpectedErr, actualErr) + }) + } + + validate(t, &testCase{ + Name: "Passing tests only", + + Data: ` + === RUN TestSymflowerIsSorted + === RUN TestSymflowerIsSorted/#00 + --- PASS: TestSymflowerIsSorted/#00 (0.00s) + === RUN TestSymflowerIsSorted/#01 + --- PASS: TestSymflowerIsSorted/#01 (0.00s) + === RUN TestSymflowerIsSorted/#02 + --- PASS: TestSymflowerIsSorted/#02 (0.00s) + === RUN TestSymflowerIsSorted/#03 + --- PASS: TestSymflowerIsSorted/#03 (0.00s) + === RUN TestSymflowerIsSorted/#04 + --- PASS: TestSymflowerIsSorted/#04 (0.00s) + --- PASS: TestSymflowerIsSorted (0.00s) + PASS + coverage: 100.0% of statements + ok isSorted 0.003s + + DONE 6 tests in 0.281s + `, + + ExpectedTestsTotal: 6, + ExpectedTestsPass: 6, + }) + validate(t, &testCase{ + Name: "Failing tests", + + Data: ` + === RUN TestSymflowerIsSorted + === RUN TestSymflowerIsSorted/#00 + isSorted_test.go:22: + Error Trace: /home/andreas/repos/eval-dev-quality/testdata/golang/transpile/isSorted/isSorted_test.go:22 + Error: Not equal: + expected: true + actual : false + Test: TestSymflowerIsSorted/#00 + --- FAIL: TestSymflowerIsSorted/#00 (0.00s) + === RUN TestSymflowerIsSorted/#01 + --- PASS: TestSymflowerIsSorted/#01 (0.00s) + === RUN TestSymflowerIsSorted/#02 + --- PASS: TestSymflowerIsSorted/#02 (0.00s) + === RUN TestSymflowerIsSorted/#03 + --- PASS: TestSymflowerIsSorted/#03 (0.00s) + === RUN TestSymflowerIsSorted/#04 + --- PASS: TestSymflowerIsSorted/#04 (0.00s) + --- FAIL: TestSymflowerIsSorted (0.00s) + FAIL + coverage: 100.0% of statements + exit status 1 + FAIL isSorted 0.002s + + === Failed + === FAIL: . TestSymflowerIsSorted/#00 (0.00s) + isSorted_test.go:22: + Error Trace: /home/andreas/repos/eval-dev-quality/testdata/golang/transpile/isSorted/isSorted_test.go:22 + Error: Not equal: + expected: true + actual : false + Test: TestSymflowerIsSorted/#00 + + === FAIL: . TestSymflowerIsSorted (0.00s) + + DONE 6 tests, 2 failures in 0.288s + `, + + ExpectedTestsTotal: 6, + ExpectedTestsPass: 4, + }) +} diff --git a/language/java/language.go b/language/java/language.go index f8744c12..33b28419 100644 --- a/language/java/language.go +++ b/language/java/language.go @@ -6,6 +6,7 @@ import ( "path/filepath" "regexp" "sort" + "strconv" "strings" pkgerrors "github.com/pkg/errors" @@ -119,7 +120,16 @@ func (l *Language) ExecuteTests(logger *log.Logger, repositoryPath string) (test return nil, nil, pkgerrors.WithMessage(pkgerrors.WithStack(err), commandOutput) } - testResult = &language.TestResult{} + testsTotal, testsPass, err := parseSymflowerTestOutput(commandOutput) + if err != nil { + return nil, nil, err + } + + testResult = &language.TestResult{ + TestsTotal: uint(testsTotal), + TestsPass: uint(testsPass), + } + testResult.Coverage, err = language.CoverageObjectCountOfFile(logger, coverageFilePath) if err != nil { return nil, nil, pkgerrors.WithMessage(pkgerrors.WithStack(err), commandOutput) @@ -128,6 +138,33 @@ func (l *Language) ExecuteTests(logger *log.Logger, repositoryPath string) (test return testResult, nil, nil } +var languageMavenTestSummaryRE = regexp.MustCompile(`Tests run: (\d+), Failures: (\d+), Errors: (\d+)`) + +func parseSymflowerTestOutput(data string) (testsTotal int, testsPass int, err error) { + testSummaries := languageMavenTestSummaryRE.FindAllStringSubmatch(data, -1) + if len(testSummaries) == 0 { + return 0, 0, pkgerrors.WithMessage(pkgerrors.WithStack(language.ErrCannotParseTestSummary), data) + } + + testSummary := testSummaries[len(testSummaries)-1] + testsTotal, err = strconv.Atoi(testSummary[1]) + if err != nil { + return 0, 0, pkgerrors.WithStack(err) + } + testsFail, err := strconv.Atoi(testSummary[2]) + if err != nil { + return 0, 0, pkgerrors.WithStack(err) + } + testsError, err := strconv.Atoi(testSummary[3]) + if err != nil { + return 0, 0, pkgerrors.WithStack(err) + } + + testsPass = testsTotal - testsFail - testsError + + return testsTotal, testsPass, nil +} + // Mistakes builds a Java repository and returns the list of mistakes found. func (l *Language) Mistakes(logger *log.Logger, repositoryPath string) (mistakes []string, err error) { output, err := util.CommandWithResult(context.Background(), logger, &util.Command{ diff --git a/language/java/language_test.go b/language/java/language_test.go index ed99179d..8c03a1a6 100644 --- a/language/java/language_test.go +++ b/language/java/language_test.go @@ -176,6 +176,9 @@ func TestLanguageExecute(t *testing.T) { }, ExpectedTestResult: &language.TestResult{ + TestsTotal: 1, + TestsPass: 1, + Coverage: 1, }, }) @@ -203,6 +206,9 @@ func TestLanguageExecute(t *testing.T) { }, ExpectedTestResult: &language.TestResult{ + TestsTotal: 1, + TestsPass: 0, + Coverage: 3, }, }) @@ -338,3 +344,236 @@ func TestExtractMistakes(t *testing.T) { }, }) } + +func TestParseSymflowerTestOutput(t *testing.T) { + type testCase struct { + Name string + + Data string + + ExpectedTestsTotal int + ExpectedTestsPass int + ExpectedErr error + } + + validate := func(t *testing.T, tc *testCase) { + t.Run(tc.Name, func(t *testing.T) { + actualTestsTotal, actualTestsPass, actualErr := parseSymflowerTestOutput(bytesutil.StringTrimIndentations(tc.Data)) + + assert.Equal(t, tc.ExpectedTestsTotal, actualTestsTotal) + assert.Equal(t, tc.ExpectedTestsPass, actualTestsPass) + assert.Equal(t, tc.ExpectedErr, actualErr) + }) + } + + validate(t, &testCase{ + Name: "Passing tests only", + + Data: ` + [INFO] Scanning for projects... + [WARNING] + [WARNING] Some problems were encountered while building the effective model for eval.dev.quality:is-sorted:jar:SNAPSHOT + [WARNING] 'version' uses an unsupported snapshot version format, should be '*-SNAPSHOT' instead. @ line 8, column 11 + [WARNING] + [WARNING] It is highly recommended to fix these problems because they threaten the stability of your build. + [WARNING] + [WARNING] For this reason, future Maven versions might no longer support building such malformed projects. + [WARNING] + [INFO] + [INFO] ---------------------< eval.dev.quality:is-sorted >--------------------- + [INFO] Building is-sorted SNAPSHOT + [INFO] from pom.xml + [INFO] --------------------------------[ jar ]--------------------------------- + [INFO] + [INFO] --- resources:3.3.0:resources (default-resources) @ is-sorted --- + [WARNING] Using platform encoding (UTF-8 actually) to copy filtered resources, i.e. build is platform dependent! + [INFO] skip non existing resourceDirectory /home/andreas/repos/eval-dev-quality/testdata/java/transpile/isSorted/src/main/resources + [INFO] + [INFO] --- compiler:3.10.1:compile (default-compile) @ is-sorted --- + [INFO] Changes detected - recompiling the module! + [WARNING] File encoding has not been set, using platform encoding UTF-8, i.e. build is platform dependent! + [INFO] Compiling 1 source file to /home/andreas/repos/eval-dev-quality/testdata/java/transpile/isSorted/target/classes + [INFO] + [INFO] --- resources:3.3.0:testResources (default-testResources) @ is-sorted --- + [WARNING] Using platform encoding (UTF-8 actually) to copy filtered resources, i.e. build is platform dependent! + [INFO] skip non existing resourceDirectory /home/andreas/repos/eval-dev-quality/testdata/java/transpile/isSorted/src/test/resources + [INFO] + [INFO] --- compiler:3.10.1:testCompile (default-testCompile) @ is-sorted --- + [INFO] Changes detected - recompiling the module! + [WARNING] File encoding has not been set, using platform encoding UTF-8, i.e. build is platform dependent! + [INFO] Compiling 1 source file to /home/andreas/repos/eval-dev-quality/testdata/java/transpile/isSorted/target/test-classes + [INFO] + [INFO] --- surefire:3.2.5:test (default-test) @ is-sorted --- + [INFO] Using auto detected provider org.apache.maven.surefire.junitplatform.JUnitPlatformProvider + [INFO] + [INFO] ------------------------------------------------------- + [INFO] T E S T S + [INFO] ------------------------------------------------------- + [INFO] Running com.eval.IsSortedTest + [INFO] Tests run: 5, Failures: 0, Errors: 0, Skipped: 0, Time elapsed: 0.295 s -- in com.eval.IsSortedTest + [INFO] + [INFO] Results: + [INFO] + [INFO] Tests run: 5, Failures: 0, Errors: 0, Skipped: 0 + [INFO] + [INFO] ------------------------------------------------------------------------ + [INFO] BUILD SUCCESS + [INFO] ------------------------------------------------------------------------ + [INFO] Total time: 12.959 s + [INFO] Finished at: 2024-07-17T07:17:52+02:00 + [INFO] ------------------------------------------------------------------------ + `, + + ExpectedTestsTotal: 5, + ExpectedTestsPass: 5, + }) + validate(t, &testCase{ + Name: "Failing tests", + + Data: ` + [INFO] Scanning for projects... + [WARNING] + [WARNING] Some problems were encountered while building the effective model for eval.dev.quality:is-sorted:jar:SNAPSHOT + [WARNING] 'version' uses an unsupported snapshot version format, should be '*-SNAPSHOT' instead. @ line 8, column 11 + [WARNING] + [WARNING] It is highly recommended to fix these problems because they threaten the stability of your build. + [WARNING] + [WARNING] For this reason, future Maven versions might no longer support building such malformed projects. + [WARNING] + [INFO] + [INFO] ---------------------< eval.dev.quality:is-sorted >--------------------- + [INFO] Building is-sorted SNAPSHOT + [INFO] from pom.xml + [INFO] --------------------------------[ jar ]--------------------------------- + [INFO] + [INFO] --- resources:3.3.0:resources (default-resources) @ is-sorted --- + [WARNING] Using platform encoding (UTF-8 actually) to copy filtered resources, i.e. build is platform dependent! + [INFO] skip non existing resourceDirectory /home/andreas/repos/eval-dev-quality/testdata/java/transpile/isSorted/src/main/resources + [INFO] + [INFO] --- compiler:3.10.1:compile (default-compile) @ is-sorted --- + [INFO] Nothing to compile - all classes are up to date + [INFO] + [INFO] --- resources:3.3.0:testResources (default-testResources) @ is-sorted --- + [WARNING] Using platform encoding (UTF-8 actually) to copy filtered resources, i.e. build is platform dependent! + [INFO] skip non existing resourceDirectory /home/andreas/repos/eval-dev-quality/testdata/java/transpile/isSorted/src/test/resources + [INFO] + [INFO] --- compiler:3.10.1:testCompile (default-testCompile) @ is-sorted --- + [INFO] Nothing to compile - all classes are up to date + [INFO] + [INFO] --- surefire:3.2.5:test (default-test) @ is-sorted --- + [INFO] Using auto detected provider org.apache.maven.surefire.junitplatform.JUnitPlatformProvider + [INFO] + [INFO] ------------------------------------------------------- + [INFO] T E S T S + [INFO] ------------------------------------------------------- + [INFO] Running com.eval.IsSortedTest + [ERROR] Tests run: 5, Failures: 1, Errors: 0, Skipped: 0, Time elapsed: 0.081 s <<< FAILURE! -- in com.eval.IsSortedTest + [ERROR] com.eval.IsSortedTest.isSorted4 -- Time elapsed: 0.006 s <<< FAILURE! + org.opentest4j.AssertionFailedError: expected: but was: + at org.junit.jupiter.api.AssertionUtils.fail(AssertionUtils.java:55) + at org.junit.jupiter.api.AssertFalse.assertFalse(AssertFalse.java:40) + at org.junit.jupiter.api.AssertFalse.assertFalse(AssertFalse.java:35) + at org.junit.jupiter.api.Assertions.assertFalse(Assertions.java:227) + at com.eval.IsSortedTest.isSorted4(IsSortedTest.java:38) + at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) + at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) + at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) + at java.base/java.lang.reflect.Method.invoke(Method.java:566) + at org.junit.platform.commons.util.ReflectionUtils.invokeMethod(ReflectionUtils.java:725) + at org.junit.jupiter.engine.execution.MethodInvocation.proceed(MethodInvocation.java:60) + at org.junit.jupiter.engine.execution.InvocationInterceptorChain$ValidatingInvocation.proceed(InvocationInterceptorChain.java:131) + at org.junit.jupiter.engine.extension.TimeoutExtension.intercept(TimeoutExtension.java:149) + at org.junit.jupiter.engine.extension.TimeoutExtension.interceptTestableMethod(TimeoutExtension.java:140) + at org.junit.jupiter.engine.extension.TimeoutExtension.interceptTestMethod(TimeoutExtension.java:84) + at org.junit.jupiter.engine.execution.ExecutableInvoker$ReflectiveInterceptorCall.lambda$ofVoidMethod$0(ExecutableInvoker.java:115) + at org.junit.jupiter.engine.execution.ExecutableInvoker.lambda$invoke$0(ExecutableInvoker.java:105) + at org.junit.jupiter.engine.execution.InvocationInterceptorChain$InterceptedInvocation.proceed(InvocationInterceptorChain.java:106) + at org.junit.jupiter.engine.execution.InvocationInterceptorChain.proceed(InvocationInterceptorChain.java:64) + at org.junit.jupiter.engine.execution.InvocationInterceptorChain.chainAndInvoke(InvocationInterceptorChain.java:45) + at org.junit.jupiter.engine.execution.InvocationInterceptorChain.invoke(InvocationInterceptorChain.java:37) + at org.junit.jupiter.engine.execution.ExecutableInvoker.invoke(ExecutableInvoker.java:104) + at org.junit.jupiter.engine.execution.ExecutableInvoker.invoke(ExecutableInvoker.java:98) + at org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.lambda$invokeTestMethod$7(TestMethodTestDescriptor.java:214) + at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73) + at org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.invokeTestMethod(TestMethodTestDescriptor.java:210) + at org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.execute(TestMethodTestDescriptor.java:135) + at org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.execute(TestMethodTestDescriptor.java:66) + at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$6(NodeTestTask.java:151) + at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73) + at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:141) + at org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137) + at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$9(NodeTestTask.java:139) + at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73) + at org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:138) + at org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:95) + at java.base/java.util.ArrayList.forEach(ArrayList.java:1541) + at org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.invokeAll(SameThreadHierarchicalTestExecutorService.java:41) + at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$6(NodeTestTask.java:155) + at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73) + at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:141) + at org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137) + at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$9(NodeTestTask.java:139) + at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73) + at org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:138) + at org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:95) + at java.base/java.util.ArrayList.forEach(ArrayList.java:1541) + at org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.invokeAll(SameThreadHierarchicalTestExecutorService.java:41) + at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$6(NodeTestTask.java:155) + at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73) + at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:141) + at org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137) + at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$9(NodeTestTask.java:139) + at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73) + at org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:138) + at org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:95) + at org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.submit(SameThreadHierarchicalTestExecutorService.java:35) + at org.junit.platform.engine.support.hierarchical.HierarchicalTestExecutor.execute(HierarchicalTestExecutor.java:57) + at org.junit.platform.engine.support.hierarchical.HierarchicalTestEngine.execute(HierarchicalTestEngine.java:54) + at org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:107) + at org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:88) + at org.junit.platform.launcher.core.EngineExecutionOrchestrator.lambda$execute$0(EngineExecutionOrchestrator.java:54) + at org.junit.platform.launcher.core.EngineExecutionOrchestrator.withInterceptedStreams(EngineExecutionOrchestrator.java:67) + at org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:52) + at org.junit.platform.launcher.core.DefaultLauncher.execute(DefaultLauncher.java:114) + at org.junit.platform.launcher.core.DefaultLauncher.execute(DefaultLauncher.java:86) + at org.junit.platform.launcher.core.DefaultLauncherSession$DelegatingLauncher.execute(DefaultLauncherSession.java:86) + at org.apache.maven.surefire.junitplatform.LazyLauncher.execute(LazyLauncher.java:56) + at org.apache.maven.surefire.junitplatform.JUnitPlatformProvider.execute(JUnitPlatformProvider.java:184) + at org.apache.maven.surefire.junitplatform.JUnitPlatformProvider.invokeAllTests(JUnitPlatformProvider.java:148) + at org.apache.maven.surefire.junitplatform.JUnitPlatformProvider.invoke(JUnitPlatformProvider.java:122) + at org.apache.maven.surefire.booter.ForkedBooter.runSuitesInProcess(ForkedBooter.java:385) + at org.apache.maven.surefire.booter.ForkedBooter.execute(ForkedBooter.java:162) + at org.apache.maven.surefire.booter.ForkedBooter.run(ForkedBooter.java:507) + at org.apache.maven.surefire.booter.ForkedBooter.main(ForkedBooter.java:495) + + [INFO] + [INFO] Results: + [INFO] + [ERROR] Failures: + [ERROR] IsSortedTest.isSorted4:38 expected: but was: + [INFO] + [ERROR] Tests run: 5, Failures: 1, Errors: 0, Skipped: 0 + [INFO] + [INFO] ------------------------------------------------------------------------ + [INFO] BUILD FAILURE + [INFO] ------------------------------------------------------------------------ + [INFO] Total time: 1.652 s + [INFO] Finished at: 2024-07-17T07:19:39+02:00 + [INFO] ------------------------------------------------------------------------ + [ERROR] Failed to execute goal org.apache.maven.plugins:maven-surefire-plugin:3.2.5:test (default-test) on project is-sorted: There are test failures. + [ERROR] + [ERROR] Please refer to /home/andreas/repos/eval-dev-quality/testdata/java/transpile/isSorted/target/surefire-reports for the individual test results. + [ERROR] Please refer to dump files (if any exist) [date].dump, [date]-jvmRun[N].dump and [date].dumpstream. + [ERROR] -> [Help 1] + [ERROR] + [ERROR] To see the full stack trace of the errors, re-run Maven with the -e switch. + [ERROR] Re-run Maven using the -X switch to enable full debug logging. + [ERROR] + [ERROR] For more information about the errors and possible solutions, please read the following articles: + [ERROR] [Help 1] http://cwiki.apache.org/confluence/display/MAVEN/MojoFailureException + `, + + ExpectedTestsTotal: 5, + ExpectedTestsPass: 4, + }) +} diff --git a/language/language.go b/language/language.go index f2afda25..6ca51505 100644 --- a/language/language.go +++ b/language/language.go @@ -1,6 +1,7 @@ package language import ( + "errors" "os" "path/filepath" "sort" @@ -11,6 +12,11 @@ import ( "github.com/symflower/eval-dev-quality/log" ) +var ( + // ErrCannotParseTestSummary indicates that the test summary cannot be parsed. + ErrCannotParseTestSummary = errors.New("cannot parse test summary") +) + // DefaultExecutionTimeout defines the timeout for an execution. // WORKAROUND For now we define the timeout as a global variable but it should eventually be moved to the "symflower test" command. var DefaultExecutionTimeout = 5 * time.Minute @@ -77,5 +83,13 @@ func RepositoriesForLanguage(language Language, testdataPath string) (relativeRe // TestResult holds the result of running tests. type TestResult struct { + TestsTotal uint + TestsPass uint + Coverage uint64 } + +// PassingTestsPercentage returns the percentage of passing tests. +func (tr *TestResult) PassingTestsPercentage() (percentage uint) { + return tr.TestsPass / tr.TestsTotal * 100 +}