From 4326f25cf229cae9c0761e837203550c5087ddd1 Mon Sep 17 00:00:00 2001 From: Simon Bauer Date: Fri, 18 Oct 2024 14:53:05 +0200 Subject: [PATCH] Allow custom validation for "write-test" task so one can ensure that tests for Spring Boot actually spin up Spring Boot Part of #365 --- README.md | 15 +++ evaluate/task/write-test.go | 4 +- evaluate/task/write-test_test.go | 147 +++++++++++++++------ language/golang/language.go | 2 + language/java/language.go | 2 + language/language.go | 2 + language/ruby/language.go | 2 + language/testing/language.go | 1 + task/config.go | 25 ++++ testdata/java/spring-plain/repository.json | 5 + 10 files changed, 160 insertions(+), 45 deletions(-) diff --git a/README.md b/README.md index 3a179f7f..d6b7c85e 100644 --- a/README.md +++ b/README.md @@ -250,6 +250,21 @@ It is possible to configure some model prompt parameters through `repository.jso This `prompt.test-framework` setting is currently only respected for the test generation task `write-tests`. +When task results are validated, some repositories might require custom logic. For example: generating tests for a Spring Boot project requires ensuring that the tests used an actual Spring context (i.e. Spring Boot was initialized when the tests were executed). Therefore, the `repository.json` supports adding rudimentary custom validation: + +```json +{ + "tasks": ["write-tests"], + "validation": { + "execution": { + "stdout": "Initializing Spring" // Ensure the string "Initializing Spring" is contained in the execution output. + } + } +} +``` + +This `validation.execution.stdout` setting is currently only respected for the test generation task `write-tests`. + ## Tasks ### Task: Test Generation diff --git a/evaluate/task/write-test.go b/evaluate/task/write-test.go index 0b0aa718..9bab328f 100644 --- a/evaluate/task/write-test.go +++ b/evaluate/task/write-test.go @@ -176,7 +176,7 @@ func runModelAndSymflowerFix(ctx evaltask.Context, taskLogger *taskLogger, model problems = append(problems, ps...) if err != nil { problems = append(problems, pkgerrors.WithMessage(err, filePath)) - } else { + } else if ctx.Repository.Configuration().Validation.Execution.Validate(testResult.StdOut) { taskLogger.Printf("Executes tests with %d coverage objects", testResult.Coverage) modelAssessment.Award(metrics.AssessmentKeyFilesExecuted) modelAssessment.AwardPoints(metrics.AssessmentKeyCoverage, testResult.Coverage) @@ -187,7 +187,7 @@ func runModelAndSymflowerFix(ctx evaltask.Context, taskLogger *taskLogger, model problems = append(problems, ps...) if err != nil { problems = append(problems, err) - } else { + } else if ctx.Repository.Configuration().Validation.Execution.Validate(withSymflowerFixTestResult.StdOut) { ctx.Logger.Printf("with symflower repair: Executes tests with %d coverage objects", withSymflowerFixTestResult.Coverage) // Symflower was able to fix a failure so now update the assessment with the improved results. diff --git a/evaluate/task/write-test_test.go b/evaluate/task/write-test_test.go index 920a35c0..35c3cd6a 100644 --- a/evaluate/task/write-test_test.go +++ b/evaluate/task/write-test_test.go @@ -395,16 +395,17 @@ func TestWriteTestsRun(t *testing.T) { }) } - { - temporaryDirectoryPath := t.TempDir() - repositoryPath := filepath.Join(temporaryDirectoryPath, "java", "spring-plain") - require.NoError(t, osutil.CopyTree(filepath.Join("..", "..", "testdata", "java", "spring-plain"), repositoryPath)) - modelMock := modeltesting.NewMockCapabilityWriteTestsNamed(t, "mocked-model") - modelMock.RegisterGenerateSuccessValidateContext(t, func(t *testing.T, c model.Context) { - args, ok := c.Arguments.(*ArgumentsWriteTest) - require.Truef(t, ok, "unexpected type %T", c.Arguments) - assert.Equal(t, "JUnit 5 for Spring", args.TestFramework) - }, filepath.Join("src", "test", "java", "com", "example", "controller", "SomeControllerTest.java"), bytesutil.StringTrimIndentations(` + t.Run("Spring Boot", func(t *testing.T) { + { + temporaryDirectoryPath := t.TempDir() + repositoryPath := filepath.Join(temporaryDirectoryPath, "java", "spring-plain") + require.NoError(t, osutil.CopyTree(filepath.Join("..", "..", "testdata", "java", "spring-plain"), repositoryPath)) + modelMock := modeltesting.NewMockCapabilityWriteTestsNamed(t, "mocked-model") + modelMock.RegisterGenerateSuccessValidateContext(t, func(t *testing.T, c model.Context) { + args, ok := c.Arguments.(*ArgumentsWriteTest) + require.Truef(t, ok, "unexpected type %T", c.Arguments) + assert.Equal(t, "JUnit 5 for Spring", args.TestFramework) + }, filepath.Join("src", "test", "java", "com", "example", "controller", "SomeControllerTest.java"), bytesutil.StringTrimIndentations(` package com.example.controller; import org.junit.jupiter.api.*; @@ -431,45 +432,105 @@ func TestWriteTestsRun(t *testing.T) { } `), metricstesting.AssessmentsWithProcessingTime) - validate(t, &tasktesting.TestCaseTask{ - Name: "Spring Boot", + validate(t, &tasktesting.TestCaseTask{ + Name: "Spring Boot Test", - Model: modelMock, - Language: &java.Language{}, - TestDataPath: temporaryDirectoryPath, - RepositoryPath: filepath.Join("java", "spring-plain"), + Model: modelMock, + Language: &java.Language{}, + TestDataPath: temporaryDirectoryPath, + RepositoryPath: filepath.Join("java", "spring-plain"), - ExpectedRepositoryAssessment: map[evaltask.Identifier]metrics.Assessments{ - IdentifierWriteTests: metrics.Assessments{ - metrics.AssessmentKeyFilesExecutedMaximumReachable: 1, - metrics.AssessmentKeyFilesExecuted: 1, - metrics.AssessmentKeyCoverage: 20, - metrics.AssessmentKeyResponseNoError: 1, + ExpectedRepositoryAssessment: map[evaltask.Identifier]metrics.Assessments{ + IdentifierWriteTests: metrics.Assessments{ + metrics.AssessmentKeyFilesExecutedMaximumReachable: 1, + metrics.AssessmentKeyFilesExecuted: 1, + metrics.AssessmentKeyCoverage: 20, + metrics.AssessmentKeyResponseNoError: 1, + }, + IdentifierWriteTestsSymflowerFix: metrics.Assessments{ + metrics.AssessmentKeyFilesExecutedMaximumReachable: 1, + metrics.AssessmentKeyFilesExecuted: 1, + metrics.AssessmentKeyCoverage: 20, + metrics.AssessmentKeyResponseNoError: 1, + }, + IdentifierWriteTestsSymflowerTemplate: metrics.Assessments{ + metrics.AssessmentKeyFilesExecutedMaximumReachable: 1, + metrics.AssessmentKeyFilesExecuted: 1, + metrics.AssessmentKeyCoverage: 20, + metrics.AssessmentKeyResponseNoError: 1, + }, + IdentifierWriteTestsSymflowerTemplateSymflowerFix: metrics.Assessments{ + metrics.AssessmentKeyFilesExecutedMaximumReachable: 1, + metrics.AssessmentKeyFilesExecuted: 1, + metrics.AssessmentKeyCoverage: 20, + metrics.AssessmentKeyResponseNoError: 1, + }, }, - IdentifierWriteTestsSymflowerFix: metrics.Assessments{ - metrics.AssessmentKeyFilesExecutedMaximumReachable: 1, - metrics.AssessmentKeyFilesExecuted: 1, - metrics.AssessmentKeyCoverage: 20, - metrics.AssessmentKeyResponseNoError: 1, + ValidateLog: func(t *testing.T, data string) { + assert.Equal(t, 2, strings.Count(data, "Starting SomeControllerTest using Java"), "Expected two successful Spring startup announcements (one bare and one for template)") }, - IdentifierWriteTestsSymflowerTemplate: metrics.Assessments{ - metrics.AssessmentKeyFilesExecutedMaximumReachable: 1, - metrics.AssessmentKeyFilesExecuted: 1, - metrics.AssessmentKeyCoverage: 20, - metrics.AssessmentKeyResponseNoError: 1, + }) + } + { + temporaryDirectoryPath := t.TempDir() + repositoryPath := filepath.Join(temporaryDirectoryPath, "java", "spring-plain") + require.NoError(t, osutil.CopyTree(filepath.Join("..", "..", "testdata", "java", "spring-plain"), repositoryPath)) + modelMock := modeltesting.NewMockCapabilityWriteTestsNamed(t, "mocked-model") + modelMock.RegisterGenerateSuccessValidateContext(t, func(t *testing.T, c model.Context) { + args, ok := c.Arguments.(*ArgumentsWriteTest) + require.Truef(t, ok, "unexpected type %T", c.Arguments) + assert.Equal(t, "JUnit 5 for Spring", args.TestFramework) + }, filepath.Join("src", "test", "java", "com", "example", "controller", "SomeControllerTest.java"), bytesutil.StringTrimIndentations(` + package com.example.controller; + + import com.example.controller.SomeController; + import org.junit.jupiter.api.Test; + + import static org.junit.jupiter.api.Assertions.assertEquals; + + class SomeControllerTests { + + @Test // Normal JUnit tests instead of Spring Boot. + void helloGet() { + SomeController controller = new SomeController(); + String result = controller.helloGet(); + assertEquals("get!", result); + } + } + `), metricstesting.AssessmentsWithProcessingTime) + + validate(t, &tasktesting.TestCaseTask{ + Name: "Plain JUnit Test", + + Model: modelMock, + Language: &java.Language{}, + TestDataPath: temporaryDirectoryPath, + RepositoryPath: filepath.Join("java", "spring-plain"), + + ExpectedRepositoryAssessment: map[evaltask.Identifier]metrics.Assessments{ + IdentifierWriteTests: metrics.Assessments{ + metrics.AssessmentKeyFilesExecutedMaximumReachable: 1, + metrics.AssessmentKeyResponseNoError: 1, + }, + IdentifierWriteTestsSymflowerFix: metrics.Assessments{ + metrics.AssessmentKeyFilesExecutedMaximumReachable: 1, + metrics.AssessmentKeyResponseNoError: 1, + }, + IdentifierWriteTestsSymflowerTemplate: metrics.Assessments{ + metrics.AssessmentKeyFilesExecutedMaximumReachable: 1, + metrics.AssessmentKeyResponseNoError: 1, + }, + IdentifierWriteTestsSymflowerTemplateSymflowerFix: metrics.Assessments{ + metrics.AssessmentKeyFilesExecutedMaximumReachable: 1, + metrics.AssessmentKeyResponseNoError: 1, + }, }, - IdentifierWriteTestsSymflowerTemplateSymflowerFix: metrics.Assessments{ - metrics.AssessmentKeyFilesExecutedMaximumReachable: 1, - metrics.AssessmentKeyFilesExecuted: 1, - metrics.AssessmentKeyCoverage: 20, - metrics.AssessmentKeyResponseNoError: 1, + ValidateLog: func(t *testing.T, data string) { + assert.Contains(t, data, "Tests run: 1") // Tests are running but they are not Spring Boot. }, - }, - ValidateLog: func(t *testing.T, data string) { - assert.Equal(t, 2, strings.Count(data, "Starting SomeControllerTest using Java"), "Expected two successful Spring startup announcements (one bare and one for template)") - }, - }) - } + }) + } + }) } func TestValidateWriteTestsRepository(t *testing.T) { diff --git a/language/golang/language.go b/language/golang/language.go index 88862d85..4a38aff4 100644 --- a/language/golang/language.go +++ b/language/golang/language.go @@ -109,6 +109,8 @@ func (l *Language) ExecuteTests(logger *log.Logger, repositoryPath string) (test testResult = &language.TestResult{ TestsTotal: uint(testsTotal), TestsPass: uint(testsPass), + + StdOut: commandOutput, } testResult.Coverage, err = language.CoverageObjectCountOfFile(logger, coverageFilePath) if err != nil { diff --git a/language/java/language.go b/language/java/language.go index 6a3b76f4..447340e2 100644 --- a/language/java/language.go +++ b/language/java/language.go @@ -107,6 +107,8 @@ func (l *Language) ExecuteTests(logger *log.Logger, repositoryPath string) (test testResult = &language.TestResult{ TestsTotal: uint(testsTotal), TestsPass: uint(testsPass), + + StdOut: commandOutput, } testResult.Coverage, err = language.CoverageObjectCountOfFile(logger, coverageFilePath) diff --git a/language/language.go b/language/language.go index 550657a8..2c939cbf 100644 --- a/language/language.go +++ b/language/language.go @@ -125,6 +125,8 @@ type TestResult struct { TestsPass uint Coverage uint64 + + StdOut string } // PassingTestsPercentage returns the percentage of passing tests. diff --git a/language/ruby/language.go b/language/ruby/language.go index 78c49ec2..cd6c7df2 100644 --- a/language/ruby/language.go +++ b/language/ruby/language.go @@ -101,6 +101,8 @@ func (l *Language) ExecuteTests(logger *log.Logger, repositoryPath string) (test testResult = &language.TestResult{ TestsTotal: uint(testsTotal), TestsPass: uint(testsPass), + + StdOut: commandOutput, } testResult.Coverage, err = language.CoverageObjectCountOfFile(logger, coverageFilePath) diff --git a/language/testing/language.go b/language/testing/language.go index 07196ed1..d3b993a9 100644 --- a/language/testing/language.go +++ b/language/testing/language.go @@ -57,6 +57,7 @@ func (tc *TestCaseExecuteTests) Validate(t *testing.T) { assert.ErrorContains(t, actualError, tc.ExpectedErrorText) } else { assert.NoError(t, actualError) + actualTestResult.StdOut = "" assert.Equal(t, tc.ExpectedTestResult, actualTestResult) } }) diff --git a/task/config.go b/task/config.go index 8129d9f7..a1c994d7 100644 --- a/task/config.go +++ b/task/config.go @@ -5,6 +5,7 @@ import ( "errors" "os" "path/filepath" + "regexp" "strings" pkgerrors "github.com/pkg/errors" @@ -24,6 +25,17 @@ type RepositoryConfiguration struct { // TestFramework overwrites the language-specific test framework to use. TestFramework string `json:"test-framework,omitempty"` } `json:",omitempty"` + + // Validation holds quality gates for evaluation. + Validation struct { + Execution RepositoryConfigurationExecution `json:",omitempty"` + } +} + +// RepositoryConfigurationExecution execution-related quality gates for evaluation. +type RepositoryConfigurationExecution struct { + // StdOutRE holds a regular expression that must be part of execution standard output. + StdOutRE string `json:"stdout,omitempty"` } // RepositoryConfigurationFileName holds the file name for a repository configuration. @@ -70,6 +82,10 @@ func (rc *RepositoryConfiguration) validate(validTasks []Identifier) (err error) } } + if _, err := regexp.Compile(rc.Validation.Execution.StdOutRE); err != nil { + return pkgerrors.WithMessagef(err, "invalid regular expression %q", rc.Validation.Execution.StdOutRE) + } + return nil } @@ -85,3 +101,12 @@ func (rc *RepositoryConfiguration) IsFilePathIgnored(filePath string) bool { return false } + +// Validate validates execution outcomes against the configured quality gates. +func (e *RepositoryConfigurationExecution) Validate(stdout string) bool { + if e.StdOutRE != "" { + return regexp.MustCompile(e.StdOutRE).MatchString(stdout) + } + + return true +} diff --git a/testdata/java/spring-plain/repository.json b/testdata/java/spring-plain/repository.json index 1f816571..7a17122e 100644 --- a/testdata/java/spring-plain/repository.json +++ b/testdata/java/spring-plain/repository.json @@ -3,5 +3,10 @@ "ignore": ["src/main/java/com/example/Application.java"], "prompt": { "test-framework": "JUnit 5 for Spring" + }, + "validation": { + "execution": { + "stdout": "Initializing Spring" + } } }