Add "smart test skeleton" model to Symflower provider to be able to t…

…estdrive Spring examples

Closes #365

evaluate/task/test-integration/task_test.go

@@ -37,7 +37,7 @@ func TestWriteTestsRun(t *testing.T) {
 	validate(t, &tasktesting.TestCaseTask{
 		Name: "Plain",
 
-		Model:          symflower.NewModel(),
+		Model:          symflower.NewModelSymbolicExecution(),
 		Language:       &golang.Language{},
 		TestDataPath:   filepath.Join("..", "..", "..", "testdata"),
 		RepositoryPath: filepath.Join("golang", "plain"),

model/symflower/symflower.go

@@ -16,34 +16,55 @@ import (
 	"github.com/symflower/eval-dev-quality/util"
 )
 
-// defaultSymbolicExecutionTimeout defines the default symbolic execution timeout.
-var defaultSymbolicExecutionTimeout = time.Duration(10 * time.Minute)
+// defaultSymflowerTimeout defines the default timeout for Symflower models.
+var defaultSymflowerTimeout = time.Duration(10 * time.Minute)
 
 // Model holds a Symflower model using the locally installed CLI.
 type Model struct {
 	// symbolicExecutionTimeout defines the symbolic execution timeout.
 	symbolicExecutionTimeout time.Duration
+
+	// command is the sub-command to use in the Symflower binary.
+	command string
+	// id is the ID of the model.
+	id string
+}
+
+// NewModelSymbolicExecution returns a symbolic execution Symflower model.
+func NewModelSymbolicExecution() (model *Model) {
+	return NewModelSymbolicExecutionWithTimeout(defaultSymflowerTimeout)
 }
 
-// NewModel returns a Symflower model.
-func NewModel() (model *Model) {
+// NewModelSymbolicExecutionWithTimeout returns a Symflower model with a given timeout.
+func NewModelSymbolicExecutionWithTimeout(timeout time.Duration) (model *Model) {
 	return &Model{
-		symbolicExecutionTimeout: defaultSymbolicExecutionTimeout,
+		symbolicExecutionTimeout: timeout,
+
+		command: "unit-tests",
+		id:      "symbolic-execution",
 	}
 }
 
-// NewModelWithTimeout returns a Symflower model with a given timeout.
-func NewModelWithTimeout(timeout time.Duration) (model *Model) {
+// NewModelSmartTemplate returns a symbolic execution Symflower model.
+func NewModelSmartTemplate() (model *Model) {
+	return NewModelSmartTemplateWithTimeout(defaultSymflowerTimeout)
+}
+
+// NewModelSmartTemplateWithTimeout returns a Symflower model with a given timeout.
+func NewModelSmartTemplateWithTimeout(timeout time.Duration) (model *Model) {
 	return &Model{
 		symbolicExecutionTimeout: timeout,
+
+		command: "unit-test-skeletons",
+		id:      "smart-template",
 	}
 }
 
 var _ model.Model = (*Model)(nil)
 
 // ID returns the unique ID of this model.
 func (m *Model) ID() (id string) {
-	return "symflower" + provider.ProviderModelSeparator + "symbolic-execution"
+	return "symflower" + provider.ProviderModelSeparator + m.id
 }
 
 // MetaInformation returns the meta information of a model.
@@ -62,7 +83,7 @@ func (m *Model) WriteTests(ctx model.Context) (assessment metrics.Assessments, e
 
 	output, err := util.CommandWithResult(ctxWithTimeout, ctx.Logger, &util.Command{
 		Command: []string{
-			tools.SymflowerPath, "unit-tests",
+			tools.SymflowerPath, m.command,
 			"--code-disable-fetch-dependencies",
 			"--workspace", ctx.RepositoryPath,
 			ctx.FilePath,

model/symflower/symflower_test.go

@@ -35,10 +35,11 @@ func TestModelGenerateTestsForFile(t *testing.T) {
 		RepositoryChange func(t *testing.T, repositoryPath string)
 		FilePath         string
 
-		ExpectedAssessment   metrics.Assessments
-		ExpectedCoverage     uint64
-		ExpectedError        error
-		ExpectedErrorHandler func(t *testing.T, err error)
+		ExpectedAssessment      metrics.Assessments
+		ExpectedCoverage        uint64
+		ExpectedError           error
+		ExpectedErrorHandler    func(t *testing.T, err error)
+		ExpectedProblemsHandler func(t *testing.T, problems []error)
 	}
 
 	validate := func(t *testing.T, tc *testCase) {
@@ -81,7 +82,11 @@ func TestModelGenerateTestsForFile(t *testing.T) {
 				assert.Equal(t, metricstesting.Clean(tc.ExpectedAssessment), metricstesting.Clean(actualAssessment))
 				actualTestResult, actualProblems, err := tc.Language.ExecuteTests(logger, repositoryPath)
 				require.NoError(t, err)
-				require.Empty(t, actualProblems)
+				if tc.ExpectedProblemsHandler != nil {
+					tc.ExpectedProblemsHandler(t, actualProblems)
+				} else {
+					require.Empty(t, actualProblems)
+				}
 				assert.Equal(t, tc.ExpectedCoverage, actualTestResult.Coverage)
 			}
 		})
@@ -123,7 +128,7 @@ func TestModelGenerateTestsForFile(t *testing.T) {
 		validate(t, &testCase{
 			Name: "Timeout",
 
-			Model:    NewModelWithTimeout(time.Duration(1 * time.Millisecond)),
+			Model:    NewModelSymbolicExecutionWithTimeout(time.Duration(1 * time.Millisecond)),
 			Language: &java.Language{},
 
 			RepositoryPath: filepath.Join("..", "..", "testdata", "java", "light"),
@@ -139,6 +144,27 @@ func TestModelGenerateTestsForFile(t *testing.T) {
 			},
 		})
 	})
+	validate(t, &testCase{
+		Name: "Skeleton",
+
+		Language: &golang.Language{},
+		Model:    NewModelSmartTemplate(),
+
+		RepositoryPath: filepath.Join("..", "..", "testdata", "golang", "light"),
+		FilePath:       "simpleIfElse.go",
+
+		ExpectedAssessment: metrics.Assessments{
+			metrics.AssessmentKeyGenerateTestsForFileCharacterCount: 389,
+			metrics.AssessmentKeyResponseCharacterCount:             389,
+			metrics.AssessmentKeyResponseNoExcess:                   1,
+			metrics.AssessmentKeyResponseWithCode:                   1,
+		},
+		ExpectedCoverage: 2,
+		ExpectedProblemsHandler: func(t *testing.T, problems []error) {
+			require.Len(t, problems, 1)
+			assert.Contains(t, problems[0].Error(), "DONE 2 tests, 2 failures") // Sub-tests count as tests as well, and we have one "t.Run" within the generated "validate" function.
+		},
+	})
 }
 
 func TestExtractGeneratedFilePaths(t *testing.T) {

provider/symflower/symflower.go

@@ -42,6 +42,7 @@ func (p *Provider) ID() (id string) {
 // Models returns which models are available to be queried via this provider.
 func (p *Provider) Models() (models []model.Model, err error) {
 	return []model.Model{
-		symflower.NewModel(),
+		symflower.NewModelSymbolicExecution(),
+		symflower.NewModelSmartTemplate(),
 	}, nil
 }