From 942e7f433937b0ec220fd3fe686798a52aaf397f Mon Sep 17 00:00:00 2001 From: Andreas Humenberger Date: Wed, 17 Jul 2024 14:11:14 +0200 Subject: [PATCH] refactor, Let the caller create the assessment of a "symflower fix" run --- evaluate/task/symflower-fix.go | 16 ++++------------ evaluate/task/task-write-test.go | 10 +++++++++- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/evaluate/task/symflower-fix.go b/evaluate/task/symflower-fix.go index a63c2412..1c8adfa4 100644 --- a/evaluate/task/symflower-fix.go +++ b/evaluate/task/symflower-fix.go @@ -5,7 +5,6 @@ import ( "time" pkgerrors "github.com/pkg/errors" - "github.com/symflower/eval-dev-quality/evaluate/metrics" "github.com/symflower/eval-dev-quality/language" "github.com/symflower/eval-dev-quality/log" evaltask "github.com/symflower/eval-dev-quality/task" @@ -33,27 +32,20 @@ func symflowerFix(logger *log.Logger, repositoryPath string, language language.L } // ExecuteWithSymflowerFix runs the "symflower fix" command and calculates the new assessments. -func ExecuteWithSymflowerFix(ctx evaltask.Context, logger *log.Logger, packagePath string) (assessments metrics.Assessments, problems []error, err error) { +func ExecuteWithSymflowerFix(ctx evaltask.Context, logger *log.Logger, packagePath string) (testResult *language.TestResult, processingTime uint64, problems []error, err error) { // Run "symflower fix" if the model response fails to execute. logger.Print("model response alone failed execution, attempting to fix with \"symflower fix \"") duration, err := symflowerFix(logger, packagePath, ctx.Language) if err != nil { - return nil, problems, pkgerrors.WithStack(err) + return nil, 0, nil, pkgerrors.WithStack(err) } testResult, ps, err := ctx.Language.ExecuteTests(logger, packagePath) problems = append(problems, ps...) if err != nil { - return nil, problems, pkgerrors.WithMessage(err, "symflower fix") + return testResult, duration, problems, pkgerrors.WithMessage(err, "symflower fix") } - logger.Printf("with symflower repair: Executes tests with %d coverage objects", testResult.Coverage) - // Symflower was able to fix a failure so now update the assessment with the improved results. - assessments = metrics.NewAssessments() - assessments[metrics.AssessmentKeyProcessingTime] = duration - assessments.Award(metrics.AssessmentKeyFilesExecuted) - assessments.AwardPoints(metrics.AssessmentKeyCoverage, testResult.Coverage) - - return assessments, problems, nil + return testResult, duration, problems, nil } diff --git a/evaluate/task/task-write-test.go b/evaluate/task/task-write-test.go index 1215a72f..9b05e1ff 100644 --- a/evaluate/task/task-write-test.go +++ b/evaluate/task/task-write-test.go @@ -96,7 +96,7 @@ func (t *TaskWriteTests) Run(ctx evaltask.Context) (repositoryAssessment map[eva // Run "symflower fix" if the model response fails to execute. if ctx.Language.ID() == "golang" { // Currently we only support Go for "symflower fix". - withSymflowerFixAssessments, ps, err := ExecuteWithSymflowerFix(ctx, taskLogger.Logger, ctx.Repository.DataPath()) + withSymflowerFixTestResult, processingTime, ps, err := ExecuteWithSymflowerFix(ctx, taskLogger.Logger, ctx.Repository.DataPath()) problems = append(problems, ps...) if err != nil { problems = append(problems, err) @@ -106,6 +106,14 @@ func (t *TaskWriteTests) Run(ctx evaltask.Context) (repositoryAssessment map[eva continue } else { + ctx.Logger.Printf("with symflower repair: Executes tests with %d coverage objects", withSymflowerFixTestResult.Coverage) + + // Symflower was able to fix a failure so now update the assessment with the improved results. + withSymflowerFixAssessments := metrics.NewAssessments() + withSymflowerFixAssessments[metrics.AssessmentKeyProcessingTime] = processingTime + withSymflowerFixAssessments.Award(metrics.AssessmentKeyFilesExecuted) + withSymflowerFixAssessments.AwardPoints(metrics.AssessmentKeyCoverage, withSymflowerFixTestResult.Coverage) + withSymflowerAssessmentForFile = metrics.CombineWithSymflowerFixAssessments(modelAssessmentForFile, withSymflowerFixAssessments) } }