Skip to content

Commit

Permalink
refactor, Let the caller create the assessment of a "symflower fix" run
Browse files Browse the repository at this point in the history
  • Loading branch information
ahumenberger committed Jul 23, 2024
1 parent 241a3c2 commit f0262c7
Show file tree
Hide file tree
Showing 2 changed files with 13 additions and 13 deletions.
16 changes: 4 additions & 12 deletions evaluate/task/symflower-fix.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ import (
"time"

pkgerrors "github.com/pkg/errors"
"github.com/symflower/eval-dev-quality/evaluate/metrics"
"github.com/symflower/eval-dev-quality/language"
"github.com/symflower/eval-dev-quality/log"
evaltask "github.com/symflower/eval-dev-quality/task"
Expand Down Expand Up @@ -33,27 +32,20 @@ func symflowerFix(logger *log.Logger, repositoryPath string, language language.L
}

// ExecuteWithSymflowerFix runs the "symflower fix" command and calculates the new assessments.
func ExecuteWithSymflowerFix(ctx evaltask.Context, logger *log.Logger, packagePath string) (assessments metrics.Assessments, problems []error, err error) {
func ExecuteWithSymflowerFix(ctx evaltask.Context, logger *log.Logger, packagePath string) (testResult *language.TestResult, processingTime uint64, problems []error, err error) {
// Run "symflower fix" if the model response fails to execute.
logger.Print("model response alone failed execution, attempting to fix with \"symflower fix \"")

duration, err := symflowerFix(logger, packagePath, ctx.Language)
if err != nil {
return nil, problems, pkgerrors.WithStack(err)
return nil, 0, nil, pkgerrors.WithStack(err)
}

testResult, ps, err := ctx.Language.ExecuteTests(logger, packagePath)
problems = append(problems, ps...)
if err != nil {
return nil, problems, pkgerrors.WithMessage(err, "symflower fix")
return testResult, duration, problems, pkgerrors.WithMessage(err, "symflower fix")
}
logger.Printf("with symflower repair: Executes tests with %d coverage objects", testResult.Coverage)

// Symflower was able to fix a failure so now update the assessment with the improved results.
assessments = metrics.NewAssessments()
assessments[metrics.AssessmentKeyProcessingTime] = duration
assessments.Award(metrics.AssessmentKeyFilesExecuted)
assessments.AwardPoints(metrics.AssessmentKeyCoverage, testResult.Coverage)

return assessments, problems, nil
return testResult, duration, problems, nil
}
10 changes: 9 additions & 1 deletion evaluate/task/task-write-test.go
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ func (t *TaskWriteTests) Run(ctx evaltask.Context) (repositoryAssessment map[eva

// Run "symflower fix" if the model response fails to execute.
if ctx.Language.ID() == "golang" { // Currently we only support Go for "symflower fix".
withSymflowerFixAssessments, ps, err := ExecuteWithSymflowerFix(ctx, taskLogger.Logger, ctx.Repository.DataPath())
withSymflowerFixTestResult, processingTime, ps, err := ExecuteWithSymflowerFix(ctx, taskLogger.Logger, ctx.Repository.DataPath())
problems = append(problems, ps...)
if err != nil {
problems = append(problems, err)
Expand All @@ -106,6 +106,14 @@ func (t *TaskWriteTests) Run(ctx evaltask.Context) (repositoryAssessment map[eva

continue
} else {
ctx.Logger.Printf("with symflower repair: Executes tests with %d coverage objects", withSymflowerFixTestResult.Coverage)

// Symflower was able to fix a failure so now update the assessment with the improved results.
withSymflowerFixAssessments := metrics.NewAssessments()
withSymflowerFixAssessments[metrics.AssessmentKeyProcessingTime] = processingTime
withSymflowerFixAssessments.Award(metrics.AssessmentKeyFilesExecuted)
withSymflowerFixAssessments.AwardPoints(metrics.AssessmentKeyCoverage, withSymflowerFixTestResult.Coverage)

withSymflowerAssessmentForFile = metrics.CombineWithSymflowerFixAssessments(modelAssessmentForFile, withSymflowerFixAssessments)
}
}
Expand Down

0 comments on commit f0262c7

Please sign in to comment.