Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Clarify wording, logs, debugging regarding symflower fix #352

Merged
merged 3 commits into from
Oct 2, 2024
Merged

Clarify wording, logs, debugging regarding symflower fix #352

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
d3a6c0d
Compare assessment tuples in map form cause otherwise the diff is hor…
bauersimon Oct 2, 2024
82e9d3f
refactor, Unify log message for "write-test" and "transpile" with cod…
bauersimon Oct 2, 2024
027facf
refactor, Reword assessment for clarity and remove "symflower-fix" fi…
bauersimon Oct 2, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion evaluate/evaluate_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -144,7 +144,7 @@ func TestEvaluate(t *testing.T) {
return nil
}))

assert.ElementsMatch(t, tc.ExpectedAssessments, actualAssessments)
assert.Equal(t, tc.ExpectedAssessments.ToMap(), actualAssessments.ToMap())
assert.Equal(t, tc.ExpectedTotalScore, actualTotalScore)

if tc.ExpectedOutputValidate != nil {
Expand Down
0 evaluate/task/symflower-fix.go → evaluate/task/symflower.go
View file
Open in desktop
File renamed without changes.
2 changes: 1 addition & 1 deletion evaluate/task/task-transpile.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -132,7 +132,7 @@ func (t *TaskTranspile) Run(ctx evaltask.Context) (repositoryAssessment map[eval
continue
} else {
testsPassing := withSymflowerFixTestResult.TestsPass
taskLogger.Printf("Executes tests with %d tests passing after \"symflower fix\"", testsPassing)
taskLogger.Printf("with symflower repair: Executes tests with %d tests passing", testsPassing)

// Symflower was able to fix a failure so now update the assessment with the improved results.
withSymflowerFixAssessments := metrics.NewAssessments()
Expand Down
16 changes: 8 additions & 8 deletions evaluate/task/task-write-test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,15 +47,15 @@ func (t *TaskWriteTests) Run(ctx evaltask.Context) (repositoryAssessment map[eva
}

modelAssessment := metrics.NewAssessments()
withSymflowerAssessment := metrics.NewAssessments()
withSymflowerFixAssessment := metrics.NewAssessments()

maximumReachableFiles := uint64(len(filePaths))
modelAssessment[metrics.AssessmentKeyFilesExecutedMaximumReachable] = maximumReachableFiles
withSymflowerAssessment[metrics.AssessmentKeyFilesExecutedMaximumReachable] = maximumReachableFiles
withSymflowerFixAssessment[metrics.AssessmentKeyFilesExecutedMaximumReachable] = maximumReachableFiles

for _, filePath := range filePaths {
modelAssessmentForFile := metrics.NewAssessments()
withSymflowerAssessmentForFile := modelAssessmentForFile // The symflower assessment tracks how the model result can be improved in case of a failure, so just link to the model assessment until a failure actually happens.
withSymflowerFixAssessmentForFile := modelAssessmentForFile // The symflower assessment tracks how the model result can be improved in case of a failure, so just link to the model assessment until a failure actually happens.

if err := ctx.Repository.Reset(ctx.Logger); err != nil {
ctx.Logger.Panicf("ERROR: unable to reset temporary repository path: %s", err)
Expand Down Expand Up @@ -89,7 +89,7 @@ func (t *TaskWriteTests) Run(ctx evaltask.Context) (repositoryAssessment map[eva
// If there is an execution timeout do not run "symflower fix" because the code itself is correct.
if errors.Is(err, context.DeadlineExceeded) {
modelAssessment.Add(modelAssessmentForFile)
withSymflowerAssessment.Add(withSymflowerAssessmentForFile)
withSymflowerFixAssessment.Add(withSymflowerFixAssessmentForFile)

continue
}
Expand All @@ -102,7 +102,7 @@ func (t *TaskWriteTests) Run(ctx evaltask.Context) (repositoryAssessment map[eva
problems = append(problems, err)

modelAssessment.Add(modelAssessmentForFile)
withSymflowerAssessment.Add(withSymflowerAssessmentForFile)
withSymflowerFixAssessment.Add(withSymflowerFixAssessmentForFile)

continue
} else {
Expand All @@ -114,7 +114,7 @@ func (t *TaskWriteTests) Run(ctx evaltask.Context) (repositoryAssessment map[eva
withSymflowerFixAssessments.Award(metrics.AssessmentKeyFilesExecuted)
withSymflowerFixAssessments.AwardPoints(metrics.AssessmentKeyCoverage, withSymflowerFixTestResult.Coverage)

withSymflowerAssessmentForFile = metrics.CombineWithSymflowerFixAssessments(modelAssessmentForFile, withSymflowerFixAssessments)
withSymflowerFixAssessmentForFile = metrics.CombineWithSymflowerFixAssessments(modelAssessmentForFile, withSymflowerFixAssessments)
}
}
} else {
Expand All @@ -124,12 +124,12 @@ func (t *TaskWriteTests) Run(ctx evaltask.Context) (repositoryAssessment map[eva
}

modelAssessment.Add(modelAssessmentForFile)
withSymflowerAssessment.Add(withSymflowerAssessmentForFile)
withSymflowerFixAssessment.Add(withSymflowerFixAssessmentForFile)
}

repositoryAssessment = map[evaltask.Identifier]metrics.Assessments{
IdentifierWriteTests: modelAssessment,
IdentifierWriteTestsSymflowerFix: withSymflowerAssessment,
IdentifierWriteTestsSymflowerFix: withSymflowerFixAssessment,
}

return repositoryAssessment, problems, nil
Expand Down