Task for code transpilation, so models can transpile Go code to Java …

…and back

Closes #201

.mockery.yml

@@ -15,6 +15,7 @@ packages:
       Model:
       CapabilityWriteTests:
       CapabilityRepairCode:
+      CapabilityTranspile:
   github.com/symflower/eval-dev-quality/provider:
     interfaces:
       Loader:

evaluate/task/task-transpile.go

@@ -0,0 +1,145 @@
+package task
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	pkgerrors "github.com/pkg/errors"
+	"github.com/symflower/eval-dev-quality/evaluate/metrics"
+	"github.com/symflower/eval-dev-quality/language"
+	"github.com/symflower/eval-dev-quality/language/golang"
+	"github.com/symflower/eval-dev-quality/language/java"
+	"github.com/symflower/eval-dev-quality/log"
+	"github.com/symflower/eval-dev-quality/model"
+	evaltask "github.com/symflower/eval-dev-quality/task"
+)
+
+// TaskTranspile holds the transpilation task.
+type TaskTranspile struct{}
+
+// TaskArgumentsTranspile holds extra arguments to be used in a query prompt.
+type TaskArgumentsTranspile struct {
+	// OriginLanguage holds the language we are transpiling from.
+	OriginLanguage language.Language
+	// StubFilePath holds the path for the file containing just the function signature of the language we are transpiling to.
+	StubFilePath string
+}
+
+var _ evaltask.Task = (*TaskTranspile)(nil)
+
+// Identifier returns the transpilation task identifier.
+func (t *TaskTranspile) Identifier() evaltask.Identifier {
+	return IdentifierTranspile
+}
+
+// Run transpiles code between Go and Java and runs predefined tests to check if the transpilation was successful.
+func (t *TaskTranspile) Run(ctx evaltask.Context) (repositoryAssessment map[evaltask.Identifier]metrics.Assessments, problems []error, err error) {
+	modelCapability, ok := ctx.Model.(model.CapabilityTranspile)
+	if !ok {
+		return nil, nil, pkgerrors.Wrap(evaltask.ErrTaskUnsupportedByModel, fmt.Sprintf("%q does not support %q", ctx.Model.ID(), string(t.Identifier())))
+	}
+
+	taskLogger, err := newTaskLogger(ctx, t)
+	if err != nil {
+		return nil, nil, err
+	}
+	defer func() {
+		taskLogger.finalize(problems)
+	}()
+
+	var packagePaths []string
+	files, err := os.ReadDir(ctx.Repository.DataPath())
+	if err != nil {
+		return nil, nil, pkgerrors.WithStack(err)
+	}
+	for _, file := range files {
+		if file.IsDir() && !strings.HasPrefix(file.Name(), ".") { // Ignore hidden directories.
+			packagePaths = append(packagePaths, file.Name())
+		}
+	}
+
+	modelAssessments := metrics.NewAssessments()
+	for _, packagePath := range packagePaths {
+		if err := ctx.Repository.Reset(ctx.Logger); err != nil {
+			ctx.Logger.Panicf("ERROR: unable to reset temporary repository path: %s", err)
+		}
+
+		var originLanguage language.Language
+		if _, ok := ctx.Language.(*golang.Language); ok {
+			originLanguage = &java.Language{}
+		} else {
+			originLanguage = &golang.Language{}
+		}
+
+		originFilePath, stubFilePath, err := t.unpackTranspilerPackage(ctx, taskLogger.Logger, originLanguage, packagePath)
+		if err != nil {
+			return nil, nil, err
+		}
+
+		modelContext := model.Context{
+			Language: ctx.Language,
+
+			RepositoryPath: filepath.Join(ctx.Repository.DataPath(), packagePath),
+			FilePath:       originFilePath,
+
+			Arguments: &TaskArgumentsTranspile{
+				OriginLanguage: originLanguage,
+				StubFilePath:   stubFilePath,
+			},
+
+			Logger: taskLogger.Logger,
+		}
+		assessments, err := modelCapability.Transpile(modelContext)
+		if err != nil {
+			problems = append(problems, pkgerrors.WithMessage(err, originFilePath))
+
+			continue
+		}
+		if assessments[metrics.AssessmentKeyProcessingTime] == 0 {
+			return nil, nil, pkgerrors.Errorf("no model response time measurement present for %q at repository %q", ctx.Model.ID(), ctx.Repository.Name())
+		}
+		modelAssessments.Add(assessments)
+		modelAssessments.Award(metrics.AssessmentKeyResponseNoError)
+
+		coverage, ps, err := ctx.Language.Execute(taskLogger.Logger, filepath.Join(ctx.Repository.DataPath(), packagePath))
+		problems = append(problems, ps...)
+		if err != nil {
+			problems = append(problems, pkgerrors.WithMessage(err, originFilePath))
+
+			continue
+		}
+		taskLogger.Printf("Executes tests with %d coverage objects", coverage)
+		modelAssessments.Award(metrics.AssessmentKeyFilesExecuted)
+		modelAssessments.AwardPoints(metrics.AssessmentKeyCoverage, coverage)
+	}
+
+	repositoryAssessment = map[evaltask.Identifier]metrics.Assessments{
+		IdentifierTranspile: modelAssessments,
+	}
+
+	return repositoryAssessment, problems, nil
+}
+
+// unpackTranspilerPackage checks if the testdata repository for the transpilation task is well-formed and returns the path to the implementation file and also the path to the file that holds the stub.
+func (t *TaskTranspile) unpackTranspilerPackage(ctx evaltask.Context, fileLogger *log.Logger, originLanguage language.Language, packagePath string) (originFilePath string, stubFilePath string, err error) {
+	packagePathAbsolute := filepath.Join(ctx.Repository.DataPath(), packagePath)
+	// Check if the package path has a directory called "implementation" with a source file in the language to transpile from.
+	files, err := originLanguage.Files(fileLogger, filepath.Join(packagePathAbsolute, "implementation"))
+	if err != nil {
+		return "", "", pkgerrors.WithStack(err)
+	} else if len(files) != 1 {
+		return "", "", pkgerrors.Errorf("package %q in repository %q must have an \"implementation\" directory with just one %s source file to transpile", packagePath, ctx.Repository.Name(), originLanguage.Name())
+	} else if strings.HasSuffix(files[0], originLanguage.DefaultTestFileSuffix()) {
+		return "", "", pkgerrors.Errorf("package %q in repository %q must have an \"implementation\" directory with only a %s source file, but found a test file %q", packagePath, ctx.Repository.Name(), originLanguage.Name(), originFilePath)
+	}
+	originFilePath = filepath.Join("implementation", files[0])
+
+	stubFilePath, err = packageHasSourceAndTestFile(fileLogger, ctx.Repository.Name(), packagePathAbsolute, ctx.Language)
+	if err != nil {
+		return "", "", err
+	}
+
+	return originFilePath, stubFilePath, nil
+}