Task for code transpilation, so models can transpile Go code to Java …

…and back

Closes #201

.mockery.yml

@@ -15,6 +15,7 @@ packages:
       Model:
       CapabilityWriteTests:
       CapabilityRepairCode:
+      CapabilityTranspile:
   github.com/symflower/eval-dev-quality/provider:
     interfaces:
       Loader:

evaluate/task/task-transpile.go

@@ -0,0 +1,178 @@
+package task
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	pkgerrors "github.com/pkg/errors"
+	"github.com/symflower/eval-dev-quality/evaluate/metrics"
+	"github.com/symflower/eval-dev-quality/language"
+	"github.com/symflower/eval-dev-quality/language/golang"
+	"github.com/symflower/eval-dev-quality/language/java"
+	"github.com/symflower/eval-dev-quality/log"
+	"github.com/symflower/eval-dev-quality/model"
+	evaltask "github.com/symflower/eval-dev-quality/task"
+)
+
+// TaskTranspile holds the transpilation task.
+type TaskTranspile struct{}
+
+// TaskArgumentsTranspile holds extra arguments to be used in a query prompt.
+type TaskArgumentsTranspile struct {
+	// TargetLanguage holds the target language for transpilation.
+	TargetLanguage language.Language
+	// StubFilePath holds the path for the file containing just the function signature of the language we are transpiling to.
+	StubFilePath string
+}
+
+var _ evaltask.Task = (*TaskTranspile)(nil)
+
+// Identifier returns the transpilation task identifier.
+func (t *TaskTranspile) Identifier() evaltask.Identifier {
+	return IdentifierTranspile
+}
+
+// Run transpiles code between Go and Java and runs predefined tests to check if the transpilation was successful.
+func (t *TaskTranspile) Run(ctx evaltask.Context) (repositoryAssessment map[evaltask.Identifier]metrics.Assessments, problems []error, err error) {
+	modelCapability, ok := ctx.Model.(model.CapabilityTranspile)
+	if !ok {
+		return nil, nil, pkgerrors.Wrap(evaltask.ErrTaskUnsupportedByModel, fmt.Sprintf("%q does not support %q", ctx.Model.ID(), string(t.Identifier())))
+	}
+
+	taskLogger, err := newTaskLogger(ctx, t)
+	if err != nil {
+		return nil, nil, err
+	}
+	defer func() {
+		taskLogger.finalize(problems)
+	}()
+
+	var packagePaths []string
+	files, err := os.ReadDir(ctx.Repository.DataPath())
+	if err != nil {
+		return nil, nil, pkgerrors.WithStack(err)
+	}
+	for _, file := range files {
+		if file.IsDir() && !strings.HasPrefix(file.Name(), ".") { // Ignore hidden directories.
+			packagePaths = append(packagePaths, filepath.Join(ctx.Repository.DataPath(), file.Name()))
+		}
+	}
+
+	modelAssessments := metrics.NewAssessments()
+	for _, packagePath := range packagePaths {
+		if err := ctx.Repository.Reset(ctx.Logger); err != nil {
+			ctx.Logger.Panicf("ERROR: unable to reset temporary repository path: %s", err)
+		}
+
+		var targetLanguage language.Language
+		if _, ok := ctx.Language.(*golang.Language); ok {
+			targetLanguage = &java.Language{}
+		} else {
+			targetLanguage = &golang.Language{}
+		}
+
+		sourceFilePath, stubFilePath, err := t.unpackTranspilerPackage(ctx, taskLogger.Logger, targetLanguage, packagePath)
+		if err != nil {
+			return nil, nil, err
+		}
+
+		modelContext := model.Context{
+			Language: ctx.Language,
+
+			RepositoryPath: packagePath,
+			FilePath:       sourceFilePath,
+
+			Arguments: &TaskArgumentsTranspile{
+				TargetLanguage: targetLanguage,
+				StubFilePath:   stubFilePath,
+			},
+
+			Logger: taskLogger.Logger,
+		}
+		assessments, err := modelCapability.Transpile(modelContext)
+		if err != nil {
+			problems = append(problems, pkgerrors.WithMessage(err, sourceFilePath))
+
+			continue
+		}
+		if assessments[metrics.AssessmentKeyProcessingTime] == 0 {
+			return nil, nil, pkgerrors.Errorf("no model response time measurement present for %q at repository %q", ctx.Model.ID(), ctx.Repository.Name())
+		}
+		modelAssessments.Add(assessments)
+		modelAssessments.Award(metrics.AssessmentKeyResponseNoError)
+
+		coverage, ps, err := targetLanguage.Execute(taskLogger.Logger, packagePath)
+		problems = append(problems, ps...)
+		if err != nil {
+			problems = append(problems, pkgerrors.WithMessage(err, sourceFilePath))
+
+			continue
+		}
+		taskLogger.Printf("Executes tests with %d coverage objects", coverage)
+		modelAssessments.Award(metrics.AssessmentKeyFilesExecuted)
+		modelAssessments.AwardPoints(metrics.AssessmentKeyCoverage, coverage)
+	}
+
+	repositoryAssessment = map[evaltask.Identifier]metrics.Assessments{
+		IdentifierTranspile: modelAssessments,
+	}
+
+	return repositoryAssessment, problems, nil
+}
+
+// unpackTranspilerPackage checks if the testdata repository for the transpilation task is well-formed and returns the path to the implementation file and also the path to the file that holds the stub.
+func (t *TaskTranspile) unpackTranspilerPackage(ctx evaltask.Context, fileLogger *log.Logger, targetLanguage language.Language, packagePath string) (sourceFilePath string, stubFilePath string, err error) {
+	// Check if the package path has a directory called "implementation".
+	var implementationPath string
+	packageFiles, err := os.ReadDir(packagePath)
+	if err != nil {
+		return "", "", pkgerrors.WithStack(err)
+	}
+	for _, file := range packageFiles {
+		if file.IsDir() && file.Name() == "implementation" {
+			implementationPath = filepath.Join(packagePath, file.Name())
+		}
+	}
+	if implementationPath == "" {
+		return "", "", pkgerrors.Errorf("package %q in repository %q must have an \"implementation\" directory with a %s source file to transpile", packagePath, ctx.Repository.Name(), ctx.Language.Name())
+	}
+
+	// Check if the "implementation" directory contains only one source file in the language being transpiled from.
+	implementationFiles, err := os.ReadDir(implementationPath)
+	if err != nil {
+		return "", "", pkgerrors.WithStack(err)
+	} else if len(implementationFiles) != 1 {
+		return "", "", pkgerrors.Errorf("package %q in repository %q must have an \"implementation\" directory with just one %s source file to transpile", packagePath, ctx.Repository.Name(), ctx.Language.Name())
+	}
+	sourceFilePath = filepath.Join(implementationPath, implementationFiles[0].Name())
+	if filepath.Ext(sourceFilePath) != ctx.Language.DefaultFileExtension() {
+		return "", "", pkgerrors.Errorf("package %q in repository %q must have an \"implementation\" directory with a %s source file", packagePath, ctx.Repository.Name(), ctx.Language.Name())
+	} else if strings.HasPrefix(sourceFilePath, ctx.Language.DefaultTestFileSuffix()) {
+		return "", "", pkgerrors.Errorf("package %q in repository %q must have an \"implementation\" directory with only a %s source file, but found a test file %q", packagePath, ctx.Repository.Name(), ctx.Language.Name(), sourceFilePath)
+	}
+
+	// Check if the package path contains a source file and the corresponding test file in the target language for transpilation.
+	var hasTestFile bool
+	files, err := targetLanguage.Files(fileLogger, packagePath)
+	if err != nil {
+		return "", "", pkgerrors.WithStack(err)
+	} else if len(files) != 2 {
+		return "", "", pkgerrors.Errorf("package %q in repository %q must have a %s source file and the corresponding test file", packagePath, ctx.Repository.Name(), targetLanguage.Name())
+	}
+	for _, file := range files {
+		if strings.HasSuffix(file, targetLanguage.DefaultTestFileSuffix()) {
+			hasTestFile = true
+		} else if filepath.Ext(file) == targetLanguage.DefaultFileExtension() {
+			stubFilePath = filepath.Join(packagePath, file)
+		}
+	}
+	if !hasTestFile {
+		return "", "", pkgerrors.Errorf("package %q in repository %q must have a %s test file", packagePath, ctx.Repository.Name(), targetLanguage.Name())
+	} else if stubFilePath == "" {
+		return "", "", pkgerrors.Errorf("package %q in repository %q must have a %s file with a stub function", packagePath, ctx.Repository.Name(), targetLanguage.Name())
+	}
+
+	return sourceFilePath, stubFilePath, nil
+}