Evaluate by default every available model and language through the "p…

…lain" repositories
symflower · Mar 29, 2024 · 7fd2017 · 7fd2017
1 parent 6491632
commit 7fd2017
Show file tree

Hide file tree

Showing 4 changed files with 29 additions and 6 deletions.
diff --git a/cmd/eval-symflower-codegen-testing/cmd/evaluate.go b/cmd/eval-symflower-codegen-testing/cmd/evaluate.go
@@ -2,8 +2,11 @@ package cmd
 
 import (
 	"log"
+	"path/filepath"
+	"sort"
 
 	"github.com/spf13/cobra"
+	"golang.org/x/exp/maps"
 
 	"github.com/symflower/eval-symflower-codegen-testing/evaluate"
 	"github.com/symflower/eval-symflower-codegen-testing/language"
@@ -12,13 +15,27 @@ import (
 
 var commandEvalute = &cobra.Command{
 	Use:   "evaluate",
-	Short: "Run an evaluation, by default with all defined models and benchmarks.",
+	Short: "Run an evaluation, by default with all defined models, repositories and tasks.",
 	Run: func(command *cobra.Command, arguments []string) {
-		model := &model.ModelSymflower{}
-		language := &language.LanguageGolang{}
+		// Gather languages.
+		languageIDs := maps.Keys(language.Languages)
+		sort.Strings(languageIDs)
 
-		if err := evaluate.EvaluateRepository("testdata/golang/plain", model, language); err != nil {
-			log.Fatalf("%+v", err)
+		// Gather models.
+		modelIDs := maps.Keys(model.Models)
+		sort.Strings(modelIDs)
+
+		// Check that models and languages can be evaluated by executing the "plain" repositories.
+		log.Printf("Checking that models and languages can used for evaluation")
+		for _, languageID := range languageIDs {
+			for _, modelID := range modelIDs {
+				model := model.Models[modelID]
+				language := language.Languages[languageID]
+
+				if err := evaluate.EvaluateRepository(model, language, filepath.Join("testdata", language.ID(), "plain")); err != nil {
+					log.Fatalf("%+v", err)
+				}
+			}
 		}
 	},
 }

diff --git a/evaluate/repository.go b/evaluate/repository.go
@@ -2,6 +2,7 @@ package evaluate
 
 import (
 	"errors"
+	"log"
 	"os"
 	"path/filepath"
 
@@ -13,7 +14,9 @@ import (
 )
 
 // EvaluateRepository evaluate a repository with the given model and language.
-func EvaluateRepository(repositoryPath string, model model.Model, language language.Language) (err error) {
+func EvaluateRepository(model model.Model, language language.Language, repositoryPath string) (err error) {
+	log.Printf("Evaluating model %q using language %q and repository %q", model.ID(), language.ID(), repositoryPath)
+
 	temporaryPath, err := os.MkdirTemp("", "eval-symflower-codegen-testing")
 	if err != nil {
 		return pkgerrors.WithStack(err)

diff --git a/go.mod b/go.mod
@@ -7,6 +7,7 @@ require (
 	github.com/spf13/cobra v1.8.0
 	github.com/stretchr/testify v1.9.0
 	github.com/zimmski/osutil v1.1.0
+	golang.org/x/exp v0.0.0-20240325151524-a685a6edb6d8
 )
 
 require (

diff --git a/go.sum b/go.sum
@@ -35,6 +35,8 @@ github.com/yuin/goldmark v1.7.0 h1:EfOIvIMZIzHdB/R/zVrikYLPPwJlfMcNczJFMs1m6sA=
 github.com/yuin/goldmark v1.7.0/go.mod h1:uzxRWxtg69N339t3louHJ7+O03ezfj6PlliRlaOzY1E=
 github.com/zimmski/osutil v1.1.0 h1:laxALUxv1fHD+g/Db5swvl4FNYkh1SSGxtdqBiZziSA=
 github.com/zimmski/osutil v1.1.0/go.mod h1:TZrA1ZvRIeylQ0ECaANmCVlT0WR/62zJxMdQX9SyLvY=
+golang.org/x/exp v0.0.0-20240325151524-a685a6edb6d8 h1:aAcj0Da7eBAtrTp03QXWvm88pSyOt+UgdZw2BFZ+lEw=
+golang.org/x/exp v0.0.0-20240325151524-a685a6edb6d8/go.mod h1:CQ1k9gNrJ50XIzaKCRR2hssIjF07kZFEiieALBM/ARQ=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=