From f4e171b2a10ecb6c7e0690232f711bec711569ac Mon Sep 17 00:00:00 2001
From: Martin Treml <git@treml.dev>
Date: Tue, 9 Jul 2024 16:25:41 +0200
Subject: [PATCH] Revert "Docker runtime follow up"

---
 .github/workflows/test.yml                |   7 +-
 Dockerfile                                |  54 ++---
 Makefile                                  |   9 +-
 README.md                                 |  14 +-
 cmd/eval-dev-quality/cmd/command.go       |  10 -
 cmd/eval-dev-quality/cmd/evaluate.go      |  67 +++---
 cmd/eval-dev-quality/cmd/evaluate_test.go | 246 ----------------------
 7 files changed, 48 insertions(+), 359 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 78ca5157..e097d2f7 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -22,7 +22,7 @@ jobs:
           java-version: '11'
 
       - name: Set up Maven
-        uses: stCarolas/setup-maven@v5
+        uses: stCarolas/setup-maven@v4.5
         with:
           maven-version: '3.9.1'
 
@@ -36,11 +36,6 @@ jobs:
         run: make install
         shell: bash # Explicitly use Bash because otherwise failing Windows jobs are not erroring.
 
-      - name: Install required tools
-        run: |
-          make install-tools
-        shell: bash # Explicitly use Bash because otherwise failing Windows jobs are not erroring.
-
       - name: Install testing tools
         run: |
           make install-tools-testing
diff --git a/Dockerfile b/Dockerfile
index b2e4d506..5ba95a69 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,62 +1,40 @@
-# Builder image.
-FROM golang:latest as builder
-
-WORKDIR /app
-COPY ./ ./
-
-# Build the binary.
-RUN go mod tidy
-RUN CGO_ENABLED=0 go build -o eval-dev-quality ./cmd/eval-dev-quality
-
-# Actual running image.
 FROM ubuntu:noble
 RUN apt-get update && apt-get install -y ca-certificates wget unzip git make && update-ca-certificates
 
-# Non-root ollama need a hardcoded directory to store ssh-key.
-RUN mkdir -p /.ollama && chmod 777 /.ollama
-# Same for symflower
-RUN mkdir -p /.config && chmod 777 /.config
-RUN mkdir -p /.eval-dev-quality && chmod 777 /.eval-dev-quality
-RUN mkdir -p /.cache && chmod 777 /.cache
+WORKDIR /home/ubuntu/eval-dev-quality
+COPY ./ ./
+RUN chown -R ubuntu:ubuntu ./
 
-# Switch to the ubuntu user as we want it to run as non-root.
 USER ubuntu
-WORKDIR /app
-COPY --chown=ubuntu:ubuntu ./testdata ./testdata
-COPY --chown=ubuntu:ubuntu ./Makefile ./Makefile
-RUN mkdir -p .eval-dev-quality
+RUN mkdir -p ~/.eval-dev-quality
 
 # Install Maven
 RUN wget https://archive.apache.org/dist/maven/maven-3/3.9.1/binaries/apache-maven-3.9.1-bin.tar.gz && \
-	tar -xf apache-maven-3.9.1-bin.tar.gz -C /app/.eval-dev-quality/ && \
+	tar -xf apache-maven-3.9.1-bin.tar.gz -C ~/.eval-dev-quality/ && \
 	rm apache-maven-3.9.1-bin.tar.gz
-ENV PATH="${PATH}:/app/.eval-dev-quality/apache-maven-3.9.1/bin"
+ENV PATH="${PATH}:/home/ubuntu/.eval-dev-quality/apache-maven-3.9.1/bin"
 
 # Install Gradle
 RUN wget https://services.gradle.org/distributions/gradle-8.0.2-bin.zip && \
-	unzip gradle-8.0.2-bin.zip -d /app/.eval-dev-quality/ && \
+	unzip gradle-8.0.2-bin.zip -d ~/.eval-dev-quality/ && \
 	rm gradle-8.0.2-bin.zip
-ENV PATH="${PATH}:/app/.eval-dev-quality/gradle-8.0.2/bin"
+ENV PATH="${PATH}:/home/ubuntu/.eval-dev-quality/gradle-8.0.2/bin"
 
 # Install Java
 RUN wget https://corretto.aws/downloads/latest/amazon-corretto-11-x64-linux-jdk.tar.gz && \
-	tar -xf amazon-corretto-11-x64-linux-jdk.tar.gz -C /app/.eval-dev-quality/ && \
+	tar -xf amazon-corretto-11-x64-linux-jdk.tar.gz -C ~/.eval-dev-quality/ && \
 	rm amazon-corretto-11-x64-linux-jdk.tar.gz
-ENV JAVA_HOME="/app/.eval-dev-quality/amazon-corretto-11.0.23.9.1-linux-x64"
+ENV JAVA_HOME="/home/ubuntu/.eval-dev-quality/amazon-corretto-11.0.23.9.1-linux-x64"
 ENV PATH="${PATH}:${JAVA_HOME}/bin"
 
 # Install Go
 RUN wget https://go.dev/dl/go1.21.5.linux-amd64.tar.gz && \
-	tar -xf go1.21.5.linux-amd64.tar.gz -C /app/.eval-dev-quality/ && \
+	tar -xf go1.21.5.linux-amd64.tar.gz -C ~/.eval-dev-quality/ && \
 	rm go1.21.5.linux-amd64.tar.gz
-ENV PATH="${PATH}:/app/.eval-dev-quality/go/bin"
-RUN go env -w GOPATH=/app/.eval-dev-quality/go
+ENV PATH="${PATH}:/home/ubuntu/.eval-dev-quality/go/bin"
+ENV PATH="${PATH}:/home/ubuntu/go/bin"
 
-# Install the binary
-COPY --from=builder --chown=ubuntu:ubuntu /app/eval-dev-quality /app/.eval-dev-quality/bin/
-ENV PATH="${PATH}:/app/.eval-dev-quality/bin"
-RUN make install-tools-testing
-RUN make install-tools /app/.eval-dev-quality/bin
+# Setup the evaluation
 
-# CHMOD everything because of non-root user ids
-RUN chmod -R 777 /app
+RUN make install-all
+ENV PATH="${PATH}:/home/ubuntu/.eval-dev-quality/bin"
diff --git a/Makefile b/Makefile
index 12a83e26..509e1881 100644
--- a/Makefile
+++ b/Makefile
@@ -1,7 +1,7 @@
 export ROOT_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
 
 export PACKAGE_BASE := github.com/symflower/eval-dev-quality
-export UNIT_TEST_TIMEOUT := 720
+export UNIT_TEST_TIMEOUT := 480
 
 ARGS := $(wordlist 2,$(words $(MAKECMDGOALS)),$(MAKECMDGOALS))
 $(eval $(ARGS):;@:) # turn arguments into do-nothing targets
@@ -44,16 +44,13 @@ install: # [<Go package] - # Build and install everything, or only the specified
 	go install -v $(PACKAGE)
 .PHONY: install
 
-install-all: install install-tools install-tools-testing # Install everything for and of this repository.
+install-all: install install-tools-testing # Install everything for and of this repository.
 .PHONY: install-all
 
-install-tools: # Install tools that are required for running the evaluation.
-	eval-dev-quality install-tools $(if $(ARGS), --install-tools-path $(word 1,$(ARGS)))
-.PHONY: install-tools
-
 install-tools-testing: # Install tools that are used for testing.
 	go install -v github.com/vektra/mockery/v2@v2.40.3
 	go install -v gotest.tools/gotestsum@v1.11.0
+	eval-dev-quality install-tools
 .PHONY: install-tools-testing
 
 generate: # Run code generation.
diff --git a/README.md b/README.md
index 17257ae6..8a76eb35 100644
--- a/README.md
+++ b/README.md
@@ -135,16 +135,9 @@ Total coverage 100.000000%
 
 The execution by default also creates an report file `REPORT.md` that contains additional evaluation results and links to individual result files.
 
-# Containerized use
+# Docker
 
-## Notes
-
-The following parameters do have a special behavior when using a containerized runtime.
-- `--testdata`: The check if the path exists is ignored on the host system but still enforced inside the container because the paths of the host and inside the container might differ.
-
-## Docker
-
-### Setup
+## Setup
 
 Ensure that docker is installed on the system.
 
@@ -165,7 +158,8 @@ docker run -v ./:/home/ubuntu/evaluation --user $(id -u):$(id -g) eval-dev-quali
 docker run -v ./:/home/ubuntu/evaluation --user $(id -u):$(id -g) ghcr.io/symflower/eval-dev-quality:latest eval-dev-quality evaluate --model symflower/symbolic-execution --result-path /home/ubuntu/evaluation/%datetime%
 ```
 
-## Kubernetes
+
+# Kubernetes
 
 Please check the [Kubernetes](./docs/kubernetes/README.md) documentation.
 
diff --git a/cmd/eval-dev-quality/cmd/command.go b/cmd/eval-dev-quality/cmd/command.go
index 676885b7..74808743 100644
--- a/cmd/eval-dev-quality/cmd/command.go
+++ b/cmd/eval-dev-quality/cmd/command.go
@@ -31,10 +31,6 @@ func Execute(logger *log.Logger, arguments []string) {
 			c.SetLogger(logger)
 		}
 
-		if c, ok := command.(SetArguments); ok {
-			c.SetArguments(arguments)
-		}
-
 		return command.Execute(args)
 	}
 
@@ -55,9 +51,3 @@ type SetLogger interface {
 	// SetLogger sets the logger of the command.
 	SetLogger(logger *log.Logger)
 }
-
-// SetArguments defines a command that allows to set its arguments.
-type SetArguments interface {
-	// SetArguments sets the commands arguments.
-	SetArguments(args []string)
-}
diff --git a/cmd/eval-dev-quality/cmd/evaluate.go b/cmd/eval-dev-quality/cmd/evaluate.go
index 09db1f05..d4fcd554 100644
--- a/cmd/eval-dev-quality/cmd/evaluate.go
+++ b/cmd/eval-dev-quality/cmd/evaluate.go
@@ -6,7 +6,6 @@ import (
 	"fmt"
 	"os"
 	"os/exec"
-	"os/user"
 	"path/filepath"
 	"slices"
 	"sort"
@@ -83,8 +82,6 @@ type Evaluate struct {
 	// Namespace the namespace under which the kubernetes resources should be created.
 	Namespace string `long:"namespace" description:"The Namespace which should be used for kubernetes resources." default:"eval-dev-quality"`
 
-	// args holds a list of all the passed arguments.
-	args []string
 	// logger holds the logger of the command.
 	logger *log.Logger
 	// timestamp holds the timestamp of the command execution.
@@ -98,14 +95,6 @@ func (command *Evaluate) SetLogger(logger *log.Logger) {
 	command.logger = logger
 }
 
-var _ SetArguments = (*Evaluate)(nil)
-
-// SetArguments sets the commands arguments.
-func (command *Evaluate) SetArguments(args []string) {
-	availableFlags := util.Flags(command)
-	command.args = util.FilterArgsKeep(args, availableFlags)
-}
-
 // Initialize initializes the command according to the arguments.
 func (command *Evaluate) Initialize(args []string) (evaluationContext *evaluate.Context, cleanup func()) {
 	// Ensure the cleanup always runs in case there is a panic.
@@ -183,17 +172,15 @@ func (command *Evaluate) Initialize(args []string) (evaluationContext *evaluate.
 
 	// Ensure the "testdata" path exists and make it absolute.
 	{
-		if command.Runtime == "local" { // Ignore testdata path during containerized execution.
-			if err := osutil.DirExists(command.TestdataPath); err != nil {
-				command.logger.Panicf("ERROR: testdata path %q cannot be accessed: %s", command.TestdataPath, err)
-			}
-			testdataPath, err := filepath.Abs(command.TestdataPath)
-			if err != nil {
-				command.logger.Panicf("ERROR: could not resolve testdata path %q to an absolute path: %s", command.TestdataPath, err)
-			}
-			command.TestdataPath = testdataPath
-			evaluationContext.TestdataPath = testdataPath
+		if err := osutil.DirExists(command.TestdataPath); err != nil {
+			command.logger.Panicf("ERROR: testdata path %q cannot be accessed: %s", command.TestdataPath, err)
+		}
+		testdataPath, err := filepath.Abs(command.TestdataPath)
+		if err != nil {
+			command.logger.Panicf("ERROR: could not resolve testdata path %q to an absolute path: %s", command.TestdataPath, err)
 		}
+		command.TestdataPath = testdataPath
+		evaluationContext.TestdataPath = testdataPath
 	}
 
 	// Setup evaluation result directory.
@@ -459,31 +446,18 @@ func (command *Evaluate) evaluateLocal(evaluationContext *evaluate.Context) (err
 
 // evaluateDocker executes the evaluation for each model inside a docker container.
 func (command *Evaluate) evaluateDocker(ctx *evaluate.Context) (err error) {
+	availableFlags := util.Flags(command)
 	ignoredFlags := []string{
 		"model",
 		"parallel",
 		"result-path",
-		"runtime-image",
 		"runtime",
 	}
 
+	// Filter all the args to only contain flags which can be used.
+	args := util.FilterArgsKeep(os.Args[2:], availableFlags)
 	// Filter the args to remove all flags unsuited for running the container.
-	args := util.FilterArgsRemove(command.args, ignoredFlags)
-
-	// Get current user for volume ID mapping.
-	user, err := user.Current()
-	if err != nil {
-		return pkgerrors.WithStack(err)
-	}
-
-	resultPath, err := filepath.Abs(command.ResultPath)
-	if err != nil {
-		return pkgerrors.WithStack(err)
-	}
-	// Set permission 777 so the non-root docker image is able to store its results inside the result path.
-	if err := os.Chmod(resultPath, 0777); err != nil {
-		return pkgerrors.WithStack(err)
-	}
+	args = util.FilterArgsRemove(args, ignoredFlags)
 
 	parallel := util.NewParallel(command.Parallel)
 
@@ -496,15 +470,23 @@ func (command *Evaluate) evaluateDocker(ctx *evaluate.Context) (err error) {
 			continue
 		}
 
+		// Create for each model a dedicated subfolder inside the results path.
+		resultPath, err := filepath.Abs(command.ResultPath)
+		if err != nil {
+			return err
+		}
+		// Set permission 777 so the non-root docker image is able to store its results inside the result path.
+		if err := os.Chmod(resultPath, 0777); err != nil {
+			return err
+		}
+
 		// Commands regarding the docker runtime.
 		dockerCommand := []string{
 			"docker",
 			"run",
 			"-e", "PROVIDER_TOKEN",
 			"-v", // bind volume
-			resultPath + ":/app/evaluation",
-			"--user",
-			user.Uid + ":" + user.Gid,
+			resultPath + ":/home/ubuntu/evaluation",
 			"--rm", // automatically remove container after it finished
 			command.RuntimeImage,
 		}
@@ -516,7 +498,7 @@ func (command *Evaluate) evaluateDocker(ctx *evaluate.Context) (err error) {
 			"--model",
 			model.ID(),
 			"--result-path",
-			"/app/evaluation/" + model.ID(),
+			"/home/ubuntu/evaluation/" + model.ID(),
 		}
 
 		cmd := append(dockerCommand, evaluationCommand...)
@@ -550,7 +532,6 @@ func (command *Evaluate) evaluateKubernetes(ctx *evaluate.Context) (err error) {
 		"model",
 		"parallel",
 		"result-path",
-		"runtime-image",
 		"runtime",
 	}
 
diff --git a/cmd/eval-dev-quality/cmd/evaluate_test.go b/cmd/eval-dev-quality/cmd/evaluate_test.go
index 5e3651fb..83778a89 100644
--- a/cmd/eval-dev-quality/cmd/evaluate_test.go
+++ b/cmd/eval-dev-quality/cmd/evaluate_test.go
@@ -1,11 +1,9 @@
 package cmd
 
 import (
-	"context"
 	"fmt"
 	"net/url"
 	"os"
-	"os/exec"
 	"path/filepath"
 	"regexp"
 	"sort"
@@ -27,7 +25,6 @@ import (
 	providertesting "github.com/symflower/eval-dev-quality/provider/testing"
 	"github.com/symflower/eval-dev-quality/tools"
 	toolstesting "github.com/symflower/eval-dev-quality/tools/testing"
-	"github.com/symflower/eval-dev-quality/util"
 )
 
 // validateReportLinks checks if the Markdown report data contains all the links to other relevant report files.
@@ -798,249 +795,6 @@ func TestEvaluateExecute(t *testing.T) {
 		})
 	})
 
-	t.Run("Runtime", func(t *testing.T) {
-		// Skip containerized runtime tests if special cases.
-		if osutil.IsWindows() {
-			t.Skip("Docker runtime not supported on Windows")
-		} else if _, err := exec.LookPath("docker"); err != nil {
-			t.Skip("Docker runtime not found")
-		}
-
-		// TODO: Make the docker runtime run on both local and CI
-		if os.Getenv("GITHUB_ACTIONS") != "true" {
-			t.Skip("We are currently only supporting the Github CI")
-		}
-
-		// Prerequisite
-		dockerImage := ""
-		{
-			// Get current branch name
-			_, logger := log.Buffer()
-			branch, err := util.CommandWithResult(context.Background(), logger, &util.Command{
-				Command: []string{
-					"git",
-					"rev-parse",
-					"--short",
-					"HEAD",
-				},
-			})
-			assert.NoError(t, err)
-
-			dockerImage = "ghcr.io/symflower/eval-dev-quality:" + strings.TrimSpace(branch)
-
-			// Pull the image
-			_, err = util.CommandWithResult(context.Background(), logger, &util.Command{
-				Command: []string{
-					"docker",
-					"pull",
-					dockerImage,
-				},
-			})
-			assert.NoError(t, err)
-		}
-
-		validate(t, &testCase{
-			Name: "Docker",
-
-			Arguments: []string{
-				"--runtime", "docker",
-				"--model", "symflower/symbolic-execution",
-				"--model", "symflower/symbolic-execution",
-				"--model", "symflower/symbolic-execution",
-				"--testdata", "testdata/", // Our own tests set the "testdata" argument to the temporary directory that they create. This temporary directory does not exist in docker, so set the "testdata" manually here to overrule the testing behavior and use the original one.
-				"--repository", filepath.Join("golang", "plain"),
-				"--repository", filepath.Join("java", "plain"),
-				"--runs=1",
-				"--parallel=3",
-				"--runtime-image=" + dockerImage,
-			},
-
-			ExpectedOutputValidate: func(t *testing.T, output string, resultPath string) {
-				actualAssessments := validateMetrics(t, extractMetricsLogsMatch, output, []metrics.Assessments{
-					metrics.Assessments{
-						metrics.AssessmentKeyCoverage:         40,
-						metrics.AssessmentKeyFilesExecuted:    4,
-						metrics.AssessmentKeyResponseNoError:  4,
-						metrics.AssessmentKeyResponseNoExcess: 4,
-						metrics.AssessmentKeyResponseWithCode: 4,
-					},
-					metrics.Assessments{
-						metrics.AssessmentKeyCoverage:         40,
-						metrics.AssessmentKeyFilesExecuted:    4,
-						metrics.AssessmentKeyResponseNoError:  4,
-						metrics.AssessmentKeyResponseNoExcess: 4,
-						metrics.AssessmentKeyResponseWithCode: 4,
-					},
-					metrics.Assessments{
-						metrics.AssessmentKeyCoverage:         40,
-						metrics.AssessmentKeyFilesExecuted:    4,
-						metrics.AssessmentKeyResponseNoError:  4,
-						metrics.AssessmentKeyResponseNoExcess: 4,
-						metrics.AssessmentKeyResponseWithCode: 4,
-					},
-				}, []uint64{56, 56, 56})
-				// Assert non-deterministic behavior.
-				assert.Greater(t, actualAssessments[0][metrics.AssessmentKeyProcessingTime], uint64(0))
-				assert.Equal(t, uint64(786), actualAssessments[0][metrics.AssessmentKeyGenerateTestsForFileCharacterCount])
-				assert.Equal(t, uint64(786), actualAssessments[0][metrics.AssessmentKeyResponseCharacterCount])
-				assert.Equal(t, 3, strings.Count(output, "Evaluation score for"))
-			},
-			ExpectedResultFiles: map[string]func(t *testing.T, filePath string, data string){
-				filepath.Join("result-directory", "evaluation.log"): nil,
-
-				// Parallel run 1
-				filepath.Join("result-directory", "symflower", "symbolic-execution", "categories.svg"): nil,
-				filepath.Join("result-directory", "symflower", "symbolic-execution", "evaluation.csv"): func(t *testing.T, filePath, data string) {
-					actualAssessments := validateMetrics(t, extractMetricsCSVMatch, data, []metrics.Assessments{
-						metrics.Assessments{
-							metrics.AssessmentKeyCoverage:         10,
-							metrics.AssessmentKeyFilesExecuted:    1,
-							metrics.AssessmentKeyResponseNoError:  1,
-							metrics.AssessmentKeyResponseNoExcess: 1,
-							metrics.AssessmentKeyResponseWithCode: 1,
-						},
-						metrics.Assessments{
-							metrics.AssessmentKeyCoverage:         10,
-							metrics.AssessmentKeyFilesExecuted:    1,
-							metrics.AssessmentKeyResponseNoError:  1,
-							metrics.AssessmentKeyResponseNoExcess: 1,
-							metrics.AssessmentKeyResponseWithCode: 1,
-						},
-						metrics.Assessments{
-							metrics.AssessmentKeyCoverage:         10,
-							metrics.AssessmentKeyFilesExecuted:    1,
-							metrics.AssessmentKeyResponseNoError:  1,
-							metrics.AssessmentKeyResponseNoExcess: 1,
-							metrics.AssessmentKeyResponseWithCode: 1,
-						},
-						metrics.Assessments{
-							metrics.AssessmentKeyCoverage:         10,
-							metrics.AssessmentKeyFilesExecuted:    1,
-							metrics.AssessmentKeyResponseNoError:  1,
-							metrics.AssessmentKeyResponseNoExcess: 1,
-							metrics.AssessmentKeyResponseWithCode: 1,
-						},
-					}, []uint64{14, 14, 14, 14})
-					// Assert non-deterministic behavior.
-					assert.Greater(t, actualAssessments[0][metrics.AssessmentKeyProcessingTime], uint64(0))
-					assert.Equal(t, uint64(254), actualAssessments[0][metrics.AssessmentKeyGenerateTestsForFileCharacterCount])
-					assert.Equal(t, uint64(254), actualAssessments[0][metrics.AssessmentKeyResponseCharacterCount])
-				},
-				filepath.Join("result-directory", "symflower", "symbolic-execution", "evaluation.log"):    nil,
-				filepath.Join("result-directory", "symflower", "symbolic-execution", "golang-summed.csv"): nil,
-				filepath.Join("result-directory", "symflower", "symbolic-execution", "java-summed.csv"):   nil,
-				filepath.Join("result-directory", "symflower", "symbolic-execution", "models-summed.csv"): nil,
-				filepath.Join("result-directory", "symflower", "symbolic-execution", "README.md"):         nil,
-				filepath.Join("result-directory", "symflower", "symbolic-execution", string(evaluatetask.IdentifierWriteTests), "symflower_symbolic-execution", "golang", "golang", "plain.log"): func(t *testing.T, filePath, data string) {
-					assert.Equal(t, 1, strings.Count(data, `Evaluating model "symflower/symbolic-execution"`))
-				},
-				filepath.Join("result-directory", "symflower", "symbolic-execution", string(evaluatetask.IdentifierWriteTests), "symflower_symbolic-execution", "java", "java", "plain.log"): func(t *testing.T, filePath, data string) {
-					assert.Equal(t, 1, strings.Count(data, `Evaluating model "symflower/symbolic-execution"`))
-				},
-
-				// Parallel run 2
-				filepath.Join("result-directory", "symflower", "symbolic-execution-0", "categories.svg"): nil,
-				filepath.Join("result-directory", "symflower", "symbolic-execution-0", "evaluation.csv"): func(t *testing.T, filePath, data string) {
-					actualAssessments := validateMetrics(t, extractMetricsCSVMatch, data, []metrics.Assessments{
-						metrics.Assessments{
-							metrics.AssessmentKeyCoverage:         10,
-							metrics.AssessmentKeyFilesExecuted:    1,
-							metrics.AssessmentKeyResponseNoError:  1,
-							metrics.AssessmentKeyResponseNoExcess: 1,
-							metrics.AssessmentKeyResponseWithCode: 1,
-						},
-						metrics.Assessments{
-							metrics.AssessmentKeyCoverage:         10,
-							metrics.AssessmentKeyFilesExecuted:    1,
-							metrics.AssessmentKeyResponseNoError:  1,
-							metrics.AssessmentKeyResponseNoExcess: 1,
-							metrics.AssessmentKeyResponseWithCode: 1,
-						},
-						metrics.Assessments{
-							metrics.AssessmentKeyCoverage:         10,
-							metrics.AssessmentKeyFilesExecuted:    1,
-							metrics.AssessmentKeyResponseNoError:  1,
-							metrics.AssessmentKeyResponseNoExcess: 1,
-							metrics.AssessmentKeyResponseWithCode: 1,
-						},
-						metrics.Assessments{
-							metrics.AssessmentKeyCoverage:         10,
-							metrics.AssessmentKeyFilesExecuted:    1,
-							metrics.AssessmentKeyResponseNoError:  1,
-							metrics.AssessmentKeyResponseNoExcess: 1,
-							metrics.AssessmentKeyResponseWithCode: 1,
-						},
-					}, []uint64{14, 14, 14, 14})
-					// Assert non-deterministic behavior.
-					assert.Greater(t, actualAssessments[0][metrics.AssessmentKeyProcessingTime], uint64(0))
-					assert.Equal(t, uint64(254), actualAssessments[0][metrics.AssessmentKeyGenerateTestsForFileCharacterCount])
-					assert.Equal(t, uint64(254), actualAssessments[0][metrics.AssessmentKeyResponseCharacterCount])
-				},
-				filepath.Join("result-directory", "symflower", "symbolic-execution-0", "evaluation.log"):    nil,
-				filepath.Join("result-directory", "symflower", "symbolic-execution-0", "golang-summed.csv"): nil,
-				filepath.Join("result-directory", "symflower", "symbolic-execution-0", "java-summed.csv"):   nil,
-				filepath.Join("result-directory", "symflower", "symbolic-execution-0", "models-summed.csv"): nil,
-				filepath.Join("result-directory", "symflower", "symbolic-execution-0", "README.md"):         nil,
-				filepath.Join("result-directory", "symflower", "symbolic-execution-0", string(evaluatetask.IdentifierWriteTests), "symflower_symbolic-execution", "golang", "golang", "plain.log"): func(t *testing.T, filePath, data string) {
-					assert.Equal(t, 1, strings.Count(data, `Evaluating model "symflower/symbolic-execution"`))
-				},
-				filepath.Join("result-directory", "symflower", "symbolic-execution-0", string(evaluatetask.IdentifierWriteTests), "symflower_symbolic-execution", "java", "java", "plain.log"): func(t *testing.T, filePath, data string) {
-					assert.Equal(t, 1, strings.Count(data, `Evaluating model "symflower/symbolic-execution"`))
-				},
-
-				// Parallel run 3
-				filepath.Join("result-directory", "symflower", "symbolic-execution-1", "categories.svg"): nil,
-				filepath.Join("result-directory", "symflower", "symbolic-execution-1", "evaluation.csv"): func(t *testing.T, filePath, data string) {
-					actualAssessments := validateMetrics(t, extractMetricsCSVMatch, data, []metrics.Assessments{
-						metrics.Assessments{
-							metrics.AssessmentKeyCoverage:         10,
-							metrics.AssessmentKeyFilesExecuted:    1,
-							metrics.AssessmentKeyResponseNoError:  1,
-							metrics.AssessmentKeyResponseNoExcess: 1,
-							metrics.AssessmentKeyResponseWithCode: 1,
-						},
-						metrics.Assessments{
-							metrics.AssessmentKeyCoverage:         10,
-							metrics.AssessmentKeyFilesExecuted:    1,
-							metrics.AssessmentKeyResponseNoError:  1,
-							metrics.AssessmentKeyResponseNoExcess: 1,
-							metrics.AssessmentKeyResponseWithCode: 1,
-						},
-						metrics.Assessments{
-							metrics.AssessmentKeyCoverage:         10,
-							metrics.AssessmentKeyFilesExecuted:    1,
-							metrics.AssessmentKeyResponseNoError:  1,
-							metrics.AssessmentKeyResponseNoExcess: 1,
-							metrics.AssessmentKeyResponseWithCode: 1,
-						},
-						metrics.Assessments{
-							metrics.AssessmentKeyCoverage:         10,
-							metrics.AssessmentKeyFilesExecuted:    1,
-							metrics.AssessmentKeyResponseNoError:  1,
-							metrics.AssessmentKeyResponseNoExcess: 1,
-							metrics.AssessmentKeyResponseWithCode: 1,
-						},
-					}, []uint64{14, 14, 14, 14})
-					// Assert non-deterministic behavior.
-					assert.Greater(t, actualAssessments[0][metrics.AssessmentKeyProcessingTime], uint64(0))
-					assert.Equal(t, uint64(254), actualAssessments[0][metrics.AssessmentKeyGenerateTestsForFileCharacterCount])
-					assert.Equal(t, uint64(254), actualAssessments[0][metrics.AssessmentKeyResponseCharacterCount])
-				},
-				filepath.Join("result-directory", "symflower", "symbolic-execution-1", "evaluation.log"):    nil,
-				filepath.Join("result-directory", "symflower", "symbolic-execution-1", "golang-summed.csv"): nil,
-				filepath.Join("result-directory", "symflower", "symbolic-execution-1", "java-summed.csv"):   nil,
-				filepath.Join("result-directory", "symflower", "symbolic-execution-1", "models-summed.csv"): nil,
-				filepath.Join("result-directory", "symflower", "symbolic-execution-1", "README.md"):         nil,
-				filepath.Join("result-directory", "symflower", "symbolic-execution-1", string(evaluatetask.IdentifierWriteTests), "symflower_symbolic-execution", "golang", "golang", "plain.log"): func(t *testing.T, filePath, data string) {
-					assert.Equal(t, 1, strings.Count(data, `Evaluating model "symflower/symbolic-execution"`))
-				},
-				filepath.Join("result-directory", "symflower", "symbolic-execution-1", string(evaluatetask.IdentifierWriteTests), "symflower_symbolic-execution", "java", "java", "plain.log"): func(t *testing.T, filePath, data string) {
-					assert.Equal(t, 1, strings.Count(data, `Evaluating model "symflower/symbolic-execution"`))
-				},
-			},
-		})
-	})
-
 	// This case checks a beautiful bug where the Markdown export crashed when the current working directory contained a README.md file. While this is not the case during the tests (as the current work directory is the directory of this file), it certainly caused problems when our binary was executed from the repository root (which of course contained a README.md). Therefore, we sadly have to modify the current work directory right within the tests of this case to reproduce the problem and fix it forever.
 	validate(t, &testCase{
 		Name: "Current work directory contains a README.md",