Integrated code lifecycle: Configure checkout path and timeout for pr…

…ogramming exercises (#9217)

src/main/java/de/tum/cit/aet/artemis/buildagent/dto/BuildConfig.java

@@ -15,7 +15,7 @@
 @JsonInclude(JsonInclude.Include.NON_EMPTY)
 public record BuildConfig(String buildScript, String dockerImage, String commitHashToBuild, String assignmentCommitHash, String testCommitHash, String branch,
         ProgrammingLanguage programmingLanguage, ProjectType projectType, boolean scaEnabled, boolean sequentialTestRunsEnabled, boolean testwiseCoverageEnabled,
-        List<String> resultPaths) implements Serializable {
+        List<String> resultPaths, int timeoutSeconds, String assignmentCheckoutPath, String testCheckoutPath, String solutionCheckoutPath) implements Serializable {
 
     @Override
     public String dockerImage() {

src/main/java/de/tum/cit/aet/artemis/buildagent/service/BuildJobContainerService.java

@@ -24,6 +24,7 @@
 import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
 import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
 import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
+import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Value;
@@ -275,11 +276,22 @@ public String getIDOfRunningContainer(String containerName) {
      * @param auxiliaryRepositoriesPaths             An array of paths for auxiliary repositories to be included in the build process.
      * @param auxiliaryRepositoryCheckoutDirectories An array of directory names within the container where each auxiliary repository should be checked out.
      * @param programmingLanguage                    The programming language of the repositories, which influences directory naming conventions.
+     * @param assignmentCheckoutPath                 The directory within the container where the assignment repository should be checked out; can be null if not applicable,
+     *                                                   default would be used.
+     * @param testCheckoutPath                       The directory within the container where the test repository should be checked out; can be null if not applicable, default
+     *                                                   would be used.
+     * @param solutionCheckoutPath                   The directory within the container where the solution repository should be checked out; can be null if not applicable, default
+     *                                                   would be used.
      */
     public void populateBuildJobContainer(String buildJobContainerId, Path assignmentRepositoryPath, Path testRepositoryPath, Path solutionRepositoryPath,
-            Path[] auxiliaryRepositoriesPaths, String[] auxiliaryRepositoryCheckoutDirectories, ProgrammingLanguage programmingLanguage) {
-        String testCheckoutPath = RepositoryCheckoutPath.TEST.forProgrammingLanguage(programmingLanguage);
-        String assignmentCheckoutPath = RepositoryCheckoutPath.ASSIGNMENT.forProgrammingLanguage(programmingLanguage);
+            Path[] auxiliaryRepositoriesPaths, String[] auxiliaryRepositoryCheckoutDirectories, ProgrammingLanguage programmingLanguage, String assignmentCheckoutPath,
+            String testCheckoutPath, String solutionCheckoutPath) {
+
+        assignmentCheckoutPath = (!StringUtils.isBlank(assignmentCheckoutPath)) ? assignmentCheckoutPath
+                : RepositoryCheckoutPath.ASSIGNMENT.forProgrammingLanguage(programmingLanguage);
+
+        String defaultTestCheckoutPath = RepositoryCheckoutPath.TEST.forProgrammingLanguage(programmingLanguage);
+        testCheckoutPath = (!StringUtils.isBlank(defaultTestCheckoutPath) && !StringUtils.isBlank(testCheckoutPath)) ? testCheckoutPath : defaultTestCheckoutPath;
 
         // Make sure to create the working directory in case it does not exist.
         // In case the test checkout path is the working directory, we only create up to the parent, as the working directory is created below.
@@ -292,7 +304,8 @@ public void populateBuildJobContainer(String buildJobContainerId, Path assignmen
         // Copy the assignment repository to the container and move it to the assignment checkout path
         addAndPrepareDirectory(buildJobContainerId, assignmentRepositoryPath, LOCALCI_WORKING_DIRECTORY + "/testing-dir/" + assignmentCheckoutPath);
         if (solutionRepositoryPath != null) {
-            String solutionCheckoutPath = RepositoryCheckoutPath.SOLUTION.forProgrammingLanguage(programmingLanguage);
+            solutionCheckoutPath = (!StringUtils.isBlank(solutionCheckoutPath)) ? solutionCheckoutPath
+                    : RepositoryCheckoutPath.SOLUTION.forProgrammingLanguage(programmingLanguage);
             addAndPrepareDirectory(buildJobContainerId, solutionRepositoryPath, LOCALCI_WORKING_DIRECTORY + "/testing-dir/" + solutionCheckoutPath);
         }
         for (int i = 0; i < auxiliaryRepositoriesPaths.length; i++) {
@@ -309,6 +322,7 @@ private void createScriptFile(String buildJobContainerId) {
 
     private void addAndPrepareDirectory(String containerId, Path repositoryPath, String newDirectoryName) {
         copyToContainer(repositoryPath.toString(), containerId);
+        addDirectory(containerId, getParentFolderPath(newDirectoryName), true);
         renameDirectoryOrFile(containerId, LOCALCI_WORKING_DIRECTORY + "/" + repositoryPath.getFileName().toString(), newDirectoryName);
     }
 
@@ -428,4 +442,9 @@ private Container getContainerForName(String containerName) {
         List<Container> containers = dockerClient.listContainersCmd().withShowAll(true).exec();
         return containers.stream().filter(container -> container.getNames()[0].equals("/" + containerName)).findFirst().orElse(null);
     }
+
+    private String getParentFolderPath(String path) {
+        Path parentPath = Paths.get(path).normalize().getParent();
+        return parentPath != null ? parentPath.toString() : "";
+    }
 }

src/main/java/de/tum/cit/aet/artemis/buildagent/service/BuildJobExecutionService.java

@@ -245,7 +245,8 @@ private BuildResult runScriptAndParseResults(BuildJobQueueItem buildJob, String
         buildLogsMap.appendBuildLogEntry(buildJob.id(), msg);
         log.debug(msg);
         buildJobContainerService.populateBuildJobContainer(containerId, assignmentRepositoryPath, testsRepositoryPath, solutionRepositoryPath, auxiliaryRepositoriesPaths,
-                buildJob.repositoryInfo().auxiliaryRepositoryCheckoutDirectories(), buildJob.buildConfig().programmingLanguage());
+                buildJob.repositoryInfo().auxiliaryRepositoryCheckoutDirectories(), buildJob.buildConfig().programmingLanguage(), buildJob.buildConfig().assignmentCheckoutPath(),
+                buildJob.buildConfig().testCheckoutPath(), buildJob.buildConfig().solutionCheckoutPath());
 
         msg = "~~~~~~~~~~~~~~~~~~~~ Executing Build Script for Build job " + buildJob.id() + " ~~~~~~~~~~~~~~~~~~~~";
         buildLogsMap.appendBuildLogEntry(buildJob.id(), msg);

src/main/java/de/tum/cit/aet/artemis/buildagent/service/BuildJobManagementService.java

@@ -58,7 +58,7 @@ public class BuildJobManagementService {
 
     private final ReentrantLock lock = new ReentrantLock();
 
-    @Value("${artemis.continuous-integration.timeout-seconds:240}")
+    @Value("${artemis.continuous-integration.timeout-seconds:120}")
     private int timeoutSeconds;
 
     @Value("${artemis.continuous-integration.asynchronous:true}")
@@ -149,9 +149,17 @@ public CompletableFuture<BuildResult> executeBuildJob(BuildJobQueueItem buildJob
             lock.unlock();
         }
 
+        int buildJobTimeoutSeconds;
+        if (buildJobItem.buildConfig().timeoutSeconds() != 0 && buildJobItem.buildConfig().timeoutSeconds() < this.timeoutSeconds) {
+            buildJobTimeoutSeconds = buildJobItem.buildConfig().timeoutSeconds();
+        }
+        else {
+            buildJobTimeoutSeconds = this.timeoutSeconds;
+        }
+
         CompletableFuture<BuildResult> futureResult = createCompletableFuture(() -> {
             try {
-                return future.get(timeoutSeconds, TimeUnit.SECONDS);
+                return future.get(buildJobTimeoutSeconds, TimeUnit.SECONDS);
             }
             catch (Exception e) {
                 // RejectedExecutionException is thrown if the queue size limit (defined in "artemis.continuous-integration.queue-size-limit") is reached.

src/main/java/de/tum/cit/aet/artemis/core/config/Constants.java

@@ -93,6 +93,8 @@ public final class Constants {
     // Used to cut off CI specific path segments when receiving static code analysis reports
     public static final String ASSIGNMENT_DIRECTORY = "/" + ASSIGNMENT_REPO_NAME + "/";
 
+    public static final String TEST_WORKING_DIRECTORY = "test";
+
     // Used as a value for <sourceDirectory> for the Java template pom.xml
     public static final String STUDENT_WORKING_DIRECTORY = ASSIGNMENT_DIRECTORY + "src";
 
@@ -390,6 +392,18 @@ public final class Constants {
      */
     public static final int MIN_SCORE_ORANGE = 40;
 
+    public static final String ASSIGNMENT_REPO_PLACEHOLDER = "${studentWorkingDirectory}";
+
+    public static final String TEST_REPO_PLACEHOLDER = "${testWorkingDirectory}";
+
+    public static final String SOLUTION_REPO_PLACEHOLDER = "${solutionWorkingDirectory}";
+
+    public static final String ASSIGNMENT_REPO_PARENT_PLACEHOLDER = "${studentParentWorkingDirectoryName}";
+
+    public static final String ASSIGNMENT_REPO_PLACEHOLDER_NO_SLASH = "${studentWorkingDirectoryNoSlash}";
+
+    public static final Pattern ALLOWED_CHECKOUT_DIRECTORY = Pattern.compile("[\\w-]+(/[\\w-]+)*$");
+
     private Constants() {
     }
 }

src/main/java/de/tum/cit/aet/artemis/programming/domain/ProgrammingExerciseBuildConfig.java

@@ -49,7 +49,13 @@ public class ProgrammingExerciseBuildConfig extends DomainObject {
     private boolean checkoutSolutionRepository = false;
 
     @Column(name = "checkout_path")
-    private String checkoutPath;
+    private String testCheckoutPath;
+
+    @Column(name = "assignment_checkout_path")
+    private String assignmentCheckoutPath;
+
+    @Column(name = "solution_checkout_path")
+    private String solutionCheckoutPath;
 
     @Column(name = "timeout_seconds")
     private int timeoutSeconds;
@@ -85,7 +91,9 @@ public ProgrammingExerciseBuildConfig() {
     public ProgrammingExerciseBuildConfig(ProgrammingExerciseBuildConfig originalBuildConfig) {
         this.setBranch(originalBuildConfig.getBranch());
         this.setBuildPlanConfiguration(originalBuildConfig.getBuildPlanConfiguration());
-        this.setCheckoutPath(originalBuildConfig.getCheckoutPath());
+        this.setTestCheckoutPath(originalBuildConfig.getTestCheckoutPath());
+        this.setAssignmentCheckoutPath(originalBuildConfig.getAssignmentCheckoutPath());
+        this.setSolutionCheckoutPath(originalBuildConfig.getSolutionCheckoutPath());
         this.setCheckoutSolutionRepository(originalBuildConfig.getCheckoutSolutionRepository());
         this.setDockerFlags(originalBuildConfig.getDockerFlags());
         this.setSequentialTestRuns(originalBuildConfig.hasSequentialTestRuns());
@@ -166,12 +174,12 @@ public void setCheckoutSolutionRepository(boolean checkoutSolutionRepository) {
         this.checkoutSolutionRepository = checkoutSolutionRepository;
     }
 
-    public String getCheckoutPath() {
-        return checkoutPath;
+    public String getTestCheckoutPath() {
+        return testCheckoutPath;
     }
 
-    public void setCheckoutPath(String checkoutPath) {
-        this.checkoutPath = checkoutPath;
+    public void setTestCheckoutPath(String testCheckoutPath) {
+        this.testCheckoutPath = testCheckoutPath;
     }
 
     public int getTimeoutSeconds() {
@@ -268,11 +276,27 @@ public void generateAndSetBuildPlanAccessSecret() {
         buildPlanAccessSecret = UUID.randomUUID().toString();
     }
 
+    public String getAssignmentCheckoutPath() {
+        return assignmentCheckoutPath;
+    }
+
+    public void setAssignmentCheckoutPath(String assignmentCheckoutPath) {
+        this.assignmentCheckoutPath = assignmentCheckoutPath;
+    }
+
+    public String getSolutionCheckoutPath() {
+        return solutionCheckoutPath;
+    }
+
+    public void setSolutionCheckoutPath(String solutionCheckoutPath) {
+        this.solutionCheckoutPath = solutionCheckoutPath;
+    }
+
     @Override
     public String toString() {
         return "BuildJobConfig{" + "id=" + getId() + ", sequentialTestRuns=" + sequentialTestRuns + ", branch='" + branch + '\'' + ", buildPlanConfiguration='"
                 + buildPlanConfiguration + '\'' + ", buildScript='" + buildScript + '\'' + ", checkoutSolutionRepository=" + checkoutSolutionRepository + ", checkoutPath='"
-                + checkoutPath + '\'' + ", timeoutSeconds=" + timeoutSeconds + ", dockerFlags='" + dockerFlags + '\'' + ", testwiseCoverageEnabled=" + testwiseCoverageEnabled
+                + testCheckoutPath + '\'' + ", timeoutSeconds=" + timeoutSeconds + ", dockerFlags='" + dockerFlags + '\'' + ", testwiseCoverageEnabled=" + testwiseCoverageEnabled
                 + ", theiaImage='" + theiaImage + '\'' + ", allowBranching=" + allowBranching + ", branchRegex='" + branchRegex + '\'' + '}';
     }
 }

src/main/java/de/tum/cit/aet/artemis/programming/service/AuxiliaryRepositoryService.java

@@ -1,13 +1,13 @@
 package de.tum.cit.aet.artemis.programming.service;
 
+import static de.tum.cit.aet.artemis.core.config.Constants.ALLOWED_CHECKOUT_DIRECTORY;
 import static de.tum.cit.aet.artemis.core.config.Constants.PROFILE_CORE;
 
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.regex.Matcher;
-import java.util.regex.Pattern;
 
 import org.springframework.context.annotation.Profile;
 import org.springframework.stereotype.Service;
@@ -27,8 +27,6 @@ public class AuxiliaryRepositoryService {
 
     private static final String AUX_REPO_ENTITY_NAME = "programmingExercise";
 
-    private static final Pattern ALLOWED_CHECKOUT_DIRECTORY = Pattern.compile("[\\w-]+(/[\\w-]+)*$");
-
     private final AuxiliaryRepositoryRepository auxiliaryRepositoryRepository;
 
     public AuxiliaryRepositoryService(AuxiliaryRepositoryRepository auxiliaryRepositoryRepository) {

src/main/java/de/tum/cit/aet/artemis/programming/service/BuildScriptProviderService.java

@@ -10,6 +10,7 @@
 import java.util.concurrent.ConcurrentHashMap;
 
 import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.boot.context.event.ApplicationReadyEvent;
@@ -18,6 +19,8 @@
 import org.springframework.core.io.Resource;
 import org.springframework.stereotype.Service;
 
+import de.tum.cit.aet.artemis.core.config.Constants;
+import de.tum.cit.aet.artemis.core.service.ProfileService;
 import de.tum.cit.aet.artemis.core.service.ResourceLoaderService;
 import de.tum.cit.aet.artemis.programming.domain.ProgrammingExercise;
 import de.tum.cit.aet.artemis.programming.domain.ProgrammingExerciseBuildConfig;
@@ -39,14 +42,17 @@ public class BuildScriptProviderService {
 
     private final Map<String, String> scriptCache = new ConcurrentHashMap<>();
 
+    private final ProfileService profileService;
+
     /**
      * Constructor for BuildScriptProvider, which loads all scripts into the cache to speed up retrieval
      * during the runtime of the application
      *
      * @param resourceLoaderService resourceLoaderService
      */
-    public BuildScriptProviderService(ResourceLoaderService resourceLoaderService) {
+    public BuildScriptProviderService(ResourceLoaderService resourceLoaderService, ProfileService profileService) {
         this.resourceLoaderService = resourceLoaderService;
+        this.profileService = profileService;
     }
 
     /**
@@ -69,6 +75,9 @@ public void cacheOnBoot() {
                 String uniqueKey = directory + "_" + filename;
                 byte[] fileContent = IOUtils.toByteArray(resource.getInputStream());
                 String script = new String(fileContent, StandardCharsets.UTF_8);
+                if (!profileService.isLocalCiActive()) {
+                    script = replacePlaceholders(script, null, null, null);
+                }
                 scriptCache.put(uniqueKey, script);
             }
             catch (IOException e) {
@@ -112,6 +121,9 @@ public String getScriptFor(ProgrammingLanguage programmingLanguage, Optional<Pro
         }
         byte[] fileContent = IOUtils.toByteArray(fileResource.getInputStream());
         String script = new String(fileContent, StandardCharsets.UTF_8);
+        if (!profileService.isLocalCiActive()) {
+            script = replacePlaceholders(script, null, null, null);
+        }
         scriptCache.put(uniqueKey, script);
         log.debug("Caching script for {}", uniqueKey);
         return script;
@@ -166,4 +178,42 @@ public String buildTemplateName(Optional<ProjectType> projectType, Boolean stati
         }
         return String.join("_", fileNameComponents) + "." + fileExtension;
     }
+
+    /**
+     * Replaces placeholders in the given result paths with the actual paths.
+     *
+     * @param resultPaths the result paths to replace the placeholders in
+     * @param buildConfig the build configuration containing the actual paths
+     * @return the result paths with the placeholders replaced
+     */
+    public List<String> replaceResultPathsPlaceholders(List<String> resultPaths, ProgrammingExerciseBuildConfig buildConfig) {
+        List<String> replacedResultPaths = new ArrayList<>();
+        for (String resultPath : resultPaths) {
+            String replacedResultPath = replacePlaceholders(resultPath, buildConfig.getAssignmentCheckoutPath(), buildConfig.getSolutionCheckoutPath(),
+                    buildConfig.getTestCheckoutPath());
+            replacedResultPaths.add(replacedResultPath);
+        }
+        return replacedResultPaths;
+    }
+
+    /**
+     * Replaces placeholders in the given original string with the actual paths.
+     *
+     * @param originalString the original string to replace the placeholders in
+     * @param assignmentRepo the assignment repository name
+     * @param solutionRepo   the solution repository name
+     * @param testRepo       the test repository name
+     * @return the original string with the placeholders replaced
+     */
+    public String replacePlaceholders(String originalString, String assignmentRepo, String solutionRepo, String testRepo) {
+        assignmentRepo = !StringUtils.isBlank(assignmentRepo) ? assignmentRepo : Constants.ASSIGNMENT_REPO_NAME;
+        solutionRepo = solutionRepo != null && !solutionRepo.isBlank() ? solutionRepo : Constants.SOLUTION_REPO_NAME;
+        testRepo = testRepo != null && !testRepo.isBlank() ? testRepo : Constants.TEST_REPO_NAME;
+
+        String replacedResultPath = originalString.replace(Constants.ASSIGNMENT_REPO_PARENT_PLACEHOLDER, assignmentRepo);
+        replacedResultPath = replacedResultPath.replace(Constants.ASSIGNMENT_REPO_PLACEHOLDER, "/" + assignmentRepo + "/src");
+        replacedResultPath = replacedResultPath.replace(Constants.SOLUTION_REPO_PLACEHOLDER, solutionRepo);
+        replacedResultPath = replacedResultPath.replace(Constants.TEST_REPO_PLACEHOLDER, testRepo);
+        return replacedResultPath;
+    }
 }