Development: Allow all filenames in programming exercise templates

settings.gradle

@@ -1,3 +1,5 @@
+import org.apache.tools.ant.DirectoryScanner
+
 pluginManagement {
     plugins {
         id 'org.springframework.boot' version "${spring_boot_version}"
@@ -11,3 +13,7 @@ pluginManagement {
 }
 
 rootProject.name = 'Artemis'
+
+// needed for programming exercise templates
+DirectoryScanner.removeDefaultExclude "**/.gitattributes"
+DirectoryScanner.removeDefaultExclude "**/.gitignore"

src/main/java/de/tum/in/www1/artemis/service/FileService.java

@@ -108,21 +108,6 @@ public class FileService implements DisposableBean {
 
     public static final String PICTURE_FILE_SUBPATH = "/api/files/drag-and-drop/drag-items/";
 
-    /**
-     * Filenames for which the template filename differs from the filename it should have in the repository.
-     */
-    // @formatter:off
-    private static final Map<String, String> FILENAME_REPLACEMENTS = Map.ofEntries(
-        Map.entry("git.ignore.file", ".gitignore"),
-        Map.entry("git.attributes.file", ".gitattributes"),
-        Map.entry("Makefile.file", "Makefile"),
-        Map.entry("dune.file", "dune"),
-        Map.entry("Fast.file", "Fastfile"),
-        Map.entry("App.file", "Appfile"),
-        Map.entry("Scan.file", "Scanfile"),
-        Map.entry("gradlew.file", "gradlew"));
-    // @formatter:on
-
     /**
      * These directories get falsely marked as files and should be ignored during copying.
      */
@@ -421,8 +406,7 @@ private Path generateTargetPath(final Resource resource, final Path prefix, fina
             filePath = Path.of(url);
         }
 
-        final Path targetPath = generateTargetPath(filePath, prefix, targetDirectory, keepParentDirectory);
-        return applyFilenameReplacements(targetPath);
+        return generateTargetPath(filePath, prefix, targetDirectory, keepParentDirectory);
     }
 
     /**
@@ -467,19 +451,6 @@ private List<Path> getPathElements(final Path path) {
         return elements;
     }
 
-    /**
-     * Replaces filenames where the template name differs from the name the file should have in the repository.
-     *
-     * @param originalTargetPath The path to a file.
-     * @return The path with replacements applied where necessary.
-     */
-    private Path applyFilenameReplacements(final Path originalTargetPath) {
-        final Path filename = originalTargetPath.getFileName();
-
-        final String newFilename = FILENAME_REPLACEMENTS.getOrDefault(filename.toString(), filename.toString());
-        return originalTargetPath.getParent().resolve(newFilename);
-    }
-
     /**
      * Checks if the given path has been identified as a file, but it actually points to a directory.
      *

src/main/java/de/tum/in/www1/artemis/service/ResourceLoaderService.java

@@ -10,6 +10,7 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Objects;
 import java.util.Optional;
@@ -37,7 +38,7 @@ public class ResourceLoaderService {
 
     private static final Logger log = LoggerFactory.getLogger(ResourceLoaderService.class);
 
-    private static final String ALL_FILES_GLOB = "**" + File.separator + "*.*";
+    private static final String ALL_PATHS_ANT_PATTERN = "**";
 
     @Value("${artemis.template-path:#{null}}")
     private Optional<Path> templateFileSystemPath;
@@ -68,7 +69,7 @@ public Resource getResource(final Path path) {
 
         // Try to load from filesystem if override is allowed for path
         if (isOverrideAllowed(path)) {
-            final String resourceLocation = getFileResourceLocation(path);
+            final String resourceLocation = getFileSystemResourceLocation(path);
             resource = resourceLoader.getResource(resourceLocation);
         }
 
@@ -84,20 +85,20 @@ public Resource getResource(final Path path) {
     /**
      * Recursively loads the resources from the specified directory.
      * <p>
-     * Only relative paths are allowed.
+     * Only relative paths are allowed. Does not return directories.
      *
      * @param basePath A relative path pattern to a resource.
      * @return The resources located by the specified pathPattern.
      */
     @NotNull
-    public Resource[] getResources(final Path basePath) {
-        return getResources(basePath, ALL_FILES_GLOB);
+    public Resource[] getFileResources(final Path basePath) {
+        return getFileResources(basePath, ALL_PATHS_ANT_PATTERN);
     }
 
     /**
      * Loads the resources from the specified path pattern.
      * <p>
-     * Only relative paths are allowed.
+     * Only relative paths are allowed. Does not return directories.
      * <p>
      * Examples for path patterns: {@code *.sh}, {@code base/**}. Use forward slashes to separate path parts.
      *
@@ -106,16 +107,16 @@ public Resource[] getResources(final Path basePath) {
      * @return The resources located by the specified pathPattern.
      */
     @NotNull
-    public Resource[] getResources(final Path basePath, final String pattern) {
+    public Resource[] getFileResources(final Path basePath, final String pattern) {
         checkValidPathElseThrow(basePath);
 
         Resource[] resources = null;
 
         // Try to load from filesystem if override is allowed for pathPattern
         if (isOverrideAllowed(basePath)) {
-            final String resourceLocation = getFileResourceLocation(basePath, pattern);
+            final String resourceLocation = getFileSystemResourceLocation(basePath, pattern);
             try {
-                resources = resourceLoader.getResources(resourceLocation);
+                resources = getFileResources(resourceLocation);
             }
             catch (IOException e) {
                 log.debug("Could not load resources '{}' from filesystem.", resourceLocation, e);
@@ -126,7 +127,7 @@ public Resource[] getResources(final Path basePath, final String pattern) {
         if (resources == null || resources.length == 0) {
             final String resourceLocation = getClassPathResourceLocation(basePath, pattern);
             try {
-                resources = resourceLoader.getResources(resourceLocation);
+                resources = getFileResources(resourceLocation);
             }
             catch (IOException e) {
                 log.debug("Could not load resources '{}' from classpath.", resourceLocation, e);
@@ -136,17 +137,30 @@ public Resource[] getResources(final Path basePath, final String pattern) {
         return Objects.requireNonNullElseGet(resources, () -> new Resource[0]);
     }
 
+    private Resource[] getFileResources(final String resourceLocation) throws IOException {
+        final var fileAndDirectoryResources = resourceLoader.getResources(resourceLocation);
+
+        final var fileResources = new ArrayList<Resource>();
+        for (final var resource : fileAndDirectoryResources) {
+            if (resource.getFile().isFile()) {
+                fileResources.add(resource);
+            }
+        }
+
+        return fileResources.toArray(Resource[]::new);
+    }
+
     private void checkValidPathElseThrow(final Path path) {
         if (path.isAbsolute()) {
             throw new IllegalArgumentException("Cannot load resources from absolute paths!");
         }
     }
 
-    private String getFileResourceLocation(final Path resourcePath) {
+    private String getFileSystemResourceLocation(final Path resourcePath) {
         return "file:" + resolveResourcePath(resourcePath).toString();
     }
 
-    private String getFileResourceLocation(final Path resourcePath, final String pathPattern) {
+    private String getFileSystemResourceLocation(final Path resourcePath, final String pathPattern) {
         final String systemPathPattern = File.separator + adaptPathPatternToSystem(pathPattern);
         return "file:" + resolveResourcePath(resourcePath).toString() + systemPathPattern;
     }

src/main/java/de/tum/in/www1/artemis/service/connectors/BuildScriptProviderService.java

@@ -57,7 +57,7 @@ public BuildScriptProviderService(ResourceLoaderService resourceLoaderService) {
      */
     @EventListener(ApplicationReadyEvent.class)
     public void cacheOnBoot() {
-        var resources = this.resourceLoaderService.getResources(Path.of("templates", "aeolus"));
+        var resources = this.resourceLoaderService.getFileResources(Path.of("templates", "aeolus"));
         for (var resource : resources) {
             try {
                 String filename = resource.getFilename();

src/main/java/de/tum/in/www1/artemis/service/connectors/aeolus/AeolusTemplateService.java

@@ -62,7 +62,7 @@ public AeolusTemplateService(ProgrammingLanguageConfiguration programmingLanguag
     @EventListener(ApplicationReadyEvent.class)
     public void cacheOnBoot() {
         // load all scripts into the cache
-        var resources = this.resourceLoaderService.getResources(Path.of("templates", "aeolus"));
+        var resources = this.resourceLoaderService.getFileResources(Path.of("templates", "aeolus"));
         for (var resource : resources) {
             try {
                 String filename = resource.getFilename();

src/main/java/de/tum/in/www1/artemis/service/programming/JavaTemplateUpgradeService.java

@@ -144,13 +144,13 @@ private Resource[] getTemplateResources(ProgrammingExercise exercise, String fil
         // Get general template resources
         final Path programmingLanguageTemplate = ProgrammingExerciseService.getProgrammingLanguageTemplatePath(exercise.getProgrammingLanguage());
 
-        Resource[] templatePoms = resourceLoaderService.getResources(programmingLanguageTemplate, filePattern);
+        Resource[] templatePoms = resourceLoaderService.getFileResources(programmingLanguageTemplate, filePattern);
 
         // Get project type specific template resources
         if (exercise.getProjectType() != null) {
             final Path projectTypeTemplate = ProgrammingExerciseService.getProgrammingLanguageProjectTypePath(exercise.getProgrammingLanguage(), exercise.getProjectType());
 
-            final Resource[] projectTypePoms = resourceLoaderService.getResources(projectTypeTemplate, filePattern);
+            final Resource[] projectTypePoms = resourceLoaderService.getFileResources(projectTypeTemplate, filePattern);
 
             // Prefer project type specific resources
             templatePoms = projectTypePoms.length > 0 ? projectTypePoms : templatePoms;

src/main/java/de/tum/in/www1/artemis/service/programming/ProgrammingExerciseRepositoryService.java

@@ -136,7 +136,7 @@ private RepositoryResources getRepositoryResources(final ProgrammingExercise pro
         // Get path, files and prefix for the programming-language dependent files. They are copied first.
         final Path generalTemplatePath = ProgrammingExerciseService.getProgrammingLanguageTemplatePath(programmingExercise.getProgrammingLanguage())
                 .resolve(projectTypeTemplateDir);
-        Resource[] resources = resourceLoaderService.getResources(generalTemplatePath);
+        Resource[] resources = resourceLoaderService.getFileResources(generalTemplatePath);
 
         Path prefix = Path.of(programmingLanguage).resolve(projectTypeTemplateDir);
 
@@ -152,7 +152,7 @@ private RepositoryResources getRepositoryResources(final ProgrammingExercise pro
             final Path projectTypeSpecificPrefix = generalProjectTypePrefix.resolve(projectTypeTemplateDir);
             final Path projectTypeTemplatePath = programmingLanguageProjectTypePath.resolve(projectTypeTemplateDir);
 
-            final Resource[] projectTypeSpecificResources = resourceLoaderService.getResources(projectTypeTemplatePath);
+            final Resource[] projectTypeSpecificResources = resourceLoaderService.getFileResources(projectTypeTemplatePath);
 
             if (ProjectType.XCODE.equals(projectType)) {
                 // For Xcode, we don't share source code, so we only copy files once
@@ -312,7 +312,7 @@ private void setupJVMTestTemplateAndPush(final RepositoryResources resources, fi
         // Java supports multiple variants as test template
         final Path projectTemplatePath = getJavaProjectTemplatePath(templatePath, projectType);
 
-        final Resource[] projectTemplate = resourceLoaderService.getResources(projectTemplatePath);
+        final Resource[] projectTemplate = resourceLoaderService.getFileResources(projectTemplatePath);
         // keep the folder structure
         fileService.copyResources(projectTemplate, Path.of("projectTemplate"), repoLocalPath, true);
 
@@ -375,7 +375,7 @@ private void setupJVMTestTemplateProjectTypeResources(final RepositoryResources
         final Path projectTypeProjectTemplatePath = projectTypeTemplatePath.resolve("projectTemplate");
 
         try {
-            final Resource[] projectTypeProjectTemplate = resourceLoaderService.getResources(projectTypeProjectTemplatePath);
+            final Resource[] projectTypeProjectTemplate = resourceLoaderService.getFileResources(projectTypeProjectTemplatePath);
             fileService.copyResources(projectTypeProjectTemplate, resources.projectTypePrefix, repoLocalPath, false);
         }
         catch (FileNotFoundException fileNotFoundException) {
@@ -397,7 +397,7 @@ private void setupTestTemplateRegularTestRuns(final RepositoryResources resource
         final ProjectType projectType = programmingExercise.getProjectType();
         final Path repoLocalPath = getRepoAbsoluteLocalPath(resources.repository);
         final Path testFilePath = templatePath.resolve(TEST_FILES_PATH);
-        final Resource[] testFileResources = resourceLoaderService.getResources(testFilePath);
+        final Resource[] testFileResources = resourceLoaderService.getFileResources(testFilePath);
         final Path packagePath = repoLocalPath.resolve(TEST_DIR).resolve(PACKAGE_NAME_FOLDER_PLACEHOLDER).toAbsolutePath();
 
         sectionsMap.put("non-sequential", true);
@@ -440,7 +440,7 @@ private void setupBuildToolProjectFile(final Path repoLocalPath, final ProjectTy
 
     private void setupStaticCodeAnalysisConfigFiles(final RepositoryResources resources, final Path templatePath, final Path repoLocalPath) throws IOException {
         final Path staticCodeAnalysisConfigPath = templatePath.resolve("staticCodeAnalysisConfig");
-        final Resource[] staticCodeAnalysisResources = resourceLoaderService.getResources(staticCodeAnalysisConfigPath);
+        final Resource[] staticCodeAnalysisResources = resourceLoaderService.getFileResources(staticCodeAnalysisConfigPath);
         fileService.copyResources(staticCodeAnalysisResources, resources.prefix, repoLocalPath, true);
     }
 
@@ -450,7 +450,7 @@ private void overwriteProjectTypeSpecificFiles(final RepositoryResources resourc
                 .resolve(TEST_DIR);
 
         try {
-            final Resource[] projectTypeTestFileResources = resourceLoaderService.getResources(projectTypeTemplatePath);
+            final Resource[] projectTypeTestFileResources = resourceLoaderService.getFileResources(projectTypeTemplatePath);
             // filter non-existing resources to avoid exceptions
             final List<Resource> existingProjectTypeTestFileResources = new ArrayList<>();
             for (final Resource resource : projectTypeTestFileResources) {
@@ -562,7 +562,7 @@ private void setupBuildStage(final Path resourcePrefix, final Path templatePath,
         }
 
         final Path buildStageResourcesPath = templatePath.resolve(TEST_FILES_PATH).resolve(buildStageTemplateSubDirectory);
-        final Resource[] buildStageResources = resourceLoaderService.getResources(buildStageResourcesPath);
+        final Resource[] buildStageResources = resourceLoaderService.getFileResources(buildStageResourcesPath);
         fileService.copyResources(buildStageResources, resourcePrefix, packagePath, false);
 
         if (projectType != null) {
@@ -574,7 +574,7 @@ private void overwriteStageFilesForProjectType(final Path resourcePrefix, final
             throws IOException {
         final Path buildStageResourcesPath = projectTemplatePath.resolve(TEST_FILES_PATH).resolve(buildStageTemplateSubDirectory);
         try {
-            final Resource[] buildStageResources = resourceLoaderService.getResources(buildStageResourcesPath);
+            final Resource[] buildStageResources = resourceLoaderService.getFileResources(buildStageResourcesPath);
             fileService.copyResources(buildStageResources, resourcePrefix, packagePath, false);
         }
         catch (FileNotFoundException fileNotFoundException) {

src/test/java/de/tum/in/www1/artemis/service/FileServiceTest.java

@@ -382,26 +382,12 @@ void testCopyResourceRemovePrefix(@TempDir Path targetDir) throws IOException {
         assertThat(expectedTargetFile).exists().isNotEmptyFile();
     }
 
-    @Test
-    void testRenameSpecialFilename(@TempDir Path targetDir) throws IOException {
-        final Path sourceFile = overridableBasePath.resolve("Makefile.file");
-        FileUtils.writeStringToFile(sourceFile.toFile(), "content", Charset.defaultCharset());
-
-        final Resource[] resources = resourceLoaderService.getResources(overridableBasePath);
-        assertThat(resources).isNotEmpty();
-
-        fileService.copyResources(resources, Path.of("templates"), targetDir, true);
-
-        final Path expectedTargetFile = targetDir.resolve("jenkins").resolve("Makefile");
-        assertThat(expectedTargetFile).exists().isNotEmptyFile();
-    }
-
     @Test
     void testIgnoreDirectoryFalsePositives(@TempDir Path targetDir) throws IOException {
         final Path sourceDirectory = overridableBasePath.resolve("package.xcworkspace");
         Files.createDirectories(sourceDirectory);
 
-        final Resource[] resources = resourceLoaderService.getResources(overridableBasePath);
+        final Resource[] resources = resourceLoaderService.getFileResources(overridableBasePath);
         assertThat(resources).isNotEmpty();
 
         fileService.copyResources(resources, Path.of("templates"), targetDir, true);