Development: Execute tests only for changed modules on non-default branchs

.github/PULL_REQUEST_TEMPLATE.md

@@ -111,7 +111,7 @@ Prerequisites:
 - [ ] Test 2
 
 ### Test Coverage
-<!-- Please add the test coverages for all changed files modified in this PR here. You can use `supporting_script/generate_code_cov_table/generate_code_cov_table.py` to automatically generate the coverage table from the corresponding artefacts of your branch (follow the ReadMe for setup details). -->
+<!-- Please add the test coverages for all changed files modified in this PR here. You can use `supporting_script/code-coverage/generate_code_cov_table/generate_code_cov_table.py` to automatically generate the coverage table from the corresponding artefacts of your branch (follow the ReadMe for setup details). -->
 <!-- Alternatively you can execute the tests locally (see build.gradle and package.json) or look into the corresponding artefacts. -->
 <!-- The line coverage must be above 90% for changes files, and you must use extensive and useful assertions for server tests and expect statements for client tests. -->
 <!-- Note: Confirm in the last column that you have implemented extensive assertions for server tests and expect statements for client tests. -->

.github/dependabot.yml

@@ -57,7 +57,7 @@ updates:
 
   # Check for version updates for Python dependencies (coverage)
   - package-ecosystem: "pip"
-    directory: "/supporting_scripts/generate_code_cov_table"
+    directory: "/supporting_scripts/code-coverage/generate_code_cov_table"
     schedule:
       interval: "weekly"
     reviewers:

.github/workflows/test.yml

@@ -60,7 +60,32 @@ jobs:
           ${{ env.java }}
         cache: 'gradle'
     - name: Java Tests
-      run: set -o pipefail && ./gradlew --console=plain test jacocoTestReport -x webapp jacocoTestCoverageVerification | tee tests.log
+      run: |
+        set -o pipefail
+
+        DEFAULT_BRANCH="${{ github.event.repository.default_branch }}"
+        CURRENT_BRANCH="${{ github.ref_name }}"
+
+        if [[ "$DEFAULT_BRANCH" != "$CURRENT_BRANCH" ]]; then
+          # Explicitly fetch as the clone action only clones the current branch
+          git fetch origin "$DEFAULT_BRANCH"
+
+          chmod +x ./supporting_scripts/get_changed_modules.sh
+          CHANGED_MODULES=$(./supporting_scripts/get_changed_modules.sh "origin/$DEFAULT_BRANCH")
+
+          # Restrict executed tests to changed modules if there is diff between this and the base branch
+          if [ -n "${CHANGED_MODULES}" ]; then
+            IFS=,
+            TEST_MODULE_TAGS=$(echo "-DincludeModules=${CHANGED_MODULES[*]}")
+
+            echo "Executing tests for modules: $CHANGED_MODULES"
+            ./gradlew --console=plain test jacocoTestReport -x webapp jacocoTestCoverageVerification "$TEST_MODULE_TAGS" | tee tests.log
+            exit 0
+          fi
+        fi
+
+        echo "Executing all tests"
+        ./gradlew --console=plain test jacocoTestReport -x webapp jacocoTestCoverageVerification | tee tests.log
     - name: Print failed tests
       if: failure()
       run: grep "Test >.* FAILED\$" tests.log || echo "No failed tests."
@@ -92,8 +117,13 @@ jobs:
       uses: actions/upload-artifact@v4
       with:
         name: Coverage Report Server Tests
-        path: build/reports/jacoco/test/html/
-
+        path: build/reports/jacoco/test/html
+    - name: Append Per-Module Coverage to Job Summary
+      if: success() || failure()
+      run: |
+        AGGREGATED_REPORT_FILE=./module_coverage_report.md
+        python3 ./supporting_scripts/code-coverage/per_module_cov_report/parse_module_coverage.py build/reports/jacoco $AGGREGATED_REPORT_FILE
+        cat $AGGREGATED_REPORT_FILE > $GITHUB_STEP_SUMMARY
 
   server-tests-mysql:
       needs: [ server-tests ]

build.gradle

@@ -118,16 +118,38 @@ modernizer {
     exclusions = ["java/util/Optional.get:()Ljava/lang/Object;"]
 }
 
+// Allow using in jacoco.gradle
+ext {
+    includedTestTags = System.getProperty("includeTags")
+    includedTags = !includedTestTags ? new String[]{} : includedTestTags.split(",") as String[]
+    includedModulesTag = System.getProperty("includeModules")
+    includedModules = !includedModulesTag ? new String[]{} : includedModulesTag.split(",") as String[]
+
+    runAllTests = includedTags.size() == 0 && includedModules.size() == 0
+    BasePath = "de/tum/cit/aet/artemis"
+}
+
 // Execute the test cases: ./gradlew test
 // Execute only architecture tests: ./gradlew test -DincludeTags="ArchitectureTest"
+// Execute tests only for specific modules: ./gradlew test -DincludeModules="atlas". Using this flag, "includeTags" will be ignored.
 test {
-    if (System.getProperty("includeTags")) {
-        useJUnitPlatform {
-            includeTags System.getProperty("includeTags")
+    if (runAllTests) {
+        useJUnitPlatform()
+        exclude "**/*IT*", "**/*IntTest*"
+    } else if (includedModules.size() == 0) {
+        // not running all tests, but not module-specific ones -> use tags
+        useJUnitPlatform() {
+            includeTags includedTags
         }
     } else {
         useJUnitPlatform()
-        exclude "**/*IT*", "**/*IntTest*"
+        // Always execute "shared"-folder when executing module-specifc tests
+        includedModules += "shared"
+        filter { testFilter ->
+            includedModules.each { val ->
+                testFilter.includeTestsMatching("de.tum.cit.aet.artemis.$val.*")
+            }
+        }
     }
 
     testLogging {
@@ -144,59 +166,12 @@ tasks.register("testReport", TestReport) {
     testResults.from(test)
 }
 
-jacoco {
-    toolVersion = "0.8.12"
-}
-
 jar {
     enabled = false
 }
 
-private excludedClassFilesForReport(classDirectories) {
-    classDirectories.setFrom(files(classDirectories.files.collect {
-        fileTree(dir: it,
-            exclude: [
-                "**/de/tum/cit/aet/artemis/**/domain/**/*_*",
-                "**/de/tum/cit/aet/artemis/core/config/migration/entries/**",
-                "**/gradle-wrapper.jar/**"
-            ]
-        )
-    }))
-}
-
-jacocoTestReport {
-    reports {
-        xml.required = true
-    }
-    // we want to ignore some generated files in the domain folders
-    afterEvaluate {
-        excludedClassFilesForReport(classDirectories)
-    }
-}
-
-jacocoTestCoverageVerification {
-    violationRules {
-        rule {
-            limit {
-                counter = "INSTRUCTION"
-                value = "COVEREDRATIO"
-                // TODO: in the future the following value should become higher than 0.92
-                minimum = 0.892
-            }
-            limit {
-                counter = "CLASS"
-                value = "MISSEDCOUNT"
-                // TODO: in the future the following value should become less than 10
-                maximum = 65
-            }
-        }
-    }
-    // we want to ignore some generated files in the domain folders
-    afterEvaluate {
-        excludedClassFilesForReport(classDirectories)
-    }
-}
-check.dependsOn jacocoTestCoverageVerification
+// Dynamic generation of jacoco test report generation-/coverage verification-tasks (per-module)
+apply from: "gradle/jacoco.gradle"
 
 configurations {
     providedRuntime

gradle/jacoco.gradle

@@ -0,0 +1,217 @@
+ext {
+    AggregatedCoverageThresholds = [
+        "INSTRUCTION": 0.900,
+        "CLASS": 10
+    ];
+    // (Isolated) thresholds when executing each module on its own
+    ModuleCoverageThresholds = [
+        "assessment"   : [
+            "INSTRUCTION": 0.774,
+            "CLASS": 9
+        ],
+        "athena"       : [
+            "INSTRUCTION": 0.863,
+            "CLASS": 2
+        ],
+        "atlas"        : [
+            "INSTRUCTION": 0.846,
+            "CLASS": 13
+        ],
+        "buildagent"   : [
+            "INSTRUCTION": 0.304,
+            "CLASS": 13
+        ],
+        "communication": [
+            "INSTRUCTION": 0.893,
+            "CLASS": 7
+        ],
+        "core"         : [
+            "INSTRUCTION": 0.658,
+            "CLASS": 69
+        ],
+        "exam"         : [
+            "INSTRUCTION": 0.914,
+            "CLASS": 1
+        ],
+        "exercise"     : [
+            "INSTRUCTION": 0.649,
+            "CLASS": 9
+        ],
+        "fileupload"   : [
+            "INSTRUCTION": 0.944,
+            "CLASS": 0
+        ],
+        "iris"        : [
+            "INSTRUCTION": 0.706,
+            "CLASS": 22
+        ],
+        "lecture"      : [
+            "INSTRUCTION": 0.867,
+            "CLASS": 0
+        ],
+        "lti"          : [
+            "INSTRUCTION": 0.770,
+            "CLASS": 3
+        ],
+        "modeling"     : [
+            "INSTRUCTION": 0.892,
+            "CLASS": 2
+        ],
+        "plagiarism"   : [
+            "INSTRUCTION": 0.770,
+            "CLASS": 1
+        ],
+        "programming"  : [
+            "INSTRUCTION": 0.861,
+            "CLASS": 13
+        ],
+        "quiz"         : [
+            "INSTRUCTION": 0.785,
+            "CLASS": 6
+        ],
+        "text"         : [
+            "INSTRUCTION": 0.847,
+            "CLASS": 0
+        ],
+        "tutorialgroup": [
+            "INSTRUCTION": 0.915,
+            "CLASS": 0
+        ],
+    ]
+    // If no explicit modules defined -> generate reports and validate for each module
+    reportedModules = includedModules.size() == 0
+        ? ModuleCoverageThresholds.collect {element -> element.key}
+        : includedModules as ArrayList
+
+    // we want to ignore some generated files in the domain folders
+    ignoredDirectories = [
+        "**/$BasePath/**/domain/**/*_*",
+        "**/$BasePath/core/config/migration/entries/**",
+        "**/gradle-wrapper.jar/**"
+    ]
+}
+
+jacoco {
+    toolVersion = "0.8.12"
+}
+
+jacocoTestReport {
+    // For the aggregated report
+    reports {
+        xml.required = true
+        xml.outputLocation = file("build/reports/jacoco/test/jacocoTestReport.xml")
+        html.required = true
+        html.outputLocation = file("build/reports/jacoco/test/html")
+    }
+
+    finalizedBy reportedModules
+        .collect { module -> registerJacocoReportTask(module as String, jacocoTestReport) }
+        .findAll { task -> task != null}
+}
+
+jacocoTestCoverageVerification {
+    // Only run full coverage when no specific modules set
+    enabled = reportedModules.size() == 0
+
+    def minInstructionCoveredRatio = AggregatedCoverageThresholds["INSTRUCTION"] as double
+    def maxNumberUncoveredClasses = AggregatedCoverageThresholds["CLASS"] as int
+    applyVerificationRule(jacocoTestCoverageVerification, minInstructionCoveredRatio, maxNumberUncoveredClasses)
+
+    finalizedBy reportedModules
+        .collect { module -> registerJacocoTestCoverageVerification(module as String, jacocoTestCoverageVerification) }
+        .findAll { task -> task != null}
+}
+check.dependsOn jacocoTestCoverageVerification
+
+/**
+ * Registers a JacocoReport task based on the provided parameters.
+ *
+ * @param moduleName The module name to include in the report.
+ * @param rootTask The root JacocoReport root task.
+ * @return The configured JacocoReport task.
+ */
+private JacocoReport registerJacocoReportTask(String moduleName, JacocoReport rootTask) {
+    def taskName = "jacocoCoverageReport-$moduleName"
+
+    JacocoReport task = project.tasks.register(taskName, JacocoReport).get()
+    task.description = "Generates JaCoCo coverage report for $moduleName"
+
+    prepareJacocoReportTask(task, moduleName, rootTask)
+
+    task.reports {
+        xml.required = true
+        xml.outputLocation = file("build/reports/jacoco/$moduleName/jacocoTestReport.xml")
+        html.required = true
+        html.outputLocation = file("build/reports/jacoco/$moduleName/html")
+    }
+
+    return task
+}
+
+/**
+ * Registers a JacocoCoverageVerification task based on the provided parameters.
+ *
+ * @param moduleName The module name to validate rules for.
+ * @param rootTask The root JacocoCoverageVerification task.
+ * @return The configured JacocoCoverageVerification task.
+ */
+private JacocoCoverageVerification registerJacocoTestCoverageVerification(String moduleName, JacocoCoverageVerification rootTask) {
+    def taskName = "jacocoTestCoverageVerification-$moduleName"
+
+    def thresholds = ModuleCoverageThresholds[moduleName]
+    if (thresholds == null) {
+        println "No coverage thresholds defined for module '$moduleName'. Skipping verification..."
+        return null
+    }
+    def minInstructionCoveredRatio = thresholds["INSTRUCTION"] as double
+    def maxNumberUncoveredClasses = thresholds["CLASS"] as int
+
+    JacocoCoverageVerification task = project.tasks.register(taskName, JacocoCoverageVerification).get()
+    task.description = "Validates JaCoCo coverage for vioalations for $moduleName"
+
+    prepareJacocoReportTask(task, moduleName, rootTask)
+    applyVerificationRule(task, minInstructionCoveredRatio, maxNumberUncoveredClasses)
+
+    return task
+}
+
+/**
+ * Prepares a Jacoco report task (report & verification) to match a specific
+ * @param task that is modified
+ * @param moduleName of the module.
+ * @param rootTask the JacocoReportBase root task
+ */
+private void prepareJacocoReportTask(JacocoReportBase task, String moduleName, JacocoReportBase rootTask) {
+    task.group = "Reporting"
+    task.executionData = project.fileTree("${project.layout.buildDirectory.get()}/jacoco") {
+        include "test.exec"
+    }
+
+    def modulePath = "$BasePath/$moduleName/**/*.class"
+    task.sourceDirectories.setFrom(project.files("src/main/java/$modulePath"))
+    task.classDirectories.setFrom(
+        files(rootTask.classDirectories.files.collect { classDir ->
+            project.fileTree(classDir) {
+                includes=[modulePath]
+                excludes=ignoredDirectories
+            }
+        })
+    )
+}
+
+private static void applyVerificationRule(JacocoCoverageVerification task, double minInstructionCoveredRatio, int maxNumberUncoveredClasses) {
+    task.violationRules {
+        rule {
+            limit {
+                counter = "INSTRUCTION"
+                value = "COVEREDRATIO"
+                minimum = minInstructionCoveredRatio
+            }
+            limit {
+                counter = "CLASS"
+                value = "MISSEDCOUNT"
+                maximum = maxNumberUncoveredClasses
+            }
+        }
+    }
+}