From 30a887eec966a93899e895329219e70cc51054a6 Mon Sep 17 00:00:00 2001 From: Ole Vester Date: Sat, 7 Dec 2024 15:56:51 +0100 Subject: [PATCH 01/69] Add dashboard for endpoint monitoring --- .../artemis/endpoint_monitoring.json | 434 ++++++++++++++++++ 1 file changed, 434 insertions(+) create mode 100644 docker/monitoring/grafana/provisioning/dashboards/artemis/endpoint_monitoring.json diff --git a/docker/monitoring/grafana/provisioning/dashboards/artemis/endpoint_monitoring.json b/docker/monitoring/grafana/provisioning/dashboards/artemis/endpoint_monitoring.json new file mode 100644 index 000000000000..e3412dd0f746 --- /dev/null +++ b/docker/monitoring/grafana/provisioning/dashboards/artemis/endpoint_monitoring.json @@ -0,0 +1,434 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 4, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_default" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_default" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "1000 * sum(rate (http_server_requests_seconds_sum[$__rate_interval])\n/\nrate(http_server_requests_seconds_count[$__rate_interval])) by (uri)", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "C", + "useBackend": false + } + ], + "title": "Average Request Duration (ms)", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_default" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_default" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum(increase(http_server_requests_seconds_count[$__rate_interval])) by (uri)", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "C", + "useBackend": false + } + ], + "title": "Request Count", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_default" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "% of total requests", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 9 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_default" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "100 * sum(rate(http_server_requests_seconds_count{status=~\"4..\"}[$__rate_interval])) by (uri)\n/\nsum(rate(http_server_requests_seconds_count[$__rate_interval])) by (uri)", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "C", + "useBackend": false + } + ], + "title": "4xx Endpoint Error Rate (%)", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_default" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "% of total requests", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 9 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_default" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum(rate(http_server_requests_seconds_count{status=~\"5..\"}[$__rate_interval])) by (uri)\n/\nsum(rate(http_server_requests_seconds_count[$__rate_interval])) by (uri)", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "C", + "useBackend": false + } + ], + "title": "5xx Endpoint Error Rate (%)", + "transformations": [], + "type": "timeseries" + } + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Endpoint Monitoring", + "uid": "ed9f6232-2a61-4a95-a8f4-58e08b53b804", + "version": 15, + "weekStart": "" +} From f584586d4364c22b2b2e4aa200483123ef66a19e Mon Sep 17 00:00:00 2001 From: Ole Vester Date: Sat, 7 Dec 2024 16:05:02 +0100 Subject: [PATCH 02/69] Update dashboard --- .../artemis/endpoint_monitoring.json | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/docker/monitoring/grafana/provisioning/dashboards/artemis/endpoint_monitoring.json b/docker/monitoring/grafana/provisioning/dashboards/artemis/endpoint_monitoring.json index e3412dd0f746..9be2acb32eaf 100644 --- a/docker/monitoring/grafana/provisioning/dashboards/artemis/endpoint_monitoring.json +++ b/docker/monitoring/grafana/provisioning/dashboards/artemis/endpoint_monitoring.json @@ -27,7 +27,7 @@ "type": "prometheus", "uid": "prometheus_default" }, - "description": "", + "description": "Endpoints that have top 20 highest average duration.", "fieldConfig": { "defaults": { "color": { @@ -104,7 +104,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "1000 * sum(rate (http_server_requests_seconds_sum[$__rate_interval])\n/\nrate(http_server_requests_seconds_count[$__rate_interval])) by (uri)", + "expr": "topk(20, 1000 * sum(rate (http_server_requests_seconds_sum[$__rate_interval])\n/\nrate(http_server_requests_seconds_count[$__rate_interval])) by (uri))", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, @@ -124,7 +124,7 @@ "type": "prometheus", "uid": "prometheus_default" }, - "description": "", + "description": "Request count per endpoint for the 20 most used endpoints.", "fieldConfig": { "defaults": { "color": { @@ -201,7 +201,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "sum(increase(http_server_requests_seconds_count[$__rate_interval])) by (uri)", + "expr": "topk(20, sum(increase(http_server_requests_seconds_count[$__rate_interval])) by (uri))", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, @@ -221,7 +221,7 @@ "type": "prometheus", "uid": "prometheus_default" }, - "description": "", + "description": "Error rates for the top 20 endpoints with the highest 4xx error rate.", "fieldConfig": { "defaults": { "color": { @@ -301,7 +301,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "100 * sum(rate(http_server_requests_seconds_count{status=~\"4..\"}[$__rate_interval])) by (uri)\n/\nsum(rate(http_server_requests_seconds_count[$__rate_interval])) by (uri)", + "expr": "topk(20, 100 * sum(rate(http_server_requests_seconds_count{status=~\"4..\"}[$__rate_interval])) by (uri)\n/\nsum(rate(http_server_requests_seconds_count[$__rate_interval])) by (uri))", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, @@ -321,7 +321,7 @@ "type": "prometheus", "uid": "prometheus_default" }, - "description": "", + "description": "Top 20 endpoints with the highest 5xx error rate.", "fieldConfig": { "defaults": { "color": { @@ -398,7 +398,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "sum(rate(http_server_requests_seconds_count{status=~\"5..\"}[$__rate_interval])) by (uri)\n/\nsum(rate(http_server_requests_seconds_count[$__rate_interval])) by (uri)", + "expr": "topk(20, 100 * sum(rate(http_server_requests_seconds_count{status=~\"5..\"}[$__rate_interval])) by (uri)\n/\nsum(rate(http_server_requests_seconds_count[$__rate_interval])) by (uri))", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, @@ -429,6 +429,6 @@ "timezone": "", "title": "Endpoint Monitoring", "uid": "ed9f6232-2a61-4a95-a8f4-58e08b53b804", - "version": 15, + "version": 18, "weekStart": "" } From 1acbe40fcb110d02cbfc3bcda47de0da0daea91c Mon Sep 17 00:00:00 2001 From: Ole Vester Date: Sat, 7 Dec 2024 18:29:54 +0100 Subject: [PATCH 03/69] Make number of top endpoints a variable --- .../artemis/endpoint_monitoring.json | 50 +++++++++++++++---- 1 file changed, 39 insertions(+), 11 deletions(-) diff --git a/docker/monitoring/grafana/provisioning/dashboards/artemis/endpoint_monitoring.json b/docker/monitoring/grafana/provisioning/dashboards/artemis/endpoint_monitoring.json index 9be2acb32eaf..0ba771cfdcb6 100644 --- a/docker/monitoring/grafana/provisioning/dashboards/artemis/endpoint_monitoring.json +++ b/docker/monitoring/grafana/provisioning/dashboards/artemis/endpoint_monitoring.json @@ -18,7 +18,7 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, - "id": 4, + "id": 5, "links": [], "liveNow": false, "panels": [ @@ -104,7 +104,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "topk(20, 1000 * sum(rate (http_server_requests_seconds_sum[$__rate_interval])\n/\nrate(http_server_requests_seconds_count[$__rate_interval])) by (uri))", + "expr": "topk($topNumberEndpoints, 1000 * sum(rate (http_server_requests_seconds_sum[$__rate_interval])\n/\nrate(http_server_requests_seconds_count[$__rate_interval])) by (uri))", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, @@ -201,7 +201,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "topk(20, sum(increase(http_server_requests_seconds_count[$__rate_interval])) by (uri))", + "expr": "topk($topNumberEndpoints, sum(increase(http_server_requests_seconds_count[$__rate_interval])) by (uri))", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, @@ -262,6 +262,8 @@ } }, "mappings": [], + "max": 100, + "min": 0, "thresholds": { "mode": "absolute", "steps": [ @@ -270,7 +272,8 @@ "value": null } ] - } + }, + "unit": "percent" }, "overrides": [] }, @@ -301,7 +304,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "topk(20, 100 * sum(rate(http_server_requests_seconds_count{status=~\"4..\"}[$__rate_interval])) by (uri)\n/\nsum(rate(http_server_requests_seconds_count[$__rate_interval])) by (uri))", + "expr": "topk($topNumberEndpoints, 100 * sum(rate(http_server_requests_seconds_count{status=~\"4..\"}[$__rate_interval])) by (uri)\n/\nsum(rate(http_server_requests_seconds_count[$__rate_interval])) by (uri))", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, @@ -359,6 +362,8 @@ } }, "mappings": [], + "max": 100, + "min": 0, "thresholds": { "mode": "absolute", "steps": [ @@ -367,7 +372,8 @@ "value": null } ] - } + }, + "unit": "percent" }, "overrides": [] }, @@ -398,7 +404,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "topk(20, 100 * sum(rate(http_server_requests_seconds_count{status=~\"5..\"}[$__rate_interval])) by (uri)\n/\nsum(rate(http_server_requests_seconds_count[$__rate_interval])) by (uri))", + "expr": "topk($topNumberEndpoints, 100 * sum(rate(http_server_requests_seconds_count{status=~\"5..\"}[$__rate_interval])) by (uri)\n/\nsum(rate(http_server_requests_seconds_count[$__rate_interval])) by (uri))", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, @@ -419,7 +425,29 @@ "style": "dark", "tags": [], "templating": { - "list": [] + "list": [ + { + "current": { + "selected": false, + "text": "20", + "value": "20" + }, + "description": "The number of endpoints that are considered in each graph, sorted by the highest values.", + "hide": 0, + "label": "Number Endpoints with Highest Value", + "name": "topNumberEndpoints", + "options": [ + { + "selected": true, + "text": "20", + "value": "20" + } + ], + "query": "20", + "skipUrlSync": false, + "type": "textbox" + } + ] }, "time": { "from": "now-1h", @@ -427,8 +455,8 @@ }, "timepicker": {}, "timezone": "", - "title": "Endpoint Monitoring", - "uid": "ed9f6232-2a61-4a95-a8f4-58e08b53b804", - "version": 18, + "title": "Endpoint Monitoring Copy", + "uid": "fc1fba24-8e72-439a-9759-b35315c1e2ee", + "version": 13, "weekStart": "" } From c9732aa89e35a280352a51230b49135828c28616 Mon Sep 17 00:00:00 2001 From: Ole Vester Date: Sat, 7 Dec 2024 18:52:49 +0100 Subject: [PATCH 04/69] Debug: Append something to job summary --- .github/workflows/test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ccdd88feed03..14ef8ef2f693 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -49,6 +49,8 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 60 steps: + - name: Debug - Appending anything to job summary + run: echo 'Hello World' >> $GITHUB_STEP_SUMMARY - uses: actions/checkout@v4 - name: Setup Java uses: actions/setup-java@v4 From 0bb533360757f13ef40c81c243e012e459cfa8e0 Mon Sep 17 00:00:00 2001 From: Ole Vester Date: Sat, 7 Dec 2024 18:53:40 +0100 Subject: [PATCH 05/69] Append per-module coverage to job summary (hardcoded module) --- .github/workflows/test.yml | 4 ++++ jacoco-parser.py | 44 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) create mode 100644 jacoco-parser.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 14ef8ef2f693..032790799f64 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -95,6 +95,10 @@ jobs: with: name: Coverage Report Server Tests path: build/reports/jacoco/test/html/ + - name: Append Per-Module Coverage to Job Summary + if: success() || failure() + run: | + python3 ./jacoco-parser.py >> $GITHUB_STEP_SUMMARY server-tests-mysql: diff --git a/jacoco-parser.py b/jacoco-parser.py new file mode 100644 index 000000000000..08f3998650e2 --- /dev/null +++ b/jacoco-parser.py @@ -0,0 +1,44 @@ +import xml.etree.ElementTree as ET + +def parse_jacoco_report(xml_file, package_path): + """ + Parse a JaCoCo XML report to extract coverage metrics for a specific package. + + :param xml_file: Path to the JaCoCo XML report. + :param package_path: The package path to search for. + :return: A dictionary containing coverage metrics for the package. + """ + tree = ET.parse(xml_file) + root = tree.getroot() + + # Search for the specific package + for package in root.findall(".//package"): + if package.get('name') == package_path: + coverage_metrics = {} + # Extract coverage metrics from elements + for counter in package.findall("counter"): + metric = counter.get("type") + covered = int(counter.get("covered")) + missed = int(counter.get("missed")) + total = covered + missed + coverage_percentage = (covered / total) * 100 if total > 0 else 0 + coverage_metrics[metric] = { + "covered": covered, + "missed": missed, + "total": total, + "coverage_percentage": coverage_percentage + } + return coverage_metrics + + return None + +xml_file_path = "build/reports/jacoco/test/jacocoTestReport.xml" +package = "de/tum/cit/aet/artemis/buildagent" +metrics = parse_jacoco_report(xml_file_path, package) + +if metrics: + print(f"Coverage metrics for package '{package}':") + for metric, data in metrics.items(): + print(f"{metric}: {data['coverage_percentage']:.2f}% (Covered: {data['covered']}, Missed: {data['missed']}, Total: {data['total']})") +else: + print(f"Package '{package}' not found in the report.") From c273d60d316e0a5c9b656eccdc87845bccbe3ee6 Mon Sep 17 00:00:00 2001 From: Ole Vester Date: Sat, 7 Dec 2024 18:55:10 +0100 Subject: [PATCH 06/69] Temporarily add current branch to push triggering branches --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 032790799f64..97100c4a1480 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -16,6 +16,7 @@ on: - develop - main - release/* + - feature/grafana-endpoint-monitoring tags: '[0-9]+.[0-9]+.[0-9]+' paths-ignore: - 'README.md' From c2d0dfce8d8b8aa163c4824e113eee4ff89294c8 Mon Sep 17 00:00:00 2001 From: Ole Vester Date: Sat, 7 Dec 2024 21:31:46 +0100 Subject: [PATCH 07/69] draft for module coverage --- build.gradle | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index 27099c72db3d..04ec9adbd60b 100644 --- a/build.gradle +++ b/build.gradle @@ -175,19 +175,23 @@ jacocoTestReport { } jacocoTestCoverageVerification { + def testedModules = System.getProperty("includeTags") violationRules { rule { + enabled = testedModules == null || testedModules.contains("AssessmentTest") + element = 'PACKAGE' + includes = ['de.tum.cit.aet.artemis.assessment'] limit { counter = "INSTRUCTION" value = "COVEREDRATIO" // TODO: in the future the following value should become higher than 0.92 - minimum = 0.892 + minimum = 0.999 } limit { counter = "CLASS" value = "MISSEDCOUNT" // TODO: in the future the following value should become less than 10 - maximum = 65 + maximum = 0 } } } From 48a43960f06222c291e99b163b650ec68d6b8a69 Mon Sep 17 00:00:00 2001 From: Ole Vester Date: Tue, 10 Dec 2024 18:44:08 +0100 Subject: [PATCH 08/69] Extract jacoco gradle config from build.gradle to jacoco.gradle --- .github/workflows/test.yml | 2 +- build.gradle | 62 +++--------- jacoco.gradle | 190 +++++++++++++++++++++++++++++++++++++ 3 files changed, 202 insertions(+), 52 deletions(-) create mode 100644 jacoco.gradle diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 97100c4a1480..2239b2bb3a71 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -233,7 +233,7 @@ jobs: run: ./gradlew checkstyleMain -x webapp if: success() || failure() - name: Java Architecture Tests - run: ./gradlew test -DincludeTags='ArchitectureTest' -x webapp + run: ./gradlew test -DtestModules='ArchitectureTest' -x webapp if: success() || failure() - name: Test Report uses: dorny/test-reporter@v1 diff --git a/build.gradle b/build.gradle index 04ec9adbd60b..76ddb3a090f4 100644 --- a/build.gradle +++ b/build.gradle @@ -118,12 +118,18 @@ modernizer { exclusions = ["java/util/Optional.get:()Ljava/lang/Object;"] } +// Allow using in jacoco.gradle +ext { + testTags = System.getProperty("testTags") + testedModules = !testTags ? [] : testTags.split(",") as ArrayList +} + // Execute the test cases: ./gradlew test -// Execute only architecture tests: ./gradlew test -DincludeTags="ArchitectureTest" +// Execute only architecture tests: ./gradlew test -DtestTags="ArchitectureTest" test { - if (System.getProperty("includeTags")) { + if (testedModules.size() > 0) { useJUnitPlatform { - includeTags System.getProperty("includeTags") + includeTags testedModules.join(",") } } else { useJUnitPlatform() @@ -152,54 +158,8 @@ jar { enabled = false } -private excludedClassFilesForReport(classDirectories) { - classDirectories.setFrom(files(classDirectories.files.collect { - fileTree(dir: it, - exclude: [ - "**/de/tum/cit/aet/artemis/**/domain/**/*_*", - "**/de/tum/cit/aet/artemis/core/config/migration/entries/**", - "**/gradle-wrapper.jar/**" - ] - ) - })) -} - -jacocoTestReport { - reports { - xml.required = true - } - // we want to ignore some generated files in the domain folders - afterEvaluate { - excludedClassFilesForReport(classDirectories) - } -} - -jacocoTestCoverageVerification { - def testedModules = System.getProperty("includeTags") - violationRules { - rule { - enabled = testedModules == null || testedModules.contains("AssessmentTest") - element = 'PACKAGE' - includes = ['de.tum.cit.aet.artemis.assessment'] - limit { - counter = "INSTRUCTION" - value = "COVEREDRATIO" - // TODO: in the future the following value should become higher than 0.92 - minimum = 0.999 - } - limit { - counter = "CLASS" - value = "MISSEDCOUNT" - // TODO: in the future the following value should become less than 10 - maximum = 0 - } - } - } - // we want to ignore some generated files in the domain folders - afterEvaluate { - excludedClassFilesForReport(classDirectories) - } -} +// Dynamic generation of jacoco test report generation-/coverage verification-tasks (per-module) +apply from: "jacoco.gradle" check.dependsOn jacocoTestCoverageVerification configurations { diff --git a/jacoco.gradle b/jacoco.gradle new file mode 100644 index 000000000000..9b57fb0ba653 --- /dev/null +++ b/jacoco.gradle @@ -0,0 +1,190 @@ +ext { + BasePath = "de/tum/cit/aet/artemis" + ModuleCoverageThresholds = [ + "assessment" : [ + "INSTRUCTION": 0.891, + "CLASS": 2 + ], + "athena" : [ + "INSTRUCTION": 0.872, + "CLASS": 2 + ], + "atlas" : [ + "INSTRUCTION": 0.832, + "CLASS": 4 + ], + "buildagent" : [ + "INSTRUCTION": 0.781, + "CLASS": 1 + ], + "communication": [ + "INSTRUCTION": 0.862, + "CLASS": 3 + ], + "core" : [ + "INSTRUCTION": 0.778, + "CLASS": 21 + ], + "exam" : [ + "INSTRUCTION": 0.902, + "CLASS": 1 + ], + "exercise" : [ + "INSTRUCTION": 0.929, + "CLASS": 0 + ], + "fileupload" : [ + "INSTRUCTION": 0.954, + "CLASS": 0 + ], + "iris" : [ + "INSTRUCTION": 0.712, + "CLASS": 22 + ], + "lecture" : [ + "INSTRUCTION": 0.910, + "CLASS": 0 + ], + "lti" : [ + "INSTRUCTION": 0.881, + "CLASS": 2 + ], + "modeling" : [ + "INSTRUCTION": 0.914, + "CLASS": 2 + ], + "plagiarism" : [ + "INSTRUCTION": 0.916, + "CLASS": 0 + ], + "programming" : [ + "INSTRUCTION": 0.874, + "CLASS": 14 + ], + "quiz" : [ + "INSTRUCTION": 0.796, + "CLASS": 6 + ], + "text" : [ + "INSTRUCTION": 0.952, + "CLASS": 0 + ], + "tutorialgroup": [ + "INSTRUCTION": 0.923, + "CLASS": 0 + ], + ] + // If no explicit modules defined -> generate reports and validate for each module + reportedModules = testedModules.size() == 0 + ? ModuleCoverageThresholds.collect {element -> element.key} + : testedModules as ArrayList +} + +jacocoTestReport { + // This task is just a wrapper to dynamically create tasks (per module) + enabled = false + + dependsOn reportedModules.collect { module -> registerJacocoReportTask(module as String, classDirectories) } +} + +jacocoTestCoverageVerification { + // This task is just a wrapper to dynamically create tasks (per module) + enabled = false + + dependsOn reportedModules.collect { module -> registerJacocoTestCoverageVerification(module as String, classDirectories) } +} + +/** + * Registers a JacocoReport task based on the provided parameters. + * + * @param moduleName The module name to include in the report. + * @param rootClassDir the compiled classes directory + * @return The configured JacocoReport task. + */ +private JacocoReport registerJacocoReportTask(String moduleName, ConfigurableFileCollection rootClassDir) { + def taskName = "jacocoCoverageReport-$moduleName" + + JacocoReport task = project.tasks.register(taskName, JacocoReport).get() + task.description = "Generates JaCoCo coverage report for $moduleName" + + prepareJacocoReportTask(task, moduleName, rootClassDir) + + task.reports { + xml.required = true + } + + return task +} + +/** + * Registers a JacocoCoverageVerification task based on the provided parameters. + * + * @param moduleName The module name to validate rules for. + * @param rootClassDir the compiled classes directory + * @return The configured JacocoCoverageVerification task. + */ +private JacocoCoverageVerification registerJacocoTestCoverageVerification(String moduleName, ConfigurableFileCollection rootClassDir) { + def taskName = "jacocoTestCoverageVerification-$moduleName" + + def thresholds = ModuleCoverageThresholds[moduleName] + if (thresholds == null) { + print "No coverage thresholds defined for module '$moduleName'. Skipping..." + return null + } + def coveredInstructionRatio = thresholds.INSTRUCTION + def missedClassCount = thresholds.CLASS + + JacocoCoverageVerification task = project.tasks.register(taskName, JacocoCoverageVerification).get() + task.description = "Validates JaCoCo coverage for vioalations for $moduleName" + + prepareJacocoReportTask(task, moduleName, rootClassDir) + + task.violationRules { + rule { + limit { + counter = "INSTRUCTION" + value = "COVEREDRATIO" + minimum = coveredInstructionRatio + } + limit { + counter = "CLASS" + value = "MISSEDCOUNT" + maximum = missedClassCount + } + } + } + + return task +} + +/** + * Prepares a Jacoco report task (report & verification) to match a specific + * @param task that is modified + * @param moduleName of the module. + * @param rootClassDir the compiled classes directory + */ +private void prepareJacocoReportTask(JacocoReportBase task, String moduleName, ConfigurableFileCollection rootClassDir) { + task.group = "Reporting" + task.executionData = project.fileTree("${project.layout.buildDirectory.get()}/jacoco") { + include "test.exec" + } + + def modulePath = "$BasePath/$moduleName/**/*.class" + task.sourceDirectories.setFrom(project.files("src/main/java/$modulePath")) + task.classDirectories.setFrom( + files(rootClassDir.files.collect { classDir -> + project.fileTree(classDir) { + includes=[modulePath] + // we want to ignore some generated files in the domain folders + excludes=[ + "**/$BasePath/**/domain/**/*_*", + "**/$BasePath/core/config/migration/entries/**", + "**/gradle-wrapper.jar/**" + ] + } + } + ) + ) + + task.dependsOn compileTestJava +} From 2e4d796cb789e7044319e8df1140a4709d390fe7 Mon Sep 17 00:00:00 2001 From: Ole Vester Date: Tue, 10 Dec 2024 18:45:10 +0100 Subject: [PATCH 09/69] Only test changed modules --- .github/workflows/test.yml | 17 ++++++++++++++++- changed-modules.sh | 16 ++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) create mode 100755 changed-modules.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2239b2bb3a71..c9b0fad9173c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -63,7 +63,22 @@ jobs: ${{ env.java }} cache: 'gradle' - name: Java Tests - run: set -o pipefail && ./gradlew --console=plain test jacocoTestReport -x webapp jacocoTestCoverageVerification | tee tests.log + run: | + set -o pipefail + + chmod +x ./changed-modules.sh + CHANGED_MODULES=$(./changed-modules.sh "${{ github.base_ref }}") + + # Restrict executed tests to changed modules if there is diff between this and the base branch + if [ -n "${CHANGED_MODULES}" ]; then + # Always execute ArchitectureTests if at least one module changed + CHANGED_MODULES+=("ArchitectureTest") + + IFS=, + TEST_MODULE_TAGS=$(echo "-DtestTags=${CHANGED_MODULES[*]}") + fi + + ./gradlew --console=plain test jacocoTestReport -x webapp jacocoTestCoverageVerification $TEST_MODULE_TAGS | tee tests.log - name: Print failed tests if: failure() run: grep "Test >.* FAILED\$" tests.log || echo "No failed tests." diff --git a/changed-modules.sh b/changed-modules.sh new file mode 100755 index 000000000000..60ebf84100ef --- /dev/null +++ b/changed-modules.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +BRANCH_TO_COMPARE="$1" + +MODULE_BASE_PATH="src/main/java/de/tum/cit/aet/artemis" +MODULES=$(find "$MODULE_BASE_PATH" -maxdepth 1 -mindepth 1 -type d -exec basename {} \;) +CHANGED_MODULES=() + +for MODULE in $MODULES; do + if git diff "$BRANCH_TO_COMPARE" --name-only | grep -q "^$MODULE_BASE_PATH/$MODULE/"; then + CHANGED_MODULES+=("$MODULE") + fi +done + +IFS=, +echo "${CHANGED_MODULES[*]}" From 739f60a8a1fb4fc51c10dcf701f3dfecc54bc100 Mon Sep 17 00:00:00 2001 From: Ole Vester Date: Tue, 10 Dec 2024 18:45:25 +0100 Subject: [PATCH 10/69] Parse and append per-module coverage results --- .github/workflows/test.yml | 5 +- jacoco-parser.py | 98 ++++++++++++++++++++++---------------- 2 files changed, 59 insertions(+), 44 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c9b0fad9173c..2d187354ef65 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -114,8 +114,9 @@ jobs: - name: Append Per-Module Coverage to Job Summary if: success() || failure() run: | - python3 ./jacoco-parser.py >> $GITHUB_STEP_SUMMARY - + AGGREGATED_REPORT_FILE=./coverage_report.md + python3 ./jacoco-parser.py build/reports/jacoco $AGGREGATED_REPORT_FILE + cat $AGGREGATED_REPORT_FILE > $GITHUB_STEP_SUMMARY server-tests-mysql: needs: [ server-tests ] diff --git a/jacoco-parser.py b/jacoco-parser.py index 08f3998650e2..01f48131cbe7 100644 --- a/jacoco-parser.py +++ b/jacoco-parser.py @@ -1,44 +1,58 @@ +import os import xml.etree.ElementTree as ET +import argparse -def parse_jacoco_report(xml_file, package_path): - """ - Parse a JaCoCo XML report to extract coverage metrics for a specific package. - - :param xml_file: Path to the JaCoCo XML report. - :param package_path: The package path to search for. - :return: A dictionary containing coverage metrics for the package. - """ - tree = ET.parse(xml_file) - root = tree.getroot() - - # Search for the specific package - for package in root.findall(".//package"): - if package.get('name') == package_path: - coverage_metrics = {} - # Extract coverage metrics from elements - for counter in package.findall("counter"): - metric = counter.get("type") - covered = int(counter.get("covered")) - missed = int(counter.get("missed")) - total = covered + missed - coverage_percentage = (covered / total) * 100 if total > 0 else 0 - coverage_metrics[metric] = { - "covered": covered, - "missed": missed, - "total": total, - "coverage_percentage": coverage_percentage - } - return coverage_metrics - - return None - -xml_file_path = "build/reports/jacoco/test/jacocoTestReport.xml" -package = "de/tum/cit/aet/artemis/buildagent" -metrics = parse_jacoco_report(xml_file_path, package) - -if metrics: - print(f"Coverage metrics for package '{package}':") - for metric, data in metrics.items(): - print(f"{metric}: {data['coverage_percentage']:.2f}% (Covered: {data['covered']}, Missed: {data['missed']}, Total: {data['total']})") -else: - print(f"Package '{package}' not found in the report.") +def process_reports(input_directory, output_file): + results = [] + + for module_folder in os.listdir(input_directory): + if module_folder.startswith("jacocoCoverageReport-"): + module_name = module_folder[len("jacocoCoverageReport-"):] + module_path = os.path.join(input_directory, module_folder) + + if os.path.isdir(module_path): + report_file = os.path.join(module_path, f"jacocoCoverageReport-{module_name}.xml") + + if os.path.exists(report_file): + try: + tree = ET.parse(report_file) + root = tree.getroot() + + instruction_counter = root.find("./counter[@type='INSTRUCTION']") + class_counter = root.find("./counter[@type='CLASS']") + + instruction_covered = int(instruction_counter.get('covered', 0)) + instruction_missed = int(instruction_counter.get('missed', 0)) + total_instructions = instruction_covered + instruction_missed + instruction_coverage = (instruction_covered / total_instructions * 100) if total_instructions > 0 else 0.0 + + missed_classes = int(class_counter.get('missed', 0)) + + results.append({ + "module": module_name, + "instruction_coverage": instruction_coverage, + "missed_classes": missed_classes + }) + except Exception as e: + print(f"Error processing {module_name}: {e}") + + results = sorted(results, key=lambda x: x['module']) + + with open(output_file, "w") as f: + f.write("## Coverage Results\n\n") + f.write("| Module Name | Instruction Coverage (%) | Missed Classes |\n") + f.write("|-------------|---------------------------|----------------|\n") + for result in results: + f.write(f"| {result['module']} | {result['instruction_coverage']:.2f} | {result['missed_classes']} |\n") + + print(f"Coverage results written to {output_file}") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Process JaCoCo coverage reports.") + parser.add_argument("input_directory", type=str, help="Root directory containing JaCoCo coverage reports") + parser.add_argument("output_file", type=str, help="Output file to save the coverage results") + + args = parser.parse_args() + + process_reports(args.input_directory, args.output_file) From fd39101486912d9dde74a8ca8510195e5249063f Mon Sep 17 00:00:00 2001 From: Ole Vester Date: Tue, 10 Dec 2024 18:47:42 +0100 Subject: [PATCH 11/69] Temporarily remove codacy coverage report upload --- .github/workflows/test.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2d187354ef65..cb84e30ca82b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -82,12 +82,6 @@ jobs: - name: Print failed tests if: failure() run: grep "Test >.* FAILED\$" tests.log || echo "No failed tests." - - name: "Codacy: Report coverage" - uses: codacy/codacy-coverage-reporter-action@master - with: - project-token: ${{ secrets.CODACY_PROJECT_TOKEN }} - coverage-reports: build/reports/jacoco/test/jacocoTestReport.xml - if: (github.event.pull_request.head.repo.full_name == github.event.pull_request.base.repo.full_name) && (success() || failure()) && github.event.pull_request.user.login != 'dependabot[bot]' - name: Annotate Server Test Results uses: ashley-taylor/junit-report-annotations-action@f9c1a5cbe28479439f82b80a5402a6d3aa1990ac if: always() && github.event.pull_request.user.login != 'dependabot[bot]' From 5a409bedadbbf424222eebd220326e2c21795017 Mon Sep 17 00:00:00 2001 From: Ole Vester Date: Tue, 10 Dec 2024 18:47:54 +0100 Subject: [PATCH 12/69] Upload whole build coverage report directory --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index cb84e30ca82b..7834ba28cfc0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -104,7 +104,7 @@ jobs: uses: actions/upload-artifact@v4 with: name: Coverage Report Server Tests - path: build/reports/jacoco/test/html/ + path: build/reports/jacoco/ - name: Append Per-Module Coverage to Job Summary if: success() || failure() run: | From 9e7d3cf7118b11227d482103efdca3d76bc176b5 Mon Sep 17 00:00:00 2001 From: Ole Vester Date: Tue, 10 Dec 2024 21:21:25 +0100 Subject: [PATCH 13/69] Temporarily fix tested modules --- .github/workflows/test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7834ba28cfc0..5f4ec51f79dc 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -78,6 +78,8 @@ jobs: TEST_MODULE_TAGS=$(echo "-DtestTags=${CHANGED_MODULES[*]}") fi + TEST_MODULE_TAGS=$(echo "-DtestTags=atlas,buildagent,fileupload,lti") + ./gradlew --console=plain test jacocoTestReport -x webapp jacocoTestCoverageVerification $TEST_MODULE_TAGS | tee tests.log - name: Print failed tests if: failure() From 5083184f1fe24f20c3e0405b7acbc3818cc0ddef Mon Sep 17 00:00:00 2001 From: Ole Vester Date: Mon, 16 Dec 2024 16:59:01 +0100 Subject: [PATCH 14/69] temporarily add ArchitectureTest to have at least one test --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5f4ec51f79dc..07d340d2a44b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -78,7 +78,7 @@ jobs: TEST_MODULE_TAGS=$(echo "-DtestTags=${CHANGED_MODULES[*]}") fi - TEST_MODULE_TAGS=$(echo "-DtestTags=atlas,buildagent,fileupload,lti") + TEST_MODULE_TAGS=$(echo "-DtestTags=atlas,buildagent,fileupload,lti,ArchitectureTest") ./gradlew --console=plain test jacocoTestReport -x webapp jacocoTestCoverageVerification $TEST_MODULE_TAGS | tee tests.log - name: Print failed tests From 27b68ba69749bf356e05e073636e54f303f14e5a Mon Sep 17 00:00:00 2001 From: Ole Vester Date: Mon, 16 Dec 2024 17:03:55 +0100 Subject: [PATCH 15/69] fix GH variable name of default branch --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 07d340d2a44b..32b1ad855fc2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -67,7 +67,7 @@ jobs: set -o pipefail chmod +x ./changed-modules.sh - CHANGED_MODULES=$(./changed-modules.sh "${{ github.base_ref }}") + CHANGED_MODULES=$(./changed-modules.sh "${{ github.event.repository.default_branch }}") # Restrict executed tests to changed modules if there is diff between this and the base branch if [ -n "${CHANGED_MODULES}" ]; then From d2a6f3369a16dd066611d1a915509e3076b2bbfe Mon Sep 17 00:00:00 2001 From: Ole Vester Date: Mon, 16 Dec 2024 17:13:50 +0100 Subject: [PATCH 16/69] clone default branch if not running for the base branch --- .github/workflows/test.yml | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 32b1ad855fc2..9cf5333355ca 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -66,21 +66,29 @@ jobs: run: | set -o pipefail - chmod +x ./changed-modules.sh - CHANGED_MODULES=$(./changed-modules.sh "${{ github.event.repository.default_branch }}") + DEFAULT_BRANCH="${{ github.event.repository.default_branch }}" + CURRENT_BRANCH="${{ github.ref_name }}" - # Restrict executed tests to changed modules if there is diff between this and the base branch - if [ -n "${CHANGED_MODULES}" ]; then - # Always execute ArchitectureTests if at least one module changed - CHANGED_MODULES+=("ArchitectureTest") + if [[ "$DEFAULT_BRANCH" != "$CURRENT_BRANCH" ]]; then + # Explicitly fetch as the clone action only clones the current branch + git fetch origin "$DEFAULT_BRANCH" - IFS=, - TEST_MODULE_TAGS=$(echo "-DtestTags=${CHANGED_MODULES[*]}") + chmod +x ./changed-modules.sh + CHANGED_MODULES=$(./changed-modules.sh "$DEFAULT_BRANCH") + + # Restrict executed tests to changed modules if there is diff between this and the base branch + if [ -n "${CHANGED_MODULES}" ]; then + # Always execute ArchitectureTests if at least one module changed + CHANGED_MODULES+=("ArchitectureTest") + + IFS=, + TEST_MODULE_TAGS=$(echo "-DtestTags=${CHANGED_MODULES[*]}") + fi fi TEST_MODULE_TAGS=$(echo "-DtestTags=atlas,buildagent,fileupload,lti,ArchitectureTest") - ./gradlew --console=plain test jacocoTestReport -x webapp jacocoTestCoverageVerification $TEST_MODULE_TAGS | tee tests.log + ./gradlew --console=plain test jacocoTestReport -x webapp jacocoTestCoverageVerification $TEST_MODULE_TAGS | tee tests.log - name: Print failed tests if: failure() run: grep "Test >.* FAILED\$" tests.log || echo "No failed tests." From 5cbeac41358f4c8671ff7c7ec1a542853fca497c Mon Sep 17 00:00:00 2001 From: Ole Vester Date: Mon, 16 Dec 2024 17:16:10 +0100 Subject: [PATCH 17/69] prefix branch name to compare with "origin/" --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9cf5333355ca..ce66a3526d4f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -74,7 +74,7 @@ jobs: git fetch origin "$DEFAULT_BRANCH" chmod +x ./changed-modules.sh - CHANGED_MODULES=$(./changed-modules.sh "$DEFAULT_BRANCH") + CHANGED_MODULES=$(./changed-modules.sh "origin/$DEFAULT_BRANCH") # Restrict executed tests to changed modules if there is diff between this and the base branch if [ -n "${CHANGED_MODULES}" ]; then From f7e1408d7582db8b905bf5a86107b6a53a542c3e Mon Sep 17 00:00:00 2001 From: Ole Vester Date: Mon, 16 Dec 2024 17:36:37 +0100 Subject: [PATCH 18/69] remove null tasks from dependant list --- jacoco.gradle | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/jacoco.gradle b/jacoco.gradle index 9b57fb0ba653..8cd6edd75bc9 100644 --- a/jacoco.gradle +++ b/jacoco.gradle @@ -84,14 +84,18 @@ jacocoTestReport { // This task is just a wrapper to dynamically create tasks (per module) enabled = false - dependsOn reportedModules.collect { module -> registerJacocoReportTask(module as String, classDirectories) } + dependsOn reportedModules + .collect { module -> registerJacocoReportTask(module as String, classDirectories) } + .findAll { task -> task != null} } jacocoTestCoverageVerification { // This task is just a wrapper to dynamically create tasks (per module) enabled = false - dependsOn reportedModules.collect { module -> registerJacocoTestCoverageVerification(module as String, classDirectories) } + dependsOn reportedModules + .collect { module -> registerJacocoTestCoverageVerification(module as String, classDirectories) } + .findAll { task -> task != null} } /** From 49c151f678f2b3bd2aa21b753938718293c9718a Mon Sep 17 00:00:00 2001 From: Ole Vester Date: Mon, 16 Dec 2024 17:37:07 +0100 Subject: [PATCH 19/69] correctly type testedModules as ArrayList --- build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index 76ddb3a090f4..2e5cf10fa35d 100644 --- a/build.gradle +++ b/build.gradle @@ -121,7 +121,7 @@ modernizer { // Allow using in jacoco.gradle ext { testTags = System.getProperty("testTags") - testedModules = !testTags ? [] : testTags.split(",") as ArrayList + testedModules = !testTags ? new String[]{} : testTags.split(",") as String[] } // Execute the test cases: ./gradlew test @@ -129,7 +129,7 @@ ext { test { if (testedModules.size() > 0) { useJUnitPlatform { - includeTags testedModules.join(",") + includeTags testedModules } } else { useJUnitPlatform() From e3def21cdb5985e357e3b347e06bc79674bba91e Mon Sep 17 00:00:00 2001 From: Ole Vester Date: Mon, 16 Dec 2024 17:38:11 +0100 Subject: [PATCH 20/69] (temporarily) directly set TEST_MODULE_TAGS instead of evaluating with echo --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ce66a3526d4f..a01d386b88b8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -86,7 +86,7 @@ jobs: fi fi - TEST_MODULE_TAGS=$(echo "-DtestTags=atlas,buildagent,fileupload,lti,ArchitectureTest") + TEST_MODULE_TAGS="-DtestTags=atlas,buildagent,fileupload,lti,ArchitectureTest" ./gradlew --console=plain test jacocoTestReport -x webapp jacocoTestCoverageVerification $TEST_MODULE_TAGS | tee tests.log - name: Print failed tests From 43b65eb804378e8a095937f57c18a815dbd1f8f7 Mon Sep 17 00:00:00 2001 From: Ole Vester Date: Mon, 16 Dec 2024 17:38:22 +0100 Subject: [PATCH 21/69] Add logging on executed test tags --- .github/workflows/test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a01d386b88b8..753af40d9618 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -83,6 +83,8 @@ jobs: IFS=, TEST_MODULE_TAGS=$(echo "-DtestTags=${CHANGED_MODULES[*]}") + + echo "Executing tests with JUnit tags: $TEST_MODULE_TAGS" fi fi From 476c792588465576f63fee0414319a76ea223e23 Mon Sep 17 00:00:00 2001 From: Ole Vester Date: Mon, 16 Dec 2024 18:01:04 +0100 Subject: [PATCH 22/69] quote TEST_MODULE_TAGS --- .github/workflows/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 753af40d9618..867291a2b725 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -84,13 +84,13 @@ jobs: IFS=, TEST_MODULE_TAGS=$(echo "-DtestTags=${CHANGED_MODULES[*]}") - echo "Executing tests with JUnit tags: $TEST_MODULE_TAGS" + echo "Executing tests with JUnit tags: $CHANGED_MODULES" fi fi TEST_MODULE_TAGS="-DtestTags=atlas,buildagent,fileupload,lti,ArchitectureTest" - ./gradlew --console=plain test jacocoTestReport -x webapp jacocoTestCoverageVerification $TEST_MODULE_TAGS | tee tests.log + ./gradlew --console=plain test jacocoTestReport -x webapp jacocoTestCoverageVerification "$TEST_MODULE_TAGS" | tee tests.log - name: Print failed tests if: failure() run: grep "Test >.* FAILED\$" tests.log || echo "No failed tests." From 786d172ff12c3c9dcb9c8ea59e0fbe571c2968f6 Mon Sep 17 00:00:00 2001 From: Ole Vester Date: Mon, 16 Dec 2024 18:01:26 +0100 Subject: [PATCH 23/69] log gradle command --- .github/workflows/test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 867291a2b725..829c30d1d3f1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -90,6 +90,8 @@ jobs: TEST_MODULE_TAGS="-DtestTags=atlas,buildagent,fileupload,lti,ArchitectureTest" + echo "./gradlew --console=plain test jacocoTestReport -x webapp jacocoTestCoverageVerification $TEST_MODULE_TAGS" + ./gradlew --console=plain test jacocoTestReport -x webapp jacocoTestCoverageVerification "$TEST_MODULE_TAGS" | tee tests.log - name: Print failed tests if: failure() From b31f0c80d21991d308c5cc70586021461132076a Mon Sep 17 00:00:00 2001 From: Ole Vester Date: Mon, 16 Dec 2024 18:03:03 +0100 Subject: [PATCH 24/69] Revert "log gradle command" This reverts commit 786d172ff12c3c9dcb9c8ea59e0fbe571c2968f6. --- .github/workflows/test.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 829c30d1d3f1..867291a2b725 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -90,8 +90,6 @@ jobs: TEST_MODULE_TAGS="-DtestTags=atlas,buildagent,fileupload,lti,ArchitectureTest" - echo "./gradlew --console=plain test jacocoTestReport -x webapp jacocoTestCoverageVerification $TEST_MODULE_TAGS" - ./gradlew --console=plain test jacocoTestReport -x webapp jacocoTestCoverageVerification "$TEST_MODULE_TAGS" | tee tests.log - name: Print failed tests if: failure() From 2fd593654403be4284e423ec29a659128de2205c Mon Sep 17 00:00:00 2001 From: Ole Vester Date: Mon, 16 Dec 2024 18:42:04 +0100 Subject: [PATCH 25/69] split into includeModules and includeTags --- .github/workflows/test.yml | 11 ++++------- build.gradle | 30 ++++++++++++++++++------------ jacoco.gradle | 13 +++++++++---- 3 files changed, 31 insertions(+), 23 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 867291a2b725..42074ea795b4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -78,17 +78,14 @@ jobs: # Restrict executed tests to changed modules if there is diff between this and the base branch if [ -n "${CHANGED_MODULES}" ]; then - # Always execute ArchitectureTests if at least one module changed - CHANGED_MODULES+=("ArchitectureTest") - IFS=, - TEST_MODULE_TAGS=$(echo "-DtestTags=${CHANGED_MODULES[*]}") + TEST_MODULE_TAGS=$(echo "-DincludeModules=${CHANGED_MODULES[*]}") - echo "Executing tests with JUnit tags: $CHANGED_MODULES" + echo "Executing tests for modules: $CHANGED_MODULES" fi fi - TEST_MODULE_TAGS="-DtestTags=atlas,buildagent,fileupload,lti,ArchitectureTest" + TEST_MODULE_TAGS="-DincludeModules=atlas,buildagent,fileupload,lti" ./gradlew --console=plain test jacocoTestReport -x webapp jacocoTestCoverageVerification "$TEST_MODULE_TAGS" | tee tests.log - name: Print failed tests @@ -255,7 +252,7 @@ jobs: run: ./gradlew checkstyleMain -x webapp if: success() || failure() - name: Java Architecture Tests - run: ./gradlew test -DtestModules='ArchitectureTest' -x webapp + run: ./gradlew test -DincludeTags='ArchitectureTest' -x webapp if: success() || failure() - name: Test Report uses: dorny/test-reporter@v1 diff --git a/build.gradle b/build.gradle index 2e5cf10fa35d..66aff7d5a3c5 100644 --- a/build.gradle +++ b/build.gradle @@ -120,20 +120,31 @@ modernizer { // Allow using in jacoco.gradle ext { - testTags = System.getProperty("testTags") - testedModules = !testTags ? new String[]{} : testTags.split(",") as String[] + includedTestTags = System.getProperty("includeTags") + includedTags = !includedTestTags ? new String[]{} : includedTestTags.split(",") as String[] + includedModulesTag = System.getProperty("includeModules") + includedModules = !includedModulesTag ? new String[]{} : includedModulesTag.split(",") as String[] + + runAllTests = includedTags.size() == 0 && includedModules.size() == 0 } // Execute the test cases: ./gradlew test -// Execute only architecture tests: ./gradlew test -DtestTags="ArchitectureTest" +// Execute only architecture tests: ./gradlew test -DincludeTags="ArchitectureTest" +// Execute tests only for specific modules: ./gradlew test -DincludeModules="atlas". Using this flag, "includeTags" will be ignored. test { - if (testedModules.size() > 0) { - useJUnitPlatform { - includeTags testedModules + if (runAllTests) { + useJUnitPlatform() + exclude "**/*IT*", "**/*IntTest*" + } else if (includedModules.size() == 0) { + useJUnitPlatform() { + includeTags includedTags } } else { useJUnitPlatform() - exclude "**/*IT*", "**/*IntTest*" + // Always execute "shared"-folder when executing module-specifc tests + includedModules += "shared" + def moduleBasePaths = includedModules.collect { val -> "de/tum/cit/aet/artemis/$val/**"} + include(moduleBasePaths) } testLogging { @@ -150,17 +161,12 @@ tasks.register("testReport", TestReport) { testResults.from(test) } -jacoco { - toolVersion = "0.8.12" -} - jar { enabled = false } // Dynamic generation of jacoco test report generation-/coverage verification-tasks (per-module) apply from: "jacoco.gradle" -check.dependsOn jacocoTestCoverageVerification configurations { providedRuntime diff --git a/jacoco.gradle b/jacoco.gradle index 8cd6edd75bc9..59edbec00798 100644 --- a/jacoco.gradle +++ b/jacoco.gradle @@ -11,7 +11,7 @@ ext { ], "atlas" : [ "INSTRUCTION": 0.832, - "CLASS": 4 + "CLASS": 13 ], "buildagent" : [ "INSTRUCTION": 0.781, @@ -75,9 +75,13 @@ ext { ], ] // If no explicit modules defined -> generate reports and validate for each module - reportedModules = testedModules.size() == 0 + reportedModules = includedModules.size() == 0 ? ModuleCoverageThresholds.collect {element -> element.key} - : testedModules as ArrayList + : includedModules as ArrayList +} + +jacoco { + toolVersion = "0.8.12" } jacocoTestReport { @@ -97,6 +101,7 @@ jacocoTestCoverageVerification { .collect { module -> registerJacocoTestCoverageVerification(module as String, classDirectories) } .findAll { task -> task != null} } +check.dependsOn jacocoTestCoverageVerification /** * Registers a JacocoReport task based on the provided parameters. @@ -190,5 +195,5 @@ private void prepareJacocoReportTask(JacocoReportBase task, String moduleName, C ) ) - task.dependsOn compileTestJava + task.dependsOn test } From f394c7061d2187e789d0c6d61c36201211c11a36 Mon Sep 17 00:00:00 2001 From: Ole Vester Date: Mon, 16 Dec 2024 18:54:53 +0100 Subject: [PATCH 26/69] change buildagent thresholds to match module-only results --- jacoco.gradle | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/jacoco.gradle b/jacoco.gradle index 59edbec00798..fa981ded194a 100644 --- a/jacoco.gradle +++ b/jacoco.gradle @@ -1,5 +1,6 @@ ext { BasePath = "de/tum/cit/aet/artemis" + // Thresholds when executing each module on its own ModuleCoverageThresholds = [ "assessment" : [ "INSTRUCTION": 0.891, @@ -14,8 +15,8 @@ ext { "CLASS": 13 ], "buildagent" : [ - "INSTRUCTION": 0.781, - "CLASS": 1 + "INSTRUCTION": 0.304, + "CLASS": 13 ], "communication": [ "INSTRUCTION": 0.862, From e02259276db5fac9f9ae013c9ec685a0f2d33cfa Mon Sep 17 00:00:00 2001 From: Ole Vester Date: Mon, 16 Dec 2024 19:07:01 +0100 Subject: [PATCH 27/69] Move basePath from jacoco.gradle to build.gradle --- build.gradle | 3 ++- jacoco.gradle | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index 66aff7d5a3c5..005363706406 100644 --- a/build.gradle +++ b/build.gradle @@ -126,6 +126,7 @@ ext { includedModules = !includedModulesTag ? new String[]{} : includedModulesTag.split(",") as String[] runAllTests = includedTags.size() == 0 && includedModules.size() == 0 + BasePath = "de/tum/cit/aet/artemis" } // Execute the test cases: ./gradlew test @@ -143,7 +144,7 @@ test { useJUnitPlatform() // Always execute "shared"-folder when executing module-specifc tests includedModules += "shared" - def moduleBasePaths = includedModules.collect { val -> "de/tum/cit/aet/artemis/$val/**"} + def moduleBasePaths = includedModules.collect { val -> "$BasePath/$val/**"} include(moduleBasePaths) } diff --git a/jacoco.gradle b/jacoco.gradle index fa981ded194a..41e18297c493 100644 --- a/jacoco.gradle +++ b/jacoco.gradle @@ -1,5 +1,5 @@ ext { - BasePath = "de/tum/cit/aet/artemis" + // Thresholds when executing each module on its own ModuleCoverageThresholds = [ "assessment" : [ From 7dd97307434fc2a7d05e1cdbf96374416da6afc3 Mon Sep 17 00:00:00 2001 From: Ole Vester Date: Mon, 16 Dec 2024 19:07:22 +0100 Subject: [PATCH 28/69] ensure module-specific jacoco task runs after "rootTask" --- jacoco.gradle | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/jacoco.gradle b/jacoco.gradle index 41e18297c493..2d7f5343ffb1 100644 --- a/jacoco.gradle +++ b/jacoco.gradle @@ -89,8 +89,8 @@ jacocoTestReport { // This task is just a wrapper to dynamically create tasks (per module) enabled = false - dependsOn reportedModules - .collect { module -> registerJacocoReportTask(module as String, classDirectories) } + reportedModules + .collect { module -> registerJacocoReportTask(module as String, jacocoTestReport) } .findAll { task -> task != null} } @@ -98,8 +98,8 @@ jacocoTestCoverageVerification { // This task is just a wrapper to dynamically create tasks (per module) enabled = false - dependsOn reportedModules - .collect { module -> registerJacocoTestCoverageVerification(module as String, classDirectories) } + reportedModules + .collect { module -> registerJacocoTestCoverageVerification(module as String, jacocoTestCoverageVerification) } .findAll { task -> task != null} } check.dependsOn jacocoTestCoverageVerification @@ -108,16 +108,16 @@ check.dependsOn jacocoTestCoverageVerification * Registers a JacocoReport task based on the provided parameters. * * @param moduleName The module name to include in the report. - * @param rootClassDir the compiled classes directory + * @param rootTask The root JacocoReport root task. * @return The configured JacocoReport task. */ -private JacocoReport registerJacocoReportTask(String moduleName, ConfigurableFileCollection rootClassDir) { +private JacocoReport registerJacocoReportTask(String moduleName, JacocoReport rootTask) { def taskName = "jacocoCoverageReport-$moduleName" JacocoReport task = project.tasks.register(taskName, JacocoReport).get() task.description = "Generates JaCoCo coverage report for $moduleName" - prepareJacocoReportTask(task, moduleName, rootClassDir) + prepareJacocoReportTask(task, moduleName, rootTask) task.reports { xml.required = true @@ -130,10 +130,10 @@ private JacocoReport registerJacocoReportTask(String moduleName, ConfigurableFil * Registers a JacocoCoverageVerification task based on the provided parameters. * * @param moduleName The module name to validate rules for. - * @param rootClassDir the compiled classes directory + * @param rootTask The root JacocoCoverageVerification task. * @return The configured JacocoCoverageVerification task. */ -private JacocoCoverageVerification registerJacocoTestCoverageVerification(String moduleName, ConfigurableFileCollection rootClassDir) { +private JacocoCoverageVerification registerJacocoTestCoverageVerification(String moduleName, JacocoCoverageVerification rootTask) { def taskName = "jacocoTestCoverageVerification-$moduleName" def thresholds = ModuleCoverageThresholds[moduleName] @@ -147,7 +147,7 @@ private JacocoCoverageVerification registerJacocoTestCoverageVerification(String JacocoCoverageVerification task = project.tasks.register(taskName, JacocoCoverageVerification).get() task.description = "Validates JaCoCo coverage for vioalations for $moduleName" - prepareJacocoReportTask(task, moduleName, rootClassDir) + prepareJacocoReportTask(task, moduleName, rootTask) task.violationRules { rule { @@ -171,9 +171,9 @@ private JacocoCoverageVerification registerJacocoTestCoverageVerification(String * Prepares a Jacoco report task (report & verification) to match a specific * @param task that is modified * @param moduleName of the module. - * @param rootClassDir the compiled classes directory + * @param rootTask the JacocoReportBase root task */ -private void prepareJacocoReportTask(JacocoReportBase task, String moduleName, ConfigurableFileCollection rootClassDir) { +private void prepareJacocoReportTask(JacocoReportBase task, String moduleName, JacocoReportBase rootTask) { task.group = "Reporting" task.executionData = project.fileTree("${project.layout.buildDirectory.get()}/jacoco") { include "test.exec" @@ -182,7 +182,7 @@ private void prepareJacocoReportTask(JacocoReportBase task, String moduleName, C def modulePath = "$BasePath/$moduleName/**/*.class" task.sourceDirectories.setFrom(project.files("src/main/java/$modulePath")) task.classDirectories.setFrom( - files(rootClassDir.files.collect { classDir -> + files(rootTask.classDirectories.files.collect { classDir -> project.fileTree(classDir) { includes=[modulePath] // we want to ignore some generated files in the domain folders @@ -196,5 +196,5 @@ private void prepareJacocoReportTask(JacocoReportBase task, String moduleName, C ) ) - task.dependsOn test + task.mustRunAfter(rootTask) } From 98227e817275ca4749bb5cd90a3b9e43c57a1f97 Mon Sep 17 00:00:00 2001 From: Ole Vester Date: Mon, 16 Dec 2024 19:13:20 +0100 Subject: [PATCH 29/69] add comment on including junit tags logic --- build.gradle | 1 + 1 file changed, 1 insertion(+) diff --git a/build.gradle b/build.gradle index 005363706406..7697e4b39558 100644 --- a/build.gradle +++ b/build.gradle @@ -137,6 +137,7 @@ test { useJUnitPlatform() exclude "**/*IT*", "**/*IntTest*" } else if (includedModules.size() == 0) { + // not running all tests, but not module-specific ones -> use tags useJUnitPlatform() { includeTags includedTags } From c8608493bd1767314a18bac15081294a3d8cdec8 Mon Sep 17 00:00:00 2001 From: Ole Vester Date: Mon, 16 Dec 2024 19:13:57 +0100 Subject: [PATCH 30/69] create jacoco report for full Artemis bundle also when testing specific modules --- jacoco.gradle | 3 --- 1 file changed, 3 deletions(-) diff --git a/jacoco.gradle b/jacoco.gradle index 2d7f5343ffb1..5be36d35b497 100644 --- a/jacoco.gradle +++ b/jacoco.gradle @@ -86,9 +86,6 @@ jacoco { } jacocoTestReport { - // This task is just a wrapper to dynamically create tasks (per module) - enabled = false - reportedModules .collect { module -> registerJacocoReportTask(module as String, jacocoTestReport) } .findAll { task -> task != null} From 6453837b6ac07741434a1725b5bced9a1ed5c44c Mon Sep 17 00:00:00 2001 From: Ole Vester Date: Mon, 16 Dec 2024 19:20:58 +0100 Subject: [PATCH 31/69] remove debugging step from pipeline --- .github/workflows/test.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 42074ea795b4..96f961463748 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -50,8 +50,6 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 60 steps: - - name: Debug - Appending anything to job summary - run: echo 'Hello World' >> $GITHUB_STEP_SUMMARY - uses: actions/checkout@v4 - name: Setup Java uses: actions/setup-java@v4 From 9c4348f1b40e748795a42501b2b5491f8411e1ae Mon Sep 17 00:00:00 2001 From: Ole Vester Date: Mon, 16 Dec 2024 19:22:55 +0100 Subject: [PATCH 32/69] adapt expected number of server starts --- supporting_scripts/extract_number_of_server_starts.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/supporting_scripts/extract_number_of_server_starts.sh b/supporting_scripts/extract_number_of_server_starts.sh index 80f07e848207..661c6e2d4081 100644 --- a/supporting_scripts/extract_number_of_server_starts.sh +++ b/supporting_scripts/extract_number_of_server_starts.sh @@ -9,8 +9,8 @@ then exit 1 fi -if [[ $numberOfStarts -ne 4 ]] +if [[ $numberOfStarts -le 4 ]] then - echo "The number of Server Starts should be equal to 4! Please adapt this check if the change is intended or try to fix the underlying issue causing a different number of server starts!" + echo "The number of Server Starts should be lower than/equals 4! Please adapt this check if the change is intended or try to fix the underlying issue causing a different number of server starts!" exit 1 fi From 42b9f88eebb18475be1ac58d8e4f5f530a8a078c Mon Sep 17 00:00:00 2001 From: Ole Vester Date: Mon, 16 Dec 2024 19:33:16 +0100 Subject: [PATCH 33/69] use finalizedBy instead of mustRunAfter --- jacoco.gradle | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/jacoco.gradle b/jacoco.gradle index 5be36d35b497..971d37ddce1b 100644 --- a/jacoco.gradle +++ b/jacoco.gradle @@ -86,7 +86,7 @@ jacoco { } jacocoTestReport { - reportedModules + finalizedBy reportedModules .collect { module -> registerJacocoReportTask(module as String, jacocoTestReport) } .findAll { task -> task != null} } @@ -95,7 +95,7 @@ jacocoTestCoverageVerification { // This task is just a wrapper to dynamically create tasks (per module) enabled = false - reportedModules + finalizedBy reportedModules .collect { module -> registerJacocoTestCoverageVerification(module as String, jacocoTestCoverageVerification) } .findAll { task -> task != null} } @@ -192,6 +192,4 @@ private void prepareJacocoReportTask(JacocoReportBase task, String moduleName, J } ) ) - - task.mustRunAfter(rootTask) } From 39f568d61cf758427efa3b15259ac7b62f080c02 Mon Sep 17 00:00:00 2001 From: Ole Vester Date: Mon, 16 Dec 2024 19:37:22 +0100 Subject: [PATCH 34/69] make report paths explicit --- jacoco-parser.py | 59 ++++++++++++++++++++++++------------------------ jacoco.gradle | 3 +++ 2 files changed, 33 insertions(+), 29 deletions(-) diff --git a/jacoco-parser.py b/jacoco-parser.py index 01f48131cbe7..818b19312a06 100644 --- a/jacoco-parser.py +++ b/jacoco-parser.py @@ -6,35 +6,36 @@ def process_reports(input_directory, output_file): results = [] for module_folder in os.listdir(input_directory): - if module_folder.startswith("jacocoCoverageReport-"): - module_name = module_folder[len("jacocoCoverageReport-"):] - module_path = os.path.join(input_directory, module_folder) - - if os.path.isdir(module_path): - report_file = os.path.join(module_path, f"jacocoCoverageReport-{module_name}.xml") - - if os.path.exists(report_file): - try: - tree = ET.parse(report_file) - root = tree.getroot() - - instruction_counter = root.find("./counter[@type='INSTRUCTION']") - class_counter = root.find("./counter[@type='CLASS']") - - instruction_covered = int(instruction_counter.get('covered', 0)) - instruction_missed = int(instruction_counter.get('missed', 0)) - total_instructions = instruction_covered + instruction_missed - instruction_coverage = (instruction_covered / total_instructions * 100) if total_instructions > 0 else 0.0 - - missed_classes = int(class_counter.get('missed', 0)) - - results.append({ - "module": module_name, - "instruction_coverage": instruction_coverage, - "missed_classes": missed_classes - }) - except Exception as e: - print(f"Error processing {module_name}: {e}") + module_path = os.path.join(input_directory, module_folder) + + if os.path.isdir(module_path): + report_file = os.path.join(module_path, f"jacocoCoverageReport.xml") + + if os.path.exists(report_file): + try: + tree = ET.parse(report_file) + root = tree.getroot() + + instruction_counter = root.find("./counter[@type='INSTRUCTION']") + class_counter = root.find("./counter[@type='CLASS']") + + if instruction_counter == None or class_counter == None: + continue + + instruction_covered = int(instruction_counter.get('covered', 0)) + instruction_missed = int(instruction_counter.get('missed', 0)) + total_instructions = instruction_covered + instruction_missed + instruction_coverage = (instruction_covered / total_instructions * 100) if total_instructions > 0 else 0.0 + + missed_classes = int(class_counter.get('missed', 0)) + + results.append({ + "module": module_folder, + "instruction_coverage": instruction_coverage, + "missed_classes": missed_classes + }) + except Exception as e: + print(f"Error processing {module_folder}: {e}") results = sorted(results, key=lambda x: x['module']) diff --git a/jacoco.gradle b/jacoco.gradle index 971d37ddce1b..96844c7419a7 100644 --- a/jacoco.gradle +++ b/jacoco.gradle @@ -118,6 +118,9 @@ private JacocoReport registerJacocoReportTask(String moduleName, JacocoReport ro task.reports { xml.required = true + xml.outputLocation = file("build/reports/jacoco/$moduleName/jacocoCoverageReport.xml") + html.required = true + html.outputLocation = file("build/reports/jacoco/$moduleName/html") } return task From d67a9b441d062a7672a7c10aec8311f4234a57bc Mon Sep 17 00:00:00 2001 From: Ole Vester Date: Mon, 16 Dec 2024 19:44:23 +0100 Subject: [PATCH 35/69] adjust per-module covered instruction threshold for fileupload --- jacoco.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jacoco.gradle b/jacoco.gradle index 96844c7419a7..39132d51dd96 100644 --- a/jacoco.gradle +++ b/jacoco.gradle @@ -35,7 +35,7 @@ ext { "CLASS": 0 ], "fileupload" : [ - "INSTRUCTION": 0.954, + "INSTRUCTION": 0.906, "CLASS": 0 ], "iris" : [ From e15e05f27f73a6d2e16d586d8401561d004f6d19 Mon Sep 17 00:00:00 2001 From: Ole Vester Date: Mon, 16 Dec 2024 19:44:33 +0100 Subject: [PATCH 36/69] remove hardcoded TEST_MODULE_TAGS from test pipeline --- .github/workflows/test.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 96f961463748..e9445f502b83 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -83,8 +83,6 @@ jobs: fi fi - TEST_MODULE_TAGS="-DincludeModules=atlas,buildagent,fileupload,lti" - ./gradlew --console=plain test jacocoTestReport -x webapp jacocoTestCoverageVerification "$TEST_MODULE_TAGS" | tee tests.log - name: Print failed tests if: failure() From 78f77a5ce69e0e25025b33c48f2637ade824c172 Mon Sep 17 00:00:00 2001 From: Ole Vester Date: Mon, 16 Dec 2024 19:46:51 +0100 Subject: [PATCH 37/69] move jacoco-parser to .ci-dir and rename to execute.py --- jacoco-parser.py => .ci/coverage-parser/execute.py | 0 .github/workflows/test.yml | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename jacoco-parser.py => .ci/coverage-parser/execute.py (100%) diff --git a/jacoco-parser.py b/.ci/coverage-parser/execute.py similarity index 100% rename from jacoco-parser.py rename to .ci/coverage-parser/execute.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e9445f502b83..d1bf0b56f621 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -114,7 +114,7 @@ jobs: if: success() || failure() run: | AGGREGATED_REPORT_FILE=./coverage_report.md - python3 ./jacoco-parser.py build/reports/jacoco $AGGREGATED_REPORT_FILE + python3 ./.ci/coverage-parser/execute.py build/reports/jacoco $AGGREGATED_REPORT_FILE cat $AGGREGATED_REPORT_FILE > $GITHUB_STEP_SUMMARY server-tests-mysql: From ad657cfd57642ae027924a7c2a0ec8dafc537e1f Mon Sep 17 00:00:00 2001 From: Ole Vester Date: Mon, 16 Dec 2024 19:47:58 +0100 Subject: [PATCH 38/69] move jacoco.gradle to gradle-dir --- build.gradle | 2 +- jacoco.gradle => gradle/jacoco.gradle | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename jacoco.gradle => gradle/jacoco.gradle (100%) diff --git a/build.gradle b/build.gradle index 7697e4b39558..70a19ced4e56 100644 --- a/build.gradle +++ b/build.gradle @@ -168,7 +168,7 @@ jar { } // Dynamic generation of jacoco test report generation-/coverage verification-tasks (per-module) -apply from: "jacoco.gradle" +apply from: "gradle/jacoco.gradle" configurations { providedRuntime diff --git a/jacoco.gradle b/gradle/jacoco.gradle similarity index 100% rename from jacoco.gradle rename to gradle/jacoco.gradle From cfc2f389401ccf8ab8dec18a60922ea58b6a74fc Mon Sep 17 00:00:00 2001 From: Ole Vester Date: Mon, 16 Dec 2024 19:51:38 +0100 Subject: [PATCH 39/69] move changed-modules.sh to .dir directory --- changed-modules.sh => .ci/test-selection/changed-modules.sh | 0 .github/workflows/test.yml | 4 ++-- 2 files changed, 2 insertions(+), 2 deletions(-) rename changed-modules.sh => .ci/test-selection/changed-modules.sh (100%) diff --git a/changed-modules.sh b/.ci/test-selection/changed-modules.sh similarity index 100% rename from changed-modules.sh rename to .ci/test-selection/changed-modules.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d1bf0b56f621..fa62b4f8b283 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -71,8 +71,8 @@ jobs: # Explicitly fetch as the clone action only clones the current branch git fetch origin "$DEFAULT_BRANCH" - chmod +x ./changed-modules.sh - CHANGED_MODULES=$(./changed-modules.sh "origin/$DEFAULT_BRANCH") + chmod +x ./.ci/test-selection/changed-modules.sh + CHANGED_MODULES=$(./.ci/test-selection/changed-modules.sh "origin/$DEFAULT_BRANCH") # Restrict executed tests to changed modules if there is diff between this and the base branch if [ -n "${CHANGED_MODULES}" ]; then From 8b6aa0c1d9f9f7ba8f7fcaea76415f32a28a14f1 Mon Sep 17 00:00:00 2001 From: Ole Vester Date: Mon, 16 Dec 2024 20:00:10 +0100 Subject: [PATCH 40/69] fix logic of number of server starts script --- supporting_scripts/extract_number_of_server_starts.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/supporting_scripts/extract_number_of_server_starts.sh b/supporting_scripts/extract_number_of_server_starts.sh index 661c6e2d4081..a4e40eff07f2 100644 --- a/supporting_scripts/extract_number_of_server_starts.sh +++ b/supporting_scripts/extract_number_of_server_starts.sh @@ -9,7 +9,7 @@ then exit 1 fi -if [[ $numberOfStarts -le 4 ]] +if [[ $numberOfStarts -gt 4 ]] then echo "The number of Server Starts should be lower than/equals 4! Please adapt this check if the change is intended or try to fix the underlying issue causing a different number of server starts!" exit 1 From 1f54c61a48c8721e578917dfd6096f8b912b1710 Mon Sep 17 00:00:00 2001 From: Ole Vester Date: Mon, 16 Dec 2024 21:23:20 +0100 Subject: [PATCH 41/69] update isolated coverage threshold per module --- gradle/jacoco.gradle | 51 ++++++++++++++++++++++---------------------- 1 file changed, 25 insertions(+), 26 deletions(-) diff --git a/gradle/jacoco.gradle b/gradle/jacoco.gradle index 39132d51dd96..071666c13bf1 100644 --- a/gradle/jacoco.gradle +++ b/gradle/jacoco.gradle @@ -1,17 +1,16 @@ ext { - - // Thresholds when executing each module on its own + // (Isolated) thresholds when executing each module on its own ModuleCoverageThresholds = [ "assessment" : [ - "INSTRUCTION": 0.891, - "CLASS": 2 + "INSTRUCTION": 0.774, + "CLASS": 9 ], "athena" : [ - "INSTRUCTION": 0.872, + "INSTRUCTION": 0.863, "CLASS": 2 ], "atlas" : [ - "INSTRUCTION": 0.832, + "INSTRUCTION": 0.846, "CLASS": 13 ], "buildagent" : [ @@ -19,59 +18,59 @@ ext { "CLASS": 13 ], "communication": [ - "INSTRUCTION": 0.862, - "CLASS": 3 + "INSTRUCTION": 0.893, + "CLASS": 7 ], "core" : [ - "INSTRUCTION": 0.778, - "CLASS": 21 + "INSTRUCTION": 0.658, + "CLASS": 69 ], "exam" : [ - "INSTRUCTION": 0.902, + "INSTRUCTION": 0.914, "CLASS": 1 ], "exercise" : [ - "INSTRUCTION": 0.929, - "CLASS": 0 + "INSTRUCTION": 0.649, + "CLASS": 9 ], "fileupload" : [ - "INSTRUCTION": 0.906, + "INSTRUCTION": 0.944, "CLASS": 0 ], "iris" : [ - "INSTRUCTION": 0.712, + "INSTRUCTION": 0.706, "CLASS": 22 ], "lecture" : [ - "INSTRUCTION": 0.910, + "INSTRUCTION": 0.867, "CLASS": 0 ], "lti" : [ - "INSTRUCTION": 0.881, - "CLASS": 2 + "INSTRUCTION": 0.770, + "CLASS": 3 ], "modeling" : [ - "INSTRUCTION": 0.914, + "INSTRUCTION": 0.892, "CLASS": 2 ], "plagiarism" : [ - "INSTRUCTION": 0.916, - "CLASS": 0 + "INSTRUCTION": 0.770, + "CLASS": 1 ], "programming" : [ - "INSTRUCTION": 0.874, - "CLASS": 14 + "INSTRUCTION": 0.861, + "CLASS": 13 ], "quiz" : [ - "INSTRUCTION": 0.796, + "INSTRUCTION": 0.785, "CLASS": 6 ], "text" : [ - "INSTRUCTION": 0.952, + "INSTRUCTION": 0.847, "CLASS": 0 ], "tutorialgroup": [ - "INSTRUCTION": 0.923, + "INSTRUCTION": 0.915, "CLASS": 0 ], ] From be4da5ef6e002d99f54739399ae526c6bf446d8e Mon Sep 17 00:00:00 2001 From: Ole Vester Date: Mon, 16 Dec 2024 22:07:20 +0100 Subject: [PATCH 42/69] move generate_code_cov_table to subfolder --- .github/PULL_REQUEST_TEMPLATE.md | 2 +- .github/dependabot.yml | 2 +- .../{ => code-coverage}/generate_code_cov_table/.gitignore | 0 .../{ => code-coverage}/generate_code_cov_table/README.md | 0 .../{ => code-coverage}/generate_code_cov_table/env.example | 0 .../generate_code_cov_table/generate_code_cov_table.py | 0 .../generate_code_cov_table/requirements.txt | 0 7 files changed, 2 insertions(+), 2 deletions(-) rename supporting_scripts/{ => code-coverage}/generate_code_cov_table/.gitignore (100%) rename supporting_scripts/{ => code-coverage}/generate_code_cov_table/README.md (100%) rename supporting_scripts/{ => code-coverage}/generate_code_cov_table/env.example (100%) rename supporting_scripts/{ => code-coverage}/generate_code_cov_table/generate_code_cov_table.py (100%) rename supporting_scripts/{ => code-coverage}/generate_code_cov_table/requirements.txt (100%) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 22968f1531b4..99cbd66265cc 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -111,7 +111,7 @@ Prerequisites: - [ ] Test 2 ### Test Coverage - + diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 18aa94ff1059..f29fa7b2cfaa 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -57,7 +57,7 @@ updates: # Check for version updates for Python dependencies (coverage) - package-ecosystem: "pip" - directory: "/supporting_scripts/generate_code_cov_table" + directory: "/supporting_scripts/code-coverage/generate_code_cov_table" schedule: interval: "weekly" reviewers: diff --git a/supporting_scripts/generate_code_cov_table/.gitignore b/supporting_scripts/code-coverage/generate_code_cov_table/.gitignore similarity index 100% rename from supporting_scripts/generate_code_cov_table/.gitignore rename to supporting_scripts/code-coverage/generate_code_cov_table/.gitignore diff --git a/supporting_scripts/generate_code_cov_table/README.md b/supporting_scripts/code-coverage/generate_code_cov_table/README.md similarity index 100% rename from supporting_scripts/generate_code_cov_table/README.md rename to supporting_scripts/code-coverage/generate_code_cov_table/README.md diff --git a/supporting_scripts/generate_code_cov_table/env.example b/supporting_scripts/code-coverage/generate_code_cov_table/env.example similarity index 100% rename from supporting_scripts/generate_code_cov_table/env.example rename to supporting_scripts/code-coverage/generate_code_cov_table/env.example diff --git a/supporting_scripts/generate_code_cov_table/generate_code_cov_table.py b/supporting_scripts/code-coverage/generate_code_cov_table/generate_code_cov_table.py similarity index 100% rename from supporting_scripts/generate_code_cov_table/generate_code_cov_table.py rename to supporting_scripts/code-coverage/generate_code_cov_table/generate_code_cov_table.py diff --git a/supporting_scripts/generate_code_cov_table/requirements.txt b/supporting_scripts/code-coverage/generate_code_cov_table/requirements.txt similarity index 100% rename from supporting_scripts/generate_code_cov_table/requirements.txt rename to supporting_scripts/code-coverage/generate_code_cov_table/requirements.txt From 49ddd27c49e990b21311285523db1b4160d19527 Mon Sep 17 00:00:00 2001 From: Ole Vester Date: Mon, 16 Dec 2024 22:21:44 +0100 Subject: [PATCH 43/69] move get_changed_modules.sh to supporting scripts --- .github/workflows/test.yml | 4 ++-- .../get_changed_modules.sh | 0 2 files changed, 2 insertions(+), 2 deletions(-) rename .ci/test-selection/changed-modules.sh => supporting_scripts/get_changed_modules.sh (100%) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index fa62b4f8b283..4765c92e24eb 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -71,8 +71,8 @@ jobs: # Explicitly fetch as the clone action only clones the current branch git fetch origin "$DEFAULT_BRANCH" - chmod +x ./.ci/test-selection/changed-modules.sh - CHANGED_MODULES=$(./.ci/test-selection/changed-modules.sh "origin/$DEFAULT_BRANCH") + chmod +x ./supporting_scripts/get_changed_modules.sh + CHANGED_MODULES=$(./supporting_scripts/get_changed_modules.sh "origin/$DEFAULT_BRANCH") # Restrict executed tests to changed modules if there is diff between this and the base branch if [ -n "${CHANGED_MODULES}" ]; then diff --git a/.ci/test-selection/changed-modules.sh b/supporting_scripts/get_changed_modules.sh similarity index 100% rename from .ci/test-selection/changed-modules.sh rename to supporting_scripts/get_changed_modules.sh From 46280fe3ec95323eef7da86b7e635fe73f103335 Mon Sep 17 00:00:00 2001 From: Ole Vester Date: Mon, 16 Dec 2024 22:22:02 +0100 Subject: [PATCH 44/69] move parse_module_coverage to supporting scripts --- .github/workflows/test.yml | 4 ++-- .../per_module_cov_report/parse_module_coverage.py | 0 2 files changed, 2 insertions(+), 2 deletions(-) rename .ci/coverage-parser/execute.py => supporting_scripts/code-coverage/per_module_cov_report/parse_module_coverage.py (100%) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4765c92e24eb..eb2b0c7bc11d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -113,8 +113,8 @@ jobs: - name: Append Per-Module Coverage to Job Summary if: success() || failure() run: | - AGGREGATED_REPORT_FILE=./coverage_report.md - python3 ./.ci/coverage-parser/execute.py build/reports/jacoco $AGGREGATED_REPORT_FILE + AGGREGATED_REPORT_FILE=./module_coverage_report.md + python3 ./supporting_scripts/code-coverage/per_module_cov_report/parse_module_coverage.py build/reports/jacoco $AGGREGATED_REPORT_FILE cat $AGGREGATED_REPORT_FILE > $GITHUB_STEP_SUMMARY server-tests-mysql: diff --git a/.ci/coverage-parser/execute.py b/supporting_scripts/code-coverage/per_module_cov_report/parse_module_coverage.py similarity index 100% rename from .ci/coverage-parser/execute.py rename to supporting_scripts/code-coverage/per_module_cov_report/parse_module_coverage.py From 1cad26f79128b9a0b3d41fc1e13bbe08c18b4b46 Mon Sep 17 00:00:00 2001 From: Ole Vester Date: Mon, 16 Dec 2024 22:31:08 +0100 Subject: [PATCH 45/69] add documentation to get_changed_modules.sh --- supporting_scripts/get_changed_modules.sh | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/supporting_scripts/get_changed_modules.sh b/supporting_scripts/get_changed_modules.sh index 60ebf84100ef..ec19d180676a 100755 --- a/supporting_scripts/get_changed_modules.sh +++ b/supporting_scripts/get_changed_modules.sh @@ -1,6 +1,20 @@ #!/bin/bash +# Determines the changed modules following the module-directory structure. +# Based on git-diff between the local state and an input branch name. +# Example: "./get_changed_modules.sh develop" + +# Check for the branch input argument. +if [ $# -eq 0 ]; then + echo "Usage: $0 " + exit 1 +fi + BRANCH_TO_COMPARE="$1" +if ! git show-ref --verify --quiet "refs/heads/$BRANCH_TO_COMPARE"; then + echo "Branch $BRANCH_TO_COMPARE does not exist." + exit 1 +fi MODULE_BASE_PATH="src/main/java/de/tum/cit/aet/artemis" MODULES=$(find "$MODULE_BASE_PATH" -maxdepth 1 -mindepth 1 -type d -exec basename {} \;) From bfd2c8d1061f07f191a6c82e23a7c16fb96960a3 Mon Sep 17 00:00:00 2001 From: Ole Vester Date: Mon, 16 Dec 2024 22:56:51 +0100 Subject: [PATCH 46/69] use test filters instead of "include" to ensure test results are created in case of a test error --- build.gradle | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index 70a19ced4e56..923fea2789ac 100644 --- a/build.gradle +++ b/build.gradle @@ -145,8 +145,11 @@ test { useJUnitPlatform() // Always execute "shared"-folder when executing module-specifc tests includedModules += "shared" - def moduleBasePaths = includedModules.collect { val -> "$BasePath/$val/**"} - include(moduleBasePaths) + filter { testFilter -> + includedModules.each { val -> + testFilter.includeTestsMatching("de.tum.cit.aet.artemis.$val.*") + } + } } testLogging { From 6e0baa5d870b075de025e7b17987346fabaeaad9 Mon Sep 17 00:00:00 2001 From: Ole Vester Date: Mon, 16 Dec 2024 22:57:18 +0100 Subject: [PATCH 47/69] println for missing coverage message --- gradle/jacoco.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/jacoco.gradle b/gradle/jacoco.gradle index 071666c13bf1..729269aa2b6f 100644 --- a/gradle/jacoco.gradle +++ b/gradle/jacoco.gradle @@ -137,7 +137,7 @@ private JacocoCoverageVerification registerJacocoTestCoverageVerification(String def thresholds = ModuleCoverageThresholds[moduleName] if (thresholds == null) { - print "No coverage thresholds defined for module '$moduleName'. Skipping..." + println "No coverage thresholds defined for module '$moduleName'. Skipping..." return null } def coveredInstructionRatio = thresholds.INSTRUCTION From 5ab299694a971ed985320401fa7959762d7d3365 Mon Sep 17 00:00:00 2001 From: Ole Vester Date: Mon, 16 Dec 2024 22:58:12 +0100 Subject: [PATCH 48/69] add info on return type of get_changed_modules.sh --- supporting_scripts/get_changed_modules.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/supporting_scripts/get_changed_modules.sh b/supporting_scripts/get_changed_modules.sh index ec19d180676a..e294f22b5068 100755 --- a/supporting_scripts/get_changed_modules.sh +++ b/supporting_scripts/get_changed_modules.sh @@ -2,6 +2,7 @@ # Determines the changed modules following the module-directory structure. # Based on git-diff between the local state and an input branch name. +# Returns a comma-separated list of changed modules. # Example: "./get_changed_modules.sh develop" # Check for the branch input argument. From 7fd0ad0226505c35894ef4466c4cc221c7c440ed Mon Sep 17 00:00:00 2001 From: Ole Vester Date: Mon, 16 Dec 2024 23:14:41 +0100 Subject: [PATCH 49/69] refactor: separate concerns in parse_module_coverage.py --- .../parse_module_coverage.py | 68 ++++++++++++------- 1 file changed, 44 insertions(+), 24 deletions(-) diff --git a/supporting_scripts/code-coverage/per_module_cov_report/parse_module_coverage.py b/supporting_scripts/code-coverage/per_module_cov_report/parse_module_coverage.py index 818b19312a06..927ac45bdc39 100644 --- a/supporting_scripts/code-coverage/per_module_cov_report/parse_module_coverage.py +++ b/supporting_scripts/code-coverage/per_module_cov_report/parse_module_coverage.py @@ -2,8 +2,12 @@ import xml.etree.ElementTree as ET import argparse -def process_reports(input_directory, output_file): +def get_report_by_module(input_directory): results = [] + if os.exists(input_directory) == False: + print(f"'{input_directory}' does not exist.") + return results + for module_folder in os.listdir(input_directory): module_path = os.path.join(input_directory, module_folder) @@ -11,34 +15,50 @@ def process_reports(input_directory, output_file): if os.path.isdir(module_path): report_file = os.path.join(module_path, f"jacocoCoverageReport.xml") - if os.path.exists(report_file): - try: - tree = ET.parse(report_file) - root = tree.getroot() + if os.exists(report_file): + results.append({ + "module": module_folder, + "report_file": report_file + }) + else: + print(f"No XML report file found for module: {module_folder}") + + return results + + +def extract_coverage(input_directory, output_file): + results = [] + + for report in get_report_files(input_directory): + try: + tree = ET.parse(report_file) + root = tree.getroot() - instruction_counter = root.find("./counter[@type='INSTRUCTION']") - class_counter = root.find("./counter[@type='CLASS']") + instruction_counter = root.find("./counter[@type='INSTRUCTION']") + class_counter = root.find("./counter[@type='CLASS']") - if instruction_counter == None or class_counter == None: - continue + if instruction_counter == None or class_counter == None: + continue - instruction_covered = int(instruction_counter.get('covered', 0)) - instruction_missed = int(instruction_counter.get('missed', 0)) - total_instructions = instruction_covered + instruction_missed - instruction_coverage = (instruction_covered / total_instructions * 100) if total_instructions > 0 else 0.0 + instruction_covered = int(instruction_counter.get('covered', 0)) + instruction_missed = int(instruction_counter.get('missed', 0)) + total_instructions = instruction_covered + instruction_missed + instruction_coverage = (instruction_covered / total_instructions * 100) if total_instructions > 0 else 0.0 - missed_classes = int(class_counter.get('missed', 0)) + missed_classes = int(class_counter.get('missed', 0)) - results.append({ - "module": module_folder, - "instruction_coverage": instruction_coverage, - "missed_classes": missed_classes - }) - except Exception as e: - print(f"Error processing {module_folder}: {e}") + results.append({ + "module": module_folder, + "instruction_coverage": instruction_coverage, + "missed_classes": missed_classes + }) + except Exception as e: + print(f"Error processing {module_folder}: {e}") results = sorted(results, key=lambda x: x['module']) + +def write_summary_to_file(report): with open(output_file, "w") as f: f.write("## Coverage Results\n\n") f.write("| Module Name | Instruction Coverage (%) | Missed Classes |\n") @@ -46,8 +66,6 @@ def process_reports(input_directory, output_file): for result in results: f.write(f"| {result['module']} | {result['instruction_coverage']:.2f} | {result['missed_classes']} |\n") - print(f"Coverage results written to {output_file}") - if __name__ == "__main__": parser = argparse.ArgumentParser(description="Process JaCoCo coverage reports.") @@ -56,4 +74,6 @@ def process_reports(input_directory, output_file): args = parser.parse_args() - process_reports(args.input_directory, args.output_file) + reports = get_report_by_module(args.input_directory) + extract_coverage = extract_coverage(reports) + write_summary_to_file(args.output_file) From 2a2b8d7e89b303a4488825a175b8c94b7d9acf1d Mon Sep 17 00:00:00 2001 From: Ole Vester Date: Mon, 16 Dec 2024 23:24:18 +0100 Subject: [PATCH 50/69] rename get_changed_modules.sh to test_changed_modules.sh --- .github/workflows/test.yml | 4 ++-- .../{get_changed_modules.sh => test_changed_modules.sh} | 0 2 files changed, 2 insertions(+), 2 deletions(-) rename supporting_scripts/{get_changed_modules.sh => test_changed_modules.sh} (100%) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index eb2b0c7bc11d..4af9a0a02c3e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -71,8 +71,8 @@ jobs: # Explicitly fetch as the clone action only clones the current branch git fetch origin "$DEFAULT_BRANCH" - chmod +x ./supporting_scripts/get_changed_modules.sh - CHANGED_MODULES=$(./supporting_scripts/get_changed_modules.sh "origin/$DEFAULT_BRANCH") + chmod +x ./supporting_scripts/test_changed_modules.sh + CHANGED_MODULES=$(./supporting_scripts/test_changed_modules.sh "origin/$DEFAULT_BRANCH") # Restrict executed tests to changed modules if there is diff between this and the base branch if [ -n "${CHANGED_MODULES}" ]; then diff --git a/supporting_scripts/get_changed_modules.sh b/supporting_scripts/test_changed_modules.sh similarity index 100% rename from supporting_scripts/get_changed_modules.sh rename to supporting_scripts/test_changed_modules.sh From 5de1c8a734f453c5f6b1833a17393229d1b37e95 Mon Sep 17 00:00:00 2001 From: Ole Vester Date: Mon, 16 Dec 2024 23:37:06 +0100 Subject: [PATCH 51/69] Revert "rename get_changed_modules.sh to test_changed_modules.sh" This reverts commit 2a2b8d7e89b303a4488825a175b8c94b7d9acf1d. --- .github/workflows/test.yml | 4 ++-- .../{test_changed_modules.sh => get_changed_modules.sh} | 0 2 files changed, 2 insertions(+), 2 deletions(-) rename supporting_scripts/{test_changed_modules.sh => get_changed_modules.sh} (100%) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4af9a0a02c3e..eb2b0c7bc11d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -71,8 +71,8 @@ jobs: # Explicitly fetch as the clone action only clones the current branch git fetch origin "$DEFAULT_BRANCH" - chmod +x ./supporting_scripts/test_changed_modules.sh - CHANGED_MODULES=$(./supporting_scripts/test_changed_modules.sh "origin/$DEFAULT_BRANCH") + chmod +x ./supporting_scripts/get_changed_modules.sh + CHANGED_MODULES=$(./supporting_scripts/get_changed_modules.sh "origin/$DEFAULT_BRANCH") # Restrict executed tests to changed modules if there is diff between this and the base branch if [ -n "${CHANGED_MODULES}" ]; then diff --git a/supporting_scripts/test_changed_modules.sh b/supporting_scripts/get_changed_modules.sh similarity index 100% rename from supporting_scripts/test_changed_modules.sh rename to supporting_scripts/get_changed_modules.sh From 9de9c3a7208a507fd12577d1a1008eb9fc6bdfbe Mon Sep 17 00:00:00 2001 From: Ole Vester Date: Mon, 16 Dec 2024 23:57:08 +0100 Subject: [PATCH 52/69] fix issues with parse_module_coverage.py --- .../parse_module_coverage.py | 26 ++++++++----------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/supporting_scripts/code-coverage/per_module_cov_report/parse_module_coverage.py b/supporting_scripts/code-coverage/per_module_cov_report/parse_module_coverage.py index 927ac45bdc39..f318589efc01 100644 --- a/supporting_scripts/code-coverage/per_module_cov_report/parse_module_coverage.py +++ b/supporting_scripts/code-coverage/per_module_cov_report/parse_module_coverage.py @@ -4,34 +4,29 @@ def get_report_by_module(input_directory): results = [] - if os.exists(input_directory) == False: - print(f"'{input_directory}' does not exist.") - return results - - for module_folder in os.listdir(input_directory): module_path = os.path.join(input_directory, module_folder) if os.path.isdir(module_path): report_file = os.path.join(module_path, f"jacocoCoverageReport.xml") - if os.exists(report_file): + if os.path.exists(report_file): results.append({ "module": module_folder, "report_file": report_file }) else: - print(f"No XML report file found for module: {module_folder}") + print(f"No XML report file found for module: {module_folder}. Skipping...") return results -def extract_coverage(input_directory, output_file): +def extract_coverage(reports): results = [] - for report in get_report_files(input_directory): + for report in reports: try: - tree = ET.parse(report_file) + tree = ET.parse(report['report_file']) root = tree.getroot() instruction_counter = root.find("./counter[@type='INSTRUCTION']") @@ -48,7 +43,7 @@ def extract_coverage(input_directory, output_file): missed_classes = int(class_counter.get('missed', 0)) results.append({ - "module": module_folder, + "module": report['module'], "instruction_coverage": instruction_coverage, "missed_classes": missed_classes }) @@ -56,14 +51,15 @@ def extract_coverage(input_directory, output_file): print(f"Error processing {module_folder}: {e}") results = sorted(results, key=lambda x: x['module']) + return results -def write_summary_to_file(report): +def write_summary_to_file(coverage_by_module, output_file): with open(output_file, "w") as f: f.write("## Coverage Results\n\n") f.write("| Module Name | Instruction Coverage (%) | Missed Classes |\n") f.write("|-------------|---------------------------|----------------|\n") - for result in results: + for result in coverage_by_module: f.write(f"| {result['module']} | {result['instruction_coverage']:.2f} | {result['missed_classes']} |\n") @@ -75,5 +71,5 @@ def write_summary_to_file(report): args = parser.parse_args() reports = get_report_by_module(args.input_directory) - extract_coverage = extract_coverage(reports) - write_summary_to_file(args.output_file) + coverage_by_module = extract_coverage(reports) + write_summary_to_file(coverage_by_module, args.output_file) From 4cdb0fa3fccc6dbe208f90d6ee67c3d26c87746c Mon Sep 17 00:00:00 2001 From: Ole Vester Date: Tue, 17 Dec 2024 00:02:29 +0100 Subject: [PATCH 53/69] remove branch input validation --- supporting_scripts/get_changed_modules.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/supporting_scripts/get_changed_modules.sh b/supporting_scripts/get_changed_modules.sh index e294f22b5068..7f923fe44d0d 100755 --- a/supporting_scripts/get_changed_modules.sh +++ b/supporting_scripts/get_changed_modules.sh @@ -12,10 +12,6 @@ if [ $# -eq 0 ]; then fi BRANCH_TO_COMPARE="$1" -if ! git show-ref --verify --quiet "refs/heads/$BRANCH_TO_COMPARE"; then - echo "Branch $BRANCH_TO_COMPARE does not exist." - exit 1 -fi MODULE_BASE_PATH="src/main/java/de/tum/cit/aet/artemis" MODULES=$(find "$MODULE_BASE_PATH" -maxdepth 1 -mindepth 1 -type d -exec basename {} \;) From e1748af526ddec9de2bb6c61e42effe700b6a64b Mon Sep 17 00:00:00 2001 From: Ole Vester Date: Tue, 17 Dec 2024 00:42:26 +0100 Subject: [PATCH 54/69] generate full reports when no input modules specified --- gradle/jacoco.gradle | 66 ++++++++++++++++++++++++++------------------ 1 file changed, 39 insertions(+), 27 deletions(-) diff --git a/gradle/jacoco.gradle b/gradle/jacoco.gradle index 729269aa2b6f..71b6cf23ec2c 100644 --- a/gradle/jacoco.gradle +++ b/gradle/jacoco.gradle @@ -1,4 +1,8 @@ ext { + AggregatedCoverageThresholds = [ + "INSTRUCTION": 0.900, + "CLASS": 10 + ]; // (Isolated) thresholds when executing each module on its own ModuleCoverageThresholds = [ "assessment" : [ @@ -78,6 +82,13 @@ ext { reportedModules = includedModules.size() == 0 ? ModuleCoverageThresholds.collect {element -> element.key} : includedModules as ArrayList + + // we want to ignore some generated files in the domain folders + ignoredDirectories = [ + "**/$BasePath/**/domain/**/*_*", + "**/$BasePath/core/config/migration/entries/**", + "**/gradle-wrapper.jar/**" + ] } jacoco { @@ -91,8 +102,12 @@ jacocoTestReport { } jacocoTestCoverageVerification { - // This task is just a wrapper to dynamically create tasks (per module) - enabled = false + // Only run full coverage when no specific modules set + enabled = reportedModules.size() == 0 + + def minInstructionCoveredRatio = AggregatedCoverageThresholds["INSTRUCTION"] as double + def maxNumberUncoveredClasses = AggregatedCoverageThresholds["CLASS"] as int + applyVerificationRule(jacocoTestCoverageVerification, minInstructionCoveredRatio, maxNumberUncoveredClasses) finalizedBy reportedModules .collect { module -> registerJacocoTestCoverageVerification(module as String, jacocoTestCoverageVerification) } @@ -140,28 +155,14 @@ private JacocoCoverageVerification registerJacocoTestCoverageVerification(String println "No coverage thresholds defined for module '$moduleName'. Skipping..." return null } - def coveredInstructionRatio = thresholds.INSTRUCTION - def missedClassCount = thresholds.CLASS + def minInstructionCoveredRatio = thresholds["INSTRUCTION"] as double + def maxNumberUncoveredClasses = thresholds["CLASS"] as int JacocoCoverageVerification task = project.tasks.register(taskName, JacocoCoverageVerification).get() task.description = "Validates JaCoCo coverage for vioalations for $moduleName" prepareJacocoReportTask(task, moduleName, rootTask) - - task.violationRules { - rule { - limit { - counter = "INSTRUCTION" - value = "COVEREDRATIO" - minimum = coveredInstructionRatio - } - limit { - counter = "CLASS" - value = "MISSEDCOUNT" - maximum = missedClassCount - } - } - } + applyVerificationRule(task, minInstructionCoveredRatio, maxNumberUncoveredClasses) return task } @@ -184,14 +185,25 @@ private void prepareJacocoReportTask(JacocoReportBase task, String moduleName, J files(rootTask.classDirectories.files.collect { classDir -> project.fileTree(classDir) { includes=[modulePath] - // we want to ignore some generated files in the domain folders - excludes=[ - "**/$BasePath/**/domain/**/*_*", - "**/$BasePath/core/config/migration/entries/**", - "**/gradle-wrapper.jar/**" - ] + excludes=ignoredDirectories } - } - ) + }) ) } + +private static void applyVerificationRule(JacocoCoverageVerification task, double minInstructionCoveredRatio, int maxNumberUncoveredClasses) { + task.violationRules { + rule { + limit { + counter = "INSTRUCTION" + value = "COVEREDRATIO" + minimum = minInstructionCoveredRatio + } + limit { + counter = "CLASS" + value = "MISSEDCOUNT" + maximum = maxNumberUncoveredClasses + } + } + } +} From 240aef16609828193ced3ecc3990a4ba39d9b25e Mon Sep 17 00:00:00 2001 From: Ole Vester Date: Tue, 17 Dec 2024 00:44:54 +0100 Subject: [PATCH 55/69] change message to be more explicit about skipping coverage verification --- gradle/jacoco.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/jacoco.gradle b/gradle/jacoco.gradle index 71b6cf23ec2c..a618bf9f7f83 100644 --- a/gradle/jacoco.gradle +++ b/gradle/jacoco.gradle @@ -152,7 +152,7 @@ private JacocoCoverageVerification registerJacocoTestCoverageVerification(String def thresholds = ModuleCoverageThresholds[moduleName] if (thresholds == null) { - println "No coverage thresholds defined for module '$moduleName'. Skipping..." + println "No coverage thresholds defined for module '$moduleName'. Skipping verification..." return null } def minInstructionCoveredRatio = thresholds["INSTRUCTION"] as double From b1042a45b4e9f71395bd0973a0a14f234a3540f3 Mon Sep 17 00:00:00 2001 From: Ole Vester Date: Tue, 17 Dec 2024 01:34:25 +0100 Subject: [PATCH 56/69] add debugging --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index eb2b0c7bc11d..aba6177583b5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -83,7 +83,7 @@ jobs: fi fi - ./gradlew --console=plain test jacocoTestReport -x webapp jacocoTestCoverageVerification "$TEST_MODULE_TAGS" | tee tests.log + ./gradlew --console=plain test jacocoTestReport -x webapp jacocoTestCoverageVerification --warning-mode all "$TEST_MODULE_TAGS" | tee tests.log - name: Print failed tests if: failure() run: grep "Test >.* FAILED\$" tests.log || echo "No failed tests." From e668fb9cb0574067ffbde6aab81b2b09a4412fc2 Mon Sep 17 00:00:00 2001 From: Ole Vester Date: Tue, 17 Dec 2024 01:36:37 +0100 Subject: [PATCH 57/69] Revert "add debugging" This reverts commit b1042a45b4e9f71395bd0973a0a14f234a3540f3. --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index aba6177583b5..eb2b0c7bc11d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -83,7 +83,7 @@ jobs: fi fi - ./gradlew --console=plain test jacocoTestReport -x webapp jacocoTestCoverageVerification --warning-mode all "$TEST_MODULE_TAGS" | tee tests.log + ./gradlew --console=plain test jacocoTestReport -x webapp jacocoTestCoverageVerification "$TEST_MODULE_TAGS" | tee tests.log - name: Print failed tests if: failure() run: grep "Test >.* FAILED\$" tests.log || echo "No failed tests." From 3cdfda716c308a061e51922d2ffadffa09c21842 Mon Sep 17 00:00:00 2001 From: Ole Vester Date: Tue, 17 Dec 2024 01:37:04 +0100 Subject: [PATCH 58/69] log command --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index eb2b0c7bc11d..46632befd601 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -83,6 +83,7 @@ jobs: fi fi + echo "./gradlew --console=plain test jacocoTestReport -x webapp jacocoTestCoverageVerification $TEST_MODULE_TAGS" ./gradlew --console=plain test jacocoTestReport -x webapp jacocoTestCoverageVerification "$TEST_MODULE_TAGS" | tee tests.log - name: Print failed tests if: failure() From 42d0b82f46de93357ede425737044387566a12a3 Mon Sep 17 00:00:00 2001 From: Ole Vester Date: Tue, 17 Dec 2024 01:39:30 +0100 Subject: [PATCH 59/69] move test module tags further into the command --- .github/workflows/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 46632befd601..dcb3adc77e10 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -83,8 +83,8 @@ jobs: fi fi - echo "./gradlew --console=plain test jacocoTestReport -x webapp jacocoTestCoverageVerification $TEST_MODULE_TAGS" - ./gradlew --console=plain test jacocoTestReport -x webapp jacocoTestCoverageVerification "$TEST_MODULE_TAGS" | tee tests.log + echo "./gradlew --console=plain test $TEST_MODULE_TAGS jacocoTestReport -x webapp jacocoTestCoverageVerification" + ./gradlew --console=plain test "$TEST_MODULE_TAGS" jacocoTestReport -x webapp jacocoTestCoverageVerification | tee tests.log - name: Print failed tests if: failure() run: grep "Test >.* FAILED\$" tests.log || echo "No failed tests." From 53ca8d09f462a6e085d5c3dddadee8016348bcca Mon Sep 17 00:00:00 2001 From: Ole Vester Date: Tue, 17 Dec 2024 01:44:30 +0100 Subject: [PATCH 60/69] add stacktrace logging --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index dcb3adc77e10..02f6f61e8dd3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -84,7 +84,7 @@ jobs: fi echo "./gradlew --console=plain test $TEST_MODULE_TAGS jacocoTestReport -x webapp jacocoTestCoverageVerification" - ./gradlew --console=plain test "$TEST_MODULE_TAGS" jacocoTestReport -x webapp jacocoTestCoverageVerification | tee tests.log + ./gradlew --console=plain test "$TEST_MODULE_TAGS" jacocoTestReport -x webapp jacocoTestCoverageVerification --stacktrace | tee tests.log - name: Print failed tests if: failure() run: grep "Test >.* FAILED\$" tests.log || echo "No failed tests." From 20a43dfe32cfba9d81d283867566de2eab24ba89 Mon Sep 17 00:00:00 2001 From: Ole Vester Date: Tue, 17 Dec 2024 01:49:33 +0100 Subject: [PATCH 61/69] explicitly run all or only-specific tests --- .github/workflows/test.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 02f6f61e8dd3..51426899a11d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -80,11 +80,13 @@ jobs: TEST_MODULE_TAGS=$(echo "-DincludeModules=${CHANGED_MODULES[*]}") echo "Executing tests for modules: $CHANGED_MODULES" + ./gradlew --console=plain test jacocoTestReport -x webapp jacocoTestCoverageVerification "$TEST_MODULE_TAGS" | tee tests.log + exit 0 fi fi - echo "./gradlew --console=plain test $TEST_MODULE_TAGS jacocoTestReport -x webapp jacocoTestCoverageVerification" - ./gradlew --console=plain test "$TEST_MODULE_TAGS" jacocoTestReport -x webapp jacocoTestCoverageVerification --stacktrace | tee tests.log + echo "Executing all tests" + ./gradlew --console=plain test jacocoTestReport -x webapp jacocoTestCoverageVerification | tee tests.log - name: Print failed tests if: failure() run: grep "Test >.* FAILED\$" tests.log || echo "No failed tests." From ce82e2eba442bd090b8090d48bce71ffc7c3b9bd Mon Sep 17 00:00:00 2001 From: Ole Vester Date: Tue, 17 Dec 2024 12:41:24 +0100 Subject: [PATCH 62/69] remove unused file --- .../artemis/endpoint_monitoring.json | 462 ------------------ 1 file changed, 462 deletions(-) delete mode 100644 docker/monitoring/grafana/provisioning/dashboards/artemis/endpoint_monitoring.json diff --git a/docker/monitoring/grafana/provisioning/dashboards/artemis/endpoint_monitoring.json b/docker/monitoring/grafana/provisioning/dashboards/artemis/endpoint_monitoring.json deleted file mode 100644 index 0ba771cfdcb6..000000000000 --- a/docker/monitoring/grafana/provisioning/dashboards/artemis/endpoint_monitoring.json +++ /dev/null @@ -1,462 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 5, - "links": [], - "liveNow": false, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus_default" - }, - "description": "Endpoints that have top 20 highest average duration.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 0 - }, - "id": 2, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus_default" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "topk($topNumberEndpoints, 1000 * sum(rate (http_server_requests_seconds_sum[$__rate_interval])\n/\nrate(http_server_requests_seconds_count[$__rate_interval])) by (uri))", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "C", - "useBackend": false - } - ], - "title": "Average Request Duration (ms)", - "transformations": [], - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus_default" - }, - "description": "Request count per endpoint for the 20 most used endpoints.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 0 - }, - "id": 4, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus_default" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "topk($topNumberEndpoints, sum(increase(http_server_requests_seconds_count[$__rate_interval])) by (uri))", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "C", - "useBackend": false - } - ], - "title": "Request Count", - "transformations": [], - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus_default" - }, - "description": "Error rates for the top 20 endpoints with the highest 4xx error rate.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "% of total requests", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "max": 100, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 9 - }, - "id": 1, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus_default" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "topk($topNumberEndpoints, 100 * sum(rate(http_server_requests_seconds_count{status=~\"4..\"}[$__rate_interval])) by (uri)\n/\nsum(rate(http_server_requests_seconds_count[$__rate_interval])) by (uri))", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "C", - "useBackend": false - } - ], - "title": "4xx Endpoint Error Rate (%)", - "transformations": [], - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus_default" - }, - "description": "Top 20 endpoints with the highest 5xx error rate.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "% of total requests", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "max": 100, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 9 - }, - "id": 3, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus_default" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "topk($topNumberEndpoints, 100 * sum(rate(http_server_requests_seconds_count{status=~\"5..\"}[$__rate_interval])) by (uri)\n/\nsum(rate(http_server_requests_seconds_count[$__rate_interval])) by (uri))", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "C", - "useBackend": false - } - ], - "title": "5xx Endpoint Error Rate (%)", - "transformations": [], - "type": "timeseries" - } - ], - "refresh": "", - "schemaVersion": 38, - "style": "dark", - "tags": [], - "templating": { - "list": [ - { - "current": { - "selected": false, - "text": "20", - "value": "20" - }, - "description": "The number of endpoints that are considered in each graph, sorted by the highest values.", - "hide": 0, - "label": "Number Endpoints with Highest Value", - "name": "topNumberEndpoints", - "options": [ - { - "selected": true, - "text": "20", - "value": "20" - } - ], - "query": "20", - "skipUrlSync": false, - "type": "textbox" - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "Endpoint Monitoring Copy", - "uid": "fc1fba24-8e72-439a-9759-b35315c1e2ee", - "version": 13, - "weekStart": "" -} From 4ccaa0a0511c2f812dde08676678c1ec563edc21 Mon Sep 17 00:00:00 2001 From: Ole Vester Date: Tue, 17 Dec 2024 12:41:43 +0100 Subject: [PATCH 63/69] fix module name in error message --- .../per_module_cov_report/parse_module_coverage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/supporting_scripts/code-coverage/per_module_cov_report/parse_module_coverage.py b/supporting_scripts/code-coverage/per_module_cov_report/parse_module_coverage.py index f318589efc01..4b1281614d14 100644 --- a/supporting_scripts/code-coverage/per_module_cov_report/parse_module_coverage.py +++ b/supporting_scripts/code-coverage/per_module_cov_report/parse_module_coverage.py @@ -48,7 +48,7 @@ def extract_coverage(reports): "missed_classes": missed_classes }) except Exception as e: - print(f"Error processing {module_folder}: {e}") + print(f"Error processing {report['module']}: {e}") results = sorted(results, key=lambda x: x['module']) return results From 7d3cc92af29bd9652ab166ab4e757a1b42a2b25d Mon Sep 17 00:00:00 2001 From: Ole Vester Date: Tue, 17 Dec 2024 12:41:53 +0100 Subject: [PATCH 64/69] remove fixed branch --- .github/workflows/test.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 51426899a11d..deb327c89146 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -16,7 +16,6 @@ on: - develop - main - release/* - - feature/grafana-endpoint-monitoring tags: '[0-9]+.[0-9]+.[0-9]+' paths-ignore: - 'README.md' From 8c58a39a24d8c4387d9a9d0b5b75889c9cde9a5b Mon Sep 17 00:00:00 2001 From: Ole Vester Date: Tue, 17 Dec 2024 12:42:02 +0100 Subject: [PATCH 65/69] change upload artifact path --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index deb327c89146..9b8adc53e627 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -111,7 +111,7 @@ jobs: uses: actions/upload-artifact@v4 with: name: Coverage Report Server Tests - path: build/reports/jacoco/ + path: build/reports/jacoco/test/html - name: Append Per-Module Coverage to Job Summary if: success() || failure() run: | From dbe638e7db107c734a3bc6c965edab423aee416d Mon Sep 17 00:00:00 2001 From: Ole Vester Date: Tue, 17 Dec 2024 12:42:10 +0100 Subject: [PATCH 66/69] add codacy step again --- .github/workflows/test.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9b8adc53e627..0155d5d63248 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -89,6 +89,12 @@ jobs: - name: Print failed tests if: failure() run: grep "Test >.* FAILED\$" tests.log || echo "No failed tests." + - name: "Codacy: Report coverage" + uses: codacy/codacy-coverage-reporter-action@master + with: + project-token: ${{ secrets.CODACY_PROJECT_TOKEN }} + coverage-reports: build/reports/jacoco/test/jacocoTestReport.xml + if: (github.event.pull_request.head.repo.full_name == github.event.pull_request.base.repo.full_name) && (success() || failure()) && github.event.pull_request.user.login != 'dependabot[bot]' - name: Annotate Server Test Results uses: ashley-taylor/junit-report-annotations-action@f9c1a5cbe28479439f82b80a5402a6d3aa1990ac if: always() && github.event.pull_request.user.login != 'dependabot[bot]' From 830d2bd58b2c015f3fe1ff86cb85dd06b201f8c0 Mon Sep 17 00:00:00 2001 From: Ole Vester Date: Tue, 17 Dec 2024 13:43:49 +0100 Subject: [PATCH 67/69] fix jacoco report output location for aggregated report --- gradle/jacoco.gradle | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/gradle/jacoco.gradle b/gradle/jacoco.gradle index a618bf9f7f83..cd1ec9883566 100644 --- a/gradle/jacoco.gradle +++ b/gradle/jacoco.gradle @@ -96,6 +96,14 @@ jacoco { } jacocoTestReport { + // For the aggregated report + reports { + xml.required = true + xml.outputLocation = file("build/reports/jacoco/test/jacocoCoverageReport.xml") + html.required = true + html.outputLocation = file("build/reports/jacoco/test/html") + } + finalizedBy reportedModules .collect { module -> registerJacocoReportTask(module as String, jacocoTestReport) } .findAll { task -> task != null} From 7e590594a4ce71e1e9f0611e76c9ed28c29f809a Mon Sep 17 00:00:00 2001 From: Ole Vester Date: Tue, 17 Dec 2024 14:03:48 +0100 Subject: [PATCH 68/69] rename jacocoCoverageReport.xml to jacocoTestReport.xml --- gradle/jacoco.gradle | 4 ++-- .../per_module_cov_report/parse_module_coverage.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/gradle/jacoco.gradle b/gradle/jacoco.gradle index cd1ec9883566..7100591661fd 100644 --- a/gradle/jacoco.gradle +++ b/gradle/jacoco.gradle @@ -99,7 +99,7 @@ jacocoTestReport { // For the aggregated report reports { xml.required = true - xml.outputLocation = file("build/reports/jacoco/test/jacocoCoverageReport.xml") + xml.outputLocation = file("build/reports/jacoco/test/jacocoTestReport.xml") html.required = true html.outputLocation = file("build/reports/jacoco/test/html") } @@ -140,7 +140,7 @@ private JacocoReport registerJacocoReportTask(String moduleName, JacocoReport ro task.reports { xml.required = true - xml.outputLocation = file("build/reports/jacoco/$moduleName/jacocoCoverageReport.xml") + xml.outputLocation = file("build/reports/jacoco/$moduleName/jacocoTestReport.xml") html.required = true html.outputLocation = file("build/reports/jacoco/$moduleName/html") } diff --git a/supporting_scripts/code-coverage/per_module_cov_report/parse_module_coverage.py b/supporting_scripts/code-coverage/per_module_cov_report/parse_module_coverage.py index 4b1281614d14..50d21cf9c642 100644 --- a/supporting_scripts/code-coverage/per_module_cov_report/parse_module_coverage.py +++ b/supporting_scripts/code-coverage/per_module_cov_report/parse_module_coverage.py @@ -8,7 +8,7 @@ def get_report_by_module(input_directory): module_path = os.path.join(input_directory, module_folder) if os.path.isdir(module_path): - report_file = os.path.join(module_path, f"jacocoCoverageReport.xml") + report_file = os.path.join(module_path, f"jacocoTestReport.xml") if os.path.exists(report_file): results.append({ From 631e229323cd9d71e4f9e78ebb612778963522b9 Mon Sep 17 00:00:00 2001 From: Ole Vester Date: Tue, 17 Dec 2024 20:58:55 +0100 Subject: [PATCH 69/69] include tests into determination of changed modules --- supporting_scripts/get_changed_modules.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/supporting_scripts/get_changed_modules.sh b/supporting_scripts/get_changed_modules.sh index 7f923fe44d0d..d9188d69ee24 100755 --- a/supporting_scripts/get_changed_modules.sh +++ b/supporting_scripts/get_changed_modules.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Determines the changed modules following the module-directory structure. +# Determines the changed modules following the module-directory structure for both main/test. # Based on git-diff between the local state and an input branch name. # Returns a comma-separated list of changed modules. # Example: "./get_changed_modules.sh develop" @@ -14,12 +14,16 @@ fi BRANCH_TO_COMPARE="$1" MODULE_BASE_PATH="src/main/java/de/tum/cit/aet/artemis" +MODULE_BASE_TEST_PATH="src/test/java/de/tum/cit/aet/artemis" + MODULES=$(find "$MODULE_BASE_PATH" -maxdepth 1 -mindepth 1 -type d -exec basename {} \;) CHANGED_MODULES=() for MODULE in $MODULES; do if git diff "$BRANCH_TO_COMPARE" --name-only | grep -q "^$MODULE_BASE_PATH/$MODULE/"; then CHANGED_MODULES+=("$MODULE") + elif git diff "$BRANCH_TO_COMPARE" --name-only | grep -q "^$MODULE_BASE_TEST_PATH/$MODULE/"; then + CHANGED_MODULES+=("$MODULE") fi done