From 6620d62155843739b401ee47df8ab703d67d7db5 Mon Sep 17 00:00:00 2001 From: Otto Sabart Date: Mon, 26 Aug 2024 09:00:00 +0200 Subject: [PATCH] Add support for subresults in junit report plugin flavor Define an experimental junit flavor for subresults. --- docs/releases.rst | 11 +++ tests/report/junit/test.sh | 16 ++++ tmt/steps/report/junit.py | 33 +++++-- tmt/steps/report/junit/schemas/subresults.xsd | 81 +++++++++++++++++ .../report/junit/templates/subresults.xml.j2 | 90 +++++++++++++++++++ 5 files changed, 225 insertions(+), 6 deletions(-) create mode 100644 tmt/steps/report/junit/schemas/subresults.xsd create mode 100644 tmt/steps/report/junit/templates/subresults.xml.j2 diff --git a/docs/releases.rst b/docs/releases.rst index a7104bce60..35673a84b4 100644 --- a/docs/releases.rst +++ b/docs/releases.rst @@ -4,6 +4,17 @@ Releases ====================== +tmt-1.38.0 +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The :ref:`/plugins/report/junit` report plugin now supports a new +``subresults`` JUnit flavor. This flavor adds support for tmt subresults and +changes the level of ```` and ```` tags. By using this +flavor, the ``tmt.Result`` tags become ```` tags with one +```` tag representing the parent result, and possible additional +```` tags for each ``tmt.SubResult``. + + tmt-1.37 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/tests/report/junit/test.sh b/tests/report/junit/test.sh index 8f41d396f7..af78ab54d4 100755 --- a/tests/report/junit/test.sh +++ b/tests/report/junit/test.sh @@ -58,6 +58,22 @@ rlJournalStart rlAssertGrep 'The generated XML output is not a valid XML file.' "output" rlPhaseEnd + + rlPhaseStartTest "[$method] Check the 'subresults' flavor" + rlRun "tmt run -avr execute -h $method report -h junit --file subresults-out.xml --flavor subresults 2>&1 >/dev/null | tee output" 2 + + # Parent result recorded in testuite tag + rlAssertGrep '' "subresults-out.xml" + rlAssertGrep '' "subresults-out.xml" + + # TODO: Add check for additional subresults as soon as they get saved by: + # - https://github.com/teemtee/tmt/pull/3200 + rlPhaseEnd done rlPhaseStartCleanup diff --git a/tmt/steps/report/junit.py b/tmt/steps/report/junit.py index 234f66f16a..ea84bdb6fa 100644 --- a/tmt/steps/report/junit.py +++ b/tmt/steps/report/junit.py @@ -1,7 +1,7 @@ import dataclasses import functools from collections.abc import Iterator -from typing import TYPE_CHECKING, Any, Optional, TypedDict, cast, overload +from typing import TYPE_CHECKING, Any, Optional, TypedDict, Union, cast, overload from jinja2 import FileSystemLoader, select_autoescape @@ -83,10 +83,30 @@ class PropertyDict(TypedDict): name: str value: str - def __init__(self, wrapped: tmt.Result) -> None: + def __init__( + self, + wrapped: Union[tmt.Result, tmt.result.SubResult], + subresults_context_class: 'type[ResultsContext]') -> None: + self._wrapped = wrapped + self._subresults_context_class = subresults_context_class self._properties: dict[str, str] = {} + @property + def subresult(self) -> 'ResultsContext': + """ + Override the ``tmt.Result.subresult`` and wrap all the ``tmt.result.SubResult`` instances + into the ``ResultsContext``. + """ + + # `tmt.result.SubResult.subresult` is not defined, just raise the AttributeError to silent + # the typing errors. + if isinstance(self._wrapped, tmt.result.SubResult): + raise AttributeError( + f"'{self._wrapped.__class__.__name__} object has no attribute 'subresult'") + + return self._subresults_context_class(self._wrapped.subresult) + @property def properties(self) -> list[PropertyDict]: return [{'name': k, 'value': v} for k, v in self._properties.items()] @@ -108,9 +128,10 @@ class ResultsContext: wraps all the :py:class:`tmt.Result` instances into the :py:class:`ResultWrapper`. """ - def __init__(self, results: list[tmt.Result]) -> None: - # Decorate all the tmt.Results with more attributes - self._results: list[ResultWrapper] = [ResultWrapper(r) for r in results] + def __init__(self, results: Union[list[tmt.Result], list[tmt.result.SubResult]]) -> None: + """ Decorate/wrap all the ``tmt.Results`` with more attributes """ + self._results: list[ResultWrapper] = [ + ResultWrapper(r, subresults_context_class=self.__class__) for r in results] def __iter__(self) -> Iterator[ResultWrapper]: """ Possibility to iterate over results by iterating an instance """ @@ -306,7 +327,7 @@ class ReportJUnitData(tmt.steps.report.ReportStepData): flavor: str = field( default=DEFAULT_FLAVOR_NAME, option='--flavor', - choices=[DEFAULT_FLAVOR_NAME, CUSTOM_FLAVOR_NAME], + choices=[DEFAULT_FLAVOR_NAME, CUSTOM_FLAVOR_NAME, 'subresults'], help='Name of a JUnit flavor to generate.') template_path: Optional[Path] = field( diff --git a/tmt/steps/report/junit/schemas/subresults.xsd b/tmt/steps/report/junit/schemas/subresults.xsd new file mode 100644 index 0000000000..8345211ff2 --- /dev/null +++ b/tmt/steps/report/junit/schemas/subresults.xsd @@ -0,0 +1,81 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tmt/steps/report/junit/templates/subresults.xml.j2 b/tmt/steps/report/junit/templates/subresults.xml.j2 new file mode 100644 index 0000000000..2a41bbf6a0 --- /dev/null +++ b/tmt/steps/report/junit/templates/subresults.xml.j2 @@ -0,0 +1,90 @@ +{% extends "_base.xml.j2" %} + +{# + This flavor changes the level of `` and `` tags. The + `tmt.Result` becomes ```` instead of ``testcase`` and + ```` tags become ``tmt.SubResult``. +#} + +{% block content %} + + + {% block testsuites %} + {% for result in RESULTS %} + {% set main_log = result.log | first | read_log %} + {% set main_log_failures = main_log | failures | e %} + {% set main_test_duration = result.duration | duration_to_seconds %} + + {# TODO: Fix the test counts in testsuite tag #} + {# TODO: Also fix the counts because now with an additional testcase for parent results, the counts will not match. #} + + + + {# + Always include a `testcase` representing the main result. + The `error/failure/skipped` tags must not exists inside a + `testsuite`, they are only allowed inside of a `testcase`. + #} + + {% if result.result.value == 'error' or result.result.value == 'warn' %} + {{ main_log_failures }} + {% elif result.result.value == 'fail' %} + {{ main_log_failures }} + {% elif result.result.value == 'info' %} + {{ main_log_failures }} + {% endif %} + + {% if INCLUDE_OUTPUT_LOG and main_log %} + {{ main_log | e }} + {% endif %} + + + {% for subresult in result.subresult %} + {% set subresult_log = subresult.log | first | read_log %} + {% set subresult_log_failures = main_log | failures | e %} + {% set subresult_test_duration = subresult.duration | duration_to_seconds %} + + + {% if subresult.result.value == 'error' or subresult.result.value == 'warn' %} + {{ subresult_log_failures }} + {% elif subresult.result.value == 'fail' %} + {{ subresult_log_failures }} + {% elif subresult.result.value == 'info' %} + {{ subresult_log_failures }} + {% endif %} + + {% if INCLUDE_OUTPUT_LOG and subresult_log %} + {{ subresult_log | e }} + {% endif %} + + {% endfor %} + + {# + TODO: + Optionally include testcase properties? + + Optionally add the result properties + {% if result.properties is defined %} + {% with properties=result.properties %} + include "includes/_properties.xml.j2" + {% endwith %} + {% endif %} + #} + + {% endfor %} + {% endblock %} + + {# + TODO: + Optionally include testsuites properties if they are defined. + + Optionally include the properties section in testsuites tag + {% if RESULTS.properties is defined %} + {% with properties=RESULTS.properties %} + include "includes/_properties.xml.j2" + {% endwith %} + {% endif %} + #} + + +{% endblock %}