Skip to content

Commit

Permalink
Add support for subresults in junit report plugin flavor
Browse files Browse the repository at this point in the history 
Define an experimental junit flavor for subresults.
  • Loading branch information
@seberm
seberm committed Sep 6, 2024
1 parent 9ee1638 commit 789e4ea
Show file tree
Hide file tree
Showing 4 changed files with 183 additions and 6 deletions.
6 changes: 6 additions & 0 deletions docs/releases.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,12 @@
tmt-1.37
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

The :ref:`/plugins/report/junit` report plugin now supports a new
``subresults`` JUnit flavor. This flavor adds support for tmt subresults and
changes the level of ``<testsuite>`` and ``<testcase>`` tags. The
``tmt.Result`` tags become ``<testsuite>`` (instead of ``testcase``) and
``<testcase>`` tags become ``tmt.SubResult``.

The :ref:`/plugins/report/junit` report plugin now validates all the XML
flavors against their respective XSD schemas and tries to prettify the final
XML output. These functionalities are always disabled for ``custom`` flavors.
Expand Down
32 changes: 26 additions & 6 deletions tmt/steps/report/junit.py
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import dataclasses
import functools
from collections.abc import Iterator
from typing import TYPE_CHECKING, Any, Optional, TypedDict, cast, overload
from typing import TYPE_CHECKING, Any, Optional, TypedDict, Union, cast, overload

from jinja2 import FileSystemLoader, select_autoescape

Expand Down Expand Up @@ -83,10 +83,29 @@ class PropertyDict(TypedDict):
name: str
value: str

def __init__(self, wrapped: tmt.Result) -> None:
def __init__(
self,
wrapped: Union[tmt.Result, tmt.result.SubResult],
subresults_context_class: 'type[ResultsContext]') -> None:
self._wrapped = wrapped
self._subresults_context_class = subresults_context_class
self._properties: dict[str, str] = {}

@property
def subresult(self) -> 'ResultsContext':
"""
Override the ``tmt.Result.subresult`` and wrap all the ``tmt.result.SubResult`` instances
into the ``ResultsContext``.
"""

# `tmt.result.SubResult.subresult` is not defined, just raise the AttributeError to silent
# the typing errors.
if isinstance(self._wrapped, tmt.result.SubResult):
raise AttributeError(
f"'{self._wrapped.__class__.__name__} object has no attribute 'subresult'")

return self._subresults_context_class(self._wrapped.subresult)

@property
def properties(self) -> list[PropertyDict]:
return [{'name': k, 'value': v} for k, v in self._properties.items()]
Expand All @@ -108,9 +127,10 @@ class ResultsContext:
wraps all the :py:class:`tmt.Result` instances into the :py:class:`ResultWrapper`.
"""

def __init__(self, results: list[tmt.Result]) -> None:
# Decorate all the tmt.Results with more attributes
self._results: list[ResultWrapper] = [ResultWrapper(r) for r in results]
def __init__(self, results: Union[list[tmt.Result], list[tmt.result.SubResult]]) -> None:
""" Decorate/wrap all the ``tmt.Results`` with more attributes """
self._results: list[ResultWrapper] = [
ResultWrapper(r, subresults_context_class=self.__class__) for r in results]

def __iter__(self) -> Iterator[ResultWrapper]:
""" Possibility to iterate over results by iterating an instance """
Expand Down Expand Up @@ -306,7 +326,7 @@ class ReportJUnitData(tmt.steps.report.ReportStepData):
flavor: str = field(
default=DEFAULT_FLAVOR_NAME,
option='--flavor',
choices=[DEFAULT_FLAVOR_NAME, CUSTOM_FLAVOR_NAME],
choices=[DEFAULT_FLAVOR_NAME, CUSTOM_FLAVOR_NAME, 'subresults'],
help='Name of a JUnit flavor to generate.')

template_path: Optional[Path] = field(
Expand Down
81 changes: 81 additions & 0 deletions tmt/steps/report/junit/schemas/subresults.xsd
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
<?xml version="1.0" encoding="UTF-8" ?>

<!--
This schema extends the `default` flavor and adds support for tmt
subresults. It allows multiple occurrences of testsuite elements inside a
<testsuites>.
The <testsuite> is mapped to `tmt.Result` and the <testcase> is mapped to
`tmt.SubResult`.
-->

<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema">

<xs:element name="failure">
<xs:complexType mixed="true">
<xs:attribute name="type" type="xs:string" use="required"/>
<xs:attribute name="message" type="xs:string" use="required"/>
</xs:complexType>
</xs:element>

<xs:element name="error">
<xs:complexType mixed="true">
<xs:attribute name="type" type="xs:string" use="required"/>
<xs:attribute name="message" type="xs:string" use="required"/>
</xs:complexType>
</xs:element>

<xs:element name="skipped">
<xs:complexType mixed="true">
<xs:attribute name="type" type="xs:string" use="required"/>
<xs:attribute name="message" type="xs:string" use="required"/>
</xs:complexType>
</xs:element>
<xs:element name="system-err" type="xs:string"/>
<xs:element name="system-out" type="xs:string"/>

<xs:element name="testcase">
<xs:complexType>
<xs:sequence>
<xs:element ref="skipped" minOccurs="0" maxOccurs="1"/>
<xs:element ref="error" minOccurs="0" maxOccurs="1"/>
<xs:element ref="failure" minOccurs="0" maxOccurs="1"/>
<xs:element ref="system-out" minOccurs="0" maxOccurs="1"/>
<xs:element ref="system-err" minOccurs="0" maxOccurs="1"/>
</xs:sequence>
<xs:attribute name="name" type="xs:string" use="required"/>
<xs:attribute name="time" type="xs:string" use="optional"/>
</xs:complexType>
</xs:element>

<xs:element name="testsuite">
<xs:complexType>
<xs:sequence>
<xs:element ref="testcase" minOccurs="0" maxOccurs="unbounded"/>
<xs:element ref="system-out" minOccurs="0" maxOccurs="1"/>
<xs:element ref="system-err" minOccurs="0" maxOccurs="1"/>
</xs:sequence>
<xs:attribute name="name" type="xs:string" use="required"/>
<xs:attribute name="tests" type="xs:string" use="required"/>
<xs:attribute name="failures" type="xs:string" use="required"/>
<xs:attribute name="errors" type="xs:string" use="required"/>
<xs:attribute name="disabled" type="xs:string" use="required"/>
<xs:attribute name="skipped" type="xs:string" use="required"/>
<xs:attribute name="time" type="xs:string" use="required"/>
<xs:attribute name="timestamp" type="xs:string" use="required"/>
</xs:complexType>
</xs:element>

<xs:element name="testsuites">
<xs:complexType>
<xs:sequence>
<xs:element ref="testsuite" minOccurs="0" maxOccurs="unbounded"/>
</xs:sequence>
<xs:attribute name="time" type="xs:string" use="required"/>
<xs:attribute name="tests" type="xs:string" use="required"/>
<xs:attribute name="failures" type="xs:string" use="required"/>
<xs:attribute name="disabled" type="xs:string" use="required"/>
<xs:attribute name="errors" type="xs:string" use="required"/>
</xs:complexType>
</xs:element>
</xs:schema>
70 changes: 70 additions & 0 deletions tmt/steps/report/junit/templates/subresults.xml.j2
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
{% extends "_base.xml.j2" %}

{#
This flavor changes the level of `<testsuite>` and `<testcase>` tags. The
`tmt.Result` becomes ``<testsuite>`` instead of ``testcase`` and
``<testcase>`` tags become ``tmt.SubResult``.
#}

{% block content %}
<testsuites disabled="0" errors="{{ RESULTS.errored | length }}" failures="{{ RESULTS.failed | length }}" tests="{{ RESULTS | length }}" time="{{ RESULTS.duration }}">

{% block testsuites %}
{% for result in RESULTS %}
{% set main_log = result.log | first | read_log %}
{% set main_log_failures = main_log | failures | e %}
{% set main_test_duration = result.duration | duration_to_seconds %}

<testsuite name="{{ result.name | trim | e }}" disabled="0" errors="{{ result.subresult.errored | length }}" failures="{{ result.subresult.failed | length }}" skipped="{{ result.subresult.skipped | length }}" tests="{{ result.subresult | length }}" time="{{ main_test_duration }}" timestamp="{{ result.start_time }}">

{#
TODO:
The `error`, `failure` and `skipped` tags are probably not allowed within the testsuite
{% if result.result.value == 'error' or result.result.value == 'warn' %}
<error type="error" message="{{ result.result.value | e }}">{{ main_log_failures }}</error>
{% elif result.result.value == 'fail' %}
<failure type="failure" message="{{ result.result.value | e }}">{{ main_log_failures }}</failure>
{% elif result.result.value == 'info' %}
<skipped type="skipped" message="{{ result.result.value | e }}">{{ main_log_failures }}</skipped>
{% endif %}
#}

{% if INCLUDE_OUTPUT_LOG and result_log %}
<system-out>{{ main_log | e }}</system-out>
{% endif %}

{% for subresult in result.subresult %}
{% set subresult_log = subresult.log | first | read_log %}
{% set subresult_log_failures = main_log | failures | e %}
{% set subresult_test_duration = subresult.duration | duration_to_seconds %}

<testcase name="{{ subresult.name | e }}" {% if subresult_test_duration %}time="{{ subresult_test_duration }}"{% endif %}>
{% if subresult.result.value == 'error' or subresult.result.value == 'warn' %}
<error type="error" message="{{ subresult.result.value | e }}">{{ subresult_log_failures }}</error>
{% elif subresult.result.value == 'fail' %}
<failure type="failure" message="{{ subresult.result.value | e }}">{{ subresult_log_failures }}</failure>
{% elif subresult.result.value == 'info' %}
<skipped type="skipped" message="{{ subresult.result.value | e }}">{{ subresult_log_failures }}</skipped>
{% endif %}

{% if INCLUDE_OUTPUT_LOG and subresult_log %}
<system-out>{{ subresult_log | e }}</system-out>
{% endif %}
</testcase>
{% endfor %}

{#
TODO:
Add properties from testcase into testsuite and test with Polarion?
#}
</testsuite>
{% endfor %}
{% endblock %}

{#
TODO:
Add properties
#}

</testsuites>
{% endblock %}

0 comments on commit 789e4ea

Please sign in to comment.