Skip to content

Commit

Permalink
Add support for subresults in junit report plugin flavor
Browse files Browse the repository at this point in the history 
Define an experimental junit flavor for subresults.
  • Loading branch information
@seberm
seberm committed Sep 16, 2024
1 parent 4b3394b commit 6620d62
Show file tree
Hide file tree
Showing 5 changed files with 225 additions and 6 deletions.
11 changes: 11 additions & 0 deletions docs/releases.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,17 @@
Releases
======================

tmt-1.38.0
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

The :ref:`/plugins/report/junit` report plugin now supports a new
``subresults`` JUnit flavor. This flavor adds support for tmt subresults and
changes the level of ``<testsuite>`` and ``<testcase>`` tags. By using this
flavor, the ``tmt.Result`` tags become ``<testsuite>`` tags with one
``<testcase>`` tag representing the parent result, and possible additional
``<testcase>`` tags for each ``tmt.SubResult``.


tmt-1.37
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

Expand Down
16 changes: 16 additions & 0 deletions tests/report/junit/test.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,22 @@ rlJournalStart

rlAssertGrep 'The generated XML output is not a valid XML file.' "output"
rlPhaseEnd

rlPhaseStartTest "[$method] Check the 'subresults' flavor"
rlRun "tmt run -avr execute -h $method report -h junit --file subresults-out.xml --flavor subresults 2>&1 >/dev/null | tee output" 2

# Parent result recorded in testuite tag
rlAssertGrep '<testsuite name="/test/beakerlib/fail" disabled="0" errors="1" failures="0" skipped="0" tests="1"' "subresults-out.xml"
rlAssertGrep '<testsuite name="/test/beakerlib/pass" disabled="0" errors="0" failures="0" skipped="0" tests="1" ' "subresults-out.xml"
rlAssertGrep '<testsuite name="/test/shell/fail" disabled="0" errors="1" failures="0" skipped="0" tests="1"'

# Parent result testsuite must have its respective testcase tag
rlAssertGrep '<testcase name="/test/beakerlib/fail">' "subresults-out.xml"
rlAssertGrep '<testcase name="/test/beakerlib/pass">' "subresults-out.xml"

# TODO: Add check for additional subresults as soon as they get saved by:
# - https://github.com/teemtee/tmt/pull/3200
rlPhaseEnd
done

rlPhaseStartCleanup
Expand Down
33 changes: 27 additions & 6 deletions tmt/steps/report/junit.py
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import dataclasses
import functools
from collections.abc import Iterator
from typing import TYPE_CHECKING, Any, Optional, TypedDict, cast, overload
from typing import TYPE_CHECKING, Any, Optional, TypedDict, Union, cast, overload

from jinja2 import FileSystemLoader, select_autoescape

Expand Down Expand Up @@ -83,10 +83,30 @@ class PropertyDict(TypedDict):
name: str
value: str

def __init__(self, wrapped: tmt.Result) -> None:
def __init__(
self,
wrapped: Union[tmt.Result, tmt.result.SubResult],
subresults_context_class: 'type[ResultsContext]') -> None:

self._wrapped = wrapped
self._subresults_context_class = subresults_context_class
self._properties: dict[str, str] = {}

@property
def subresult(self) -> 'ResultsContext':
"""
Override the ``tmt.Result.subresult`` and wrap all the ``tmt.result.SubResult`` instances
into the ``ResultsContext``.
"""

# `tmt.result.SubResult.subresult` is not defined, just raise the AttributeError to silent
# the typing errors.
if isinstance(self._wrapped, tmt.result.SubResult):
raise AttributeError(
f"'{self._wrapped.__class__.__name__} object has no attribute 'subresult'")

return self._subresults_context_class(self._wrapped.subresult)

@property
def properties(self) -> list[PropertyDict]:
return [{'name': k, 'value': v} for k, v in self._properties.items()]
Expand All @@ -108,9 +128,10 @@ class ResultsContext:
wraps all the :py:class:`tmt.Result` instances into the :py:class:`ResultWrapper`.
"""

def __init__(self, results: list[tmt.Result]) -> None:
# Decorate all the tmt.Results with more attributes
self._results: list[ResultWrapper] = [ResultWrapper(r) for r in results]
def __init__(self, results: Union[list[tmt.Result], list[tmt.result.SubResult]]) -> None:
""" Decorate/wrap all the ``tmt.Results`` with more attributes """
self._results: list[ResultWrapper] = [
ResultWrapper(r, subresults_context_class=self.__class__) for r in results]

def __iter__(self) -> Iterator[ResultWrapper]:
""" Possibility to iterate over results by iterating an instance """
Expand Down Expand Up @@ -306,7 +327,7 @@ class ReportJUnitData(tmt.steps.report.ReportStepData):
flavor: str = field(
default=DEFAULT_FLAVOR_NAME,
option='--flavor',
choices=[DEFAULT_FLAVOR_NAME, CUSTOM_FLAVOR_NAME],
choices=[DEFAULT_FLAVOR_NAME, CUSTOM_FLAVOR_NAME, 'subresults'],
help='Name of a JUnit flavor to generate.')

template_path: Optional[Path] = field(
Expand Down
81 changes: 81 additions & 0 deletions tmt/steps/report/junit/schemas/subresults.xsd
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
<?xml version="1.0" encoding="UTF-8" ?>

<!--
This schema extends the `default` flavor and adds support for tmt
subresults. It allows multiple occurrences of testsuite elements inside a
<testsuites>.
The <testsuite> is mapped to `tmt.Result` and the <testcase> is mapped to
`tmt.SubResult`.
-->

<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema">

<xs:element name="failure">
<xs:complexType mixed="true">
<xs:attribute name="type" type="xs:string" use="required"/>
<xs:attribute name="message" type="xs:string" use="required"/>
</xs:complexType>
</xs:element>

<xs:element name="error">
<xs:complexType mixed="true">
<xs:attribute name="type" type="xs:string" use="required"/>
<xs:attribute name="message" type="xs:string" use="required"/>
</xs:complexType>
</xs:element>

<xs:element name="skipped">
<xs:complexType mixed="true">
<xs:attribute name="type" type="xs:string" use="required"/>
<xs:attribute name="message" type="xs:string" use="required"/>
</xs:complexType>
</xs:element>
<xs:element name="system-err" type="xs:string"/>
<xs:element name="system-out" type="xs:string"/>

<xs:element name="testcase">
<xs:complexType>
<xs:sequence>
<xs:element ref="skipped" minOccurs="0" maxOccurs="1"/>
<xs:element ref="error" minOccurs="0" maxOccurs="1"/>
<xs:element ref="failure" minOccurs="0" maxOccurs="1"/>
<xs:element ref="system-out" minOccurs="0" maxOccurs="1"/>
<xs:element ref="system-err" minOccurs="0" maxOccurs="1"/>
</xs:sequence>
<xs:attribute name="name" type="xs:string" use="required"/>
<xs:attribute name="time" type="xs:string" use="optional"/>
</xs:complexType>
</xs:element>

<xs:element name="testsuite">
<xs:complexType>
<xs:sequence>
<xs:element ref="testcase" minOccurs="0" maxOccurs="unbounded"/>
<xs:element ref="system-out" minOccurs="0" maxOccurs="1"/>
<xs:element ref="system-err" minOccurs="0" maxOccurs="1"/>
</xs:sequence>
<xs:attribute name="name" type="xs:string" use="required"/>
<xs:attribute name="tests" type="xs:string" use="required"/>
<xs:attribute name="failures" type="xs:string" use="required"/>
<xs:attribute name="errors" type="xs:string" use="required"/>
<xs:attribute name="disabled" type="xs:string" use="required"/>
<xs:attribute name="skipped" type="xs:string" use="required"/>
<xs:attribute name="time" type="xs:string" use="required"/>
<xs:attribute name="timestamp" type="xs:string" use="required"/>
</xs:complexType>
</xs:element>

<xs:element name="testsuites">
<xs:complexType>
<xs:sequence>
<xs:element ref="testsuite" minOccurs="0" maxOccurs="unbounded"/>
</xs:sequence>
<xs:attribute name="time" type="xs:string" use="required"/>
<xs:attribute name="tests" type="xs:string" use="required"/>
<xs:attribute name="failures" type="xs:string" use="required"/>
<xs:attribute name="disabled" type="xs:string" use="required"/>
<xs:attribute name="errors" type="xs:string" use="required"/>
</xs:complexType>
</xs:element>
</xs:schema>
90 changes: 90 additions & 0 deletions tmt/steps/report/junit/templates/subresults.xml.j2
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
{% extends "_base.xml.j2" %}

{#
This flavor changes the level of `<testsuite>` and `<testcase>` tags. The
`tmt.Result` becomes ``<testsuite>`` instead of ``testcase`` and
``<testcase>`` tags become ``tmt.SubResult``.
#}

{% block content %}
<testsuites disabled="0" errors="{{ RESULTS.errored | length }}" failures="{{ RESULTS.failed | length }}" tests="{{ RESULTS | length }}" time="{{ RESULTS.duration }}">

{% block testsuites %}
{% for result in RESULTS %}
{% set main_log = result.log | first | read_log %}
{% set main_log_failures = main_log | failures | e %}
{% set main_test_duration = result.duration | duration_to_seconds %}

{# TODO: Fix the test counts in testsuite tag #}
{# TODO: Also fix the counts because now with an additional testcase for parent results, the counts will not match. #}

<testsuite name="{{ result.name | trim | e }}" disabled="0" errors="{{ result.subresult.errored | length }}" failures="{{ result.subresult.failed | length }}" skipped="{{ result.subresult.skipped | length }}" tests="{{ result.subresult | length }}" time="{{ main_test_duration }}" timestamp="{{ result.start_time }}">

{#
Always include a `testcase` representing the main result.
The `error/failure/skipped` tags must not exists inside a
`testsuite`, they are only allowed inside of a `testcase`.
#}
<testcase name="{{ result.name | e }}" {% if result_test_duration %}time="{{ result_test_duration }}"{% endif %}>
{% if result.result.value == 'error' or result.result.value == 'warn' %}
<error type="error" message="{{ result.result.value | e }}">{{ main_log_failures }}</error>
{% elif result.result.value == 'fail' %}
<failure type="failure" message="{{ result.result.value | e }}">{{ main_log_failures }}</failure>
{% elif result.result.value == 'info' %}
<skipped type="skipped" message="{{ result.result.value | e }}">{{ main_log_failures }}</skipped>
{% endif %}

{% if INCLUDE_OUTPUT_LOG and main_log %}
<system-out>{{ main_log | e }}</system-out>
{% endif %}
</testcase>

{% for subresult in result.subresult %}
{% set subresult_log = subresult.log | first | read_log %}
{% set subresult_log_failures = main_log | failures | e %}
{% set subresult_test_duration = subresult.duration | duration_to_seconds %}

<testcase name="{{ subresult.name | e }}" {% if subresult_test_duration %}time="{{ subresult_test_duration }}"{% endif %}>
{% if subresult.result.value == 'error' or subresult.result.value == 'warn' %}
<error type="error" message="{{ subresult.result.value | e }}">{{ subresult_log_failures }}</error>
{% elif subresult.result.value == 'fail' %}
<failure type="failure" message="{{ subresult.result.value | e }}">{{ subresult_log_failures }}</failure>
{% elif subresult.result.value == 'info' %}
<skipped type="skipped" message="{{ subresult.result.value | e }}">{{ subresult_log_failures }}</skipped>
{% endif %}

{% if INCLUDE_OUTPUT_LOG and subresult_log %}
<system-out>{{ subresult_log | e }}</system-out>
{% endif %}
</testcase>
{% endfor %}

{#
TODO:
Optionally include testcase properties?
Optionally add the result properties
{% if result.properties is defined %}
{% with properties=result.properties %}
include "includes/_properties.xml.j2"
{% endwith %}
{% endif %}
#}
</testsuite>
{% endfor %}
{% endblock %}

{#
TODO:
Optionally include testsuites properties if they are defined.
Optionally include the properties section in testsuites tag
{% if RESULTS.properties is defined %}
{% with properties=RESULTS.properties %}
include "includes/_properties.xml.j2"
{% endwith %}
{% endif %}
#}

</testsuites>
{% endblock %}

0 comments on commit 6620d62

Please sign in to comment.