Generate JUnit files using Jinja2 instead of junit-xml

teemtee · Sep 6, 2024 · 222008f · 222008f
1 parent d83a790
commit 222008f
Show file tree

Hide file tree

Showing 14 changed files with 534 additions and 95 deletions.
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -44,7 +44,7 @@ repos:
  - "urllib3>=1.26.5, <2.0" # 1.26.16 / 2.0.4
 
  # report-junit
- - "junit_xml>=1.9"
+ - "lxml>=4.6.5"
 
  - "typing-extensions>=4.9.0; python_version < '3.13'"
  - "pytest"
@@ -62,6 +62,7 @@ repos:
  - "types-jinja2"
  - "types-babel"
  - "types-docutils"
+ - "types-lxml"
 
  pass_filenames: false
  args: [--config-file=pyproject.toml]
@@ -90,7 +91,7 @@ repos:
  - "urllib3>=1.26.5, <2.0" # 1.26.16 / 2.0.4
 
  # report-junit
- - "junit_xml>=1.9"
+ - "lxml>=4.6.5"
 
  - "typing-extensions>=4.9.0; python_version < '3.13'"
  - "pytest"
@@ -108,6 +109,7 @@ repos:
  - "types-jinja2"
  - "types-babel"
  - "types-docutils"
+ - "types-lxml"
 
  - repo: https://github.com/python-jsonschema/check-jsonschema
  rev: "0.29.1"

diff --git a/docs/releases.rst b/docs/releases.rst
@@ -4,6 +4,22 @@
  Releases
 ======================
 
+tmt-1.37
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The :ref:`/plugins/report/junit` report plugin now validates all the XML
+flavors against their respective XSD schemas and tries to prettify the final
+XML output. These functionalities are always disabled for ``custom`` flavors.
+The prettify functionality can be controlled for non-custom templates by
+``--prettify`` and ``--no-prettify`` arguments.
+
+The :ref:`/plugins/report/junit` report plugin now uses Jinja instead of
+``junit-xml`` library to generate the JUnit XMLs. It also adds support for a
+new ``--flavor`` argument. Using this argument the user can choose between a
+``default`` flavor, which keeps the current behavior untouched, and a
+``custom`` flavor where user must provide a custom template using a
+``--template-path`` argument.
+
 
 tmt-1.36
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

diff --git a/pyproject.toml b/pyproject.toml
@@ -61,7 +61,8 @@ provision-virtual = [
  ]
 provision-container = []
 report-junit = [
- "junit_xml>=1.9",
+ # Required to support XML parsing and checking the XSD schemas.
+ "lxml>=4.6.5",
  ]
 report-polarion = [
  "tmt[report-junit]",
@@ -139,6 +140,7 @@ dependencies = [
  "types-jinja2",
  "types-babel",
  "types-docutils",
+ "types-lxml",
  ]
 features = ["all"]
 
@@ -216,7 +218,7 @@ module = [
  "guestfs.*",
  "html2text.*",
  "fmf.*",
- "junit_xml.*",
+ "lxml.*",
  "libvirt.*",
  "nitrate.*",
  "pylero.*",

diff --git a/tests/report/junit/data/custom.xml.j2 b/tests/report/junit/data/custom.xml.j2
@@ -0,0 +1,6 @@
+<?xml version="1.0" ?>
+<tests>
+ {% for result in RESULTS %}
+ <test name="{{ result.name | e }}" value="{{ result.result.value | e }}"/>
+ {% endfor %}
+</tests>
diff --git a/tests/report/junit/data/main.fmf b/tests/report/junit/data/main.fmf
@@ -27,3 +27,5 @@
  /timeout:
  test: sleep 10
  duration: 2s
+ /escape"<speci&l>_chars:
+ test: "echo '<speci&l>\"chars'"
diff --git a/tests/report/junit/data/non-xml-custom.j2 b/tests/report/junit/data/non-xml-custom.j2
@@ -0,0 +1,3 @@
+{% for result in RESULTS %}
+name='{{ result.name | e }}',value='{{ result.result.value | e }}'
+{% endfor %}
diff --git a/tests/report/junit/test.sh b/tests/report/junit/test.sh
@@ -8,16 +8,60 @@ rlJournalStart
  rlPhaseEnd
 
  for method in tmt; do
- rlPhaseStartTest "$method"
+ rlPhaseStartTest "[$method] Basic format checks"
  rlRun "tmt run -avr execute -h $method report -h junit --file junit.xml 2>&1 >/dev/null | tee output" 2
- rlAssertGrep "2 tests passed, 2 tests failed and 2 errors" "output"
- rlAssertGrep '<testsuite disabled="0" errors="2" failures="2" name="/plan" skipped="0" tests="6"' "junit.xml"
+ rlAssertGrep "3 tests passed, 2 tests failed and 2 errors" "output"
+ rlAssertGrep '00:00:00 pass /test/shell/escape"<speci&l>_chars (on default-0)' "output"
+ rlAssertGrep '<testsuite name="/plan" disabled="0" errors="2" failures="2" skipped="0" tests="7"' "junit.xml"
  rlAssertGrep 'fail</failure>' "junit.xml"
+
+ # Test the escape of special characters
+ rlAssertGrep '<testcase name="/test/shell/escape&quot;&lt;speci&amp;l&gt;_chars">' "junit.xml"
+ rlAssertGrep '<system-out>&lt;speci&amp;l&gt;"chars' "junit.xml"
+
+ # Check there is no schema problem reported
+ rlAssertNotGrep 'The generated XML output is not a valid XML file or it is not valid against the XSD schema\.' "output"
+ rlPhaseEnd
+
+ rlPhaseStartTest "[$method] Check the flavor argument is working"
+ rlRun "tmt run -avr execute -h $method report -h junit --file junit.xml --flavor default 2>&1 >/dev/null | tee output" 2
+ rlAssertGrep "3 tests passed, 2 tests failed and 2 errors" "output"
+
+ # Check there is no schema problem reported
+ rlAssertNotGrep 'The generated XML output is not a valid XML file or it is not valid against the XSD schema\.' "output"
+ rlPhaseEnd
+
+ rlPhaseStartTest "[$method] Check the mutually exclusive arguments"
+ rlRun "tmt run -avr execute -h $method report -h junit --file junit.xml --flavor custom 2>&1 >/dev/null | tee output" 2
+ rlAssertGrep "The 'custom' flavor requires the '--template-path' argument." "output"
+
+ rlRun "tmt run -avr execute -h $method report -h junit --file junit.xml --template-path custom.xml.j2 2>&1 >/dev/null | tee output" 2
+ rlAssertGrep "The '--template-path' can be used only with '--flavor=custom'." "output"
+
+ rlPhaseEnd
+
+ rlPhaseStartTest "[$method] Check the 'custom' flavor with a custom XML template"
+ rlRun "tmt run -avr execute -h $method report -h junit --file custom-template-out.xml --template-path custom.xml.j2 --flavor custom 2>&1 >/dev/null | tee output" 2
+
+ # There must not be a schema check when using a custom flavor
+ rlAssertGrep "The 'custom' JUnit flavor is used, you are solely responsible for the validity of the XML schema\." "output"
+
+ rlAssertGrep '<test name="/test/beakerlib/fail" value="fail"/>' "custom-template-out.xml"
+ rlAssertGrep '<test name="/test/beakerlib/pass" value="pass"/>' "custom-template-out.xml"
+ rlAssertGrep '<test name="/test/shell/pass" value="pass"/>' "custom-template-out.xml"
+ rlAssertGrep '<test name="/test/shell/timeout" value="error"/>' "custom-template-out.xml"
+ rlAssertGrep '<test name="/test/shell/escape&quot;&lt;speci&amp;l&gt;_chars" value="pass"/>' "custom-template-out.xml"
+ rlPhaseEnd
+
+ rlPhaseStartTest "[$method] The 'custom' flavor with a custom **non-XML** template must not work"
+ rlRun "tmt run -avr execute -h $method report -h junit --file custom-template-out.xml --template-path non-xml-custom.j2 --flavor custom 2>&1 >/dev/null | tee output" 2
+
+ rlAssertGrep 'The generated XML output is not a valid XML file.' "output"
  rlPhaseEnd
  done
 
  rlPhaseStartCleanup
- rlRun "rm output junit.xml"
+ rlRun "rm output junit.xml custom-template-out.xml"
  rlRun "popd"
  rlPhaseEnd
 rlJournalEnd
diff --git a/tests/report/polarion/test.sh b/tests/report/polarion/test.sh
@@ -10,10 +10,11 @@ rlJournalStart
  rlPhaseStartTest
  rlRun "tmt run -avr execute report -h polarion --project-id RHELBASEOS --no-upload --planned-in RHEL-9.1.0 --file xunit.xml 2>&1 >/dev/null | tee output" 2
  rlAssertGrep "1 test passed, 1 test failed and 1 error" "output"
- rlAssertGrep '<testsuite disabled="0" errors="1" failures="1" name="/plan" skipped="0" tests="3"' "xunit.xml"
+ rlAssertGrep '<testsuite name="/plan" disabled="0" errors="1" failures="1" skipped="0" tests="3"' "xunit.xml"
  rlAssertGrep '<property name="polarion-project-id" value="RHELBASEOS" />' "xunit.xml"
  rlAssertGrep '<property name="polarion-testcase-id" value="BASEOS-10914" />' "xunit.xml"
  rlAssertGrep '<property name="polarion-custom-plannedin" value="RHEL-9.1.0" />' "xunit.xml"
+ rlAssertGrep "Maximum test time '2s' exceeded." "xunit.xml"
  rlPhaseEnd
 
  rlPhaseStartCleanup

diff --git a/tests/unit/test_report_junit.py b/tests/unit/test_report_junit.py
@@ -120,11 +120,14 @@ def test_pass(self, report_fix):
 
  report.go()
 
- assert_xml(out_file_path, """<?xml version="1.0" ?>
+ assert_xml(out_file_path, """<?xml version='1.0' encoding='utf-8'?>
 <testsuites disabled="0" errors="0" failures="0" tests="1" time="0.0">
- <testsuite disabled="0" errors="0" failures="0" name="name" skipped="0" tests="1" time="0">
- <testcase name="/pass"/>
- </testsuite>
+ <testsuite name="name" disabled="0" errors="0" failures="0" skipped="0" tests="1" time="0.0">
+ <testcase name="/pass">
+
+
+ </testcase>
+ </testsuite>
 </testsuites>
 """)
 
@@ -133,13 +136,13 @@ def test_info(self, report_fix):
  results.append(Result(result=ResultOutcome.INFO, name="/info", serial_number=1))
  report.go()
 
- assert_xml(out_file_path, """<?xml version="1.0" ?>
+ assert_xml(out_file_path, """<?xml version='1.0' encoding='utf-8'?>
 <testsuites disabled="0" errors="0" failures="0" tests="1" time="0.0">
-  <testsuite disabled="0" errors="0" failures="0" name="name" skipped="1" tests="1" time="0">
-  <testcase name="/info">
-  <skipped type="skipped" message="info"/>
-  </testcase>
-  </testsuite>
+ <testsuite name="name" disabled="0" errors="0" failures="0" skipped="1" tests="1" time="0.0">
+ <testcase name="/info">
+ <skipped type="skipped" message="info"/>
+ </testcase>
+ </testsuite>
 </testsuites>
 """)
 
@@ -148,13 +151,13 @@ def test_warn(self, report_fix):
  results.append(Result(result=ResultOutcome.WARN, name="/warn", serial_number=1))
  report.go()
 
- assert_xml(out_file_path, """<?xml version="1.0" ?>
+ assert_xml(out_file_path, """<?xml version='1.0' encoding='utf-8'?>
 <testsuites disabled="0" errors="1" failures="0" tests="1" time="0.0">
-  <testsuite disabled="0" errors="1" failures="0" name="name" skipped="0" tests="1" time="0">
-  <testcase name="/warn">
-  <error type="error" message="warn"/>
-  </testcase>
-  </testsuite>
+ <testsuite name="name" disabled="0" errors="1" failures="0" skipped="0" tests="1" time="0.0">
+ <testcase name="/warn">
+ <error type="error" message="warn"/>
+ </testcase>
+ </testsuite>
 </testsuites>
 """)
 
@@ -163,13 +166,13 @@ def test_error(self, report_fix):
  results.append(Result(result=ResultOutcome.ERROR, name="/error", serial_number=1))
  report.go()
 
- assert_xml(out_file_path, """<?xml version="1.0" ?>
+ assert_xml(out_file_path, """<?xml version='1.0' encoding='utf-8'?>
 <testsuites disabled="0" errors="1" failures="0" tests="1" time="0.0">
-  <testsuite disabled="0" errors="1" failures="0" name="name" skipped="0" tests="1" time="0">
-  <testcase name="/error">
-  <error type="error" message="error"/>
-  </testcase>
-  </testsuite>
+ <testsuite name="name" disabled="0" errors="1" failures="0" skipped="0" tests="1" time="0.0">
+ <testcase name="/error">
+ <error type="error" message="error"/>
+ </testcase>
+ </testsuite>
 </testsuites>
 """)
 
@@ -178,12 +181,12 @@ def test_fail(self, report_fix):
  results.append(Result(result=ResultOutcome.FAIL, name="/fail", serial_number=1))
  report.go()
 
- assert_xml(out_file_path, """<?xml version="1.0" ?>
+ assert_xml(out_file_path, """<?xml version='1.0' encoding='utf-8'?>
 <testsuites disabled="0" errors="0" failures="1" tests="1" time="0.0">
-  <testsuite disabled="0" errors="0" failures="1" name="name" skipped="0" tests="1" time="0">
-  <testcase name="/fail">
-  <failure type="failure" message="fail"/>
-  </testcase>
-  </testsuite>
+ <testsuite name="name" disabled="0" errors="0" failures="1" skipped="0" tests="1" time="0.0">
+ <testcase name="/fail">
+ <failure type="failure" message="fail"/>
+ </testcase>
+ </testsuite>
 </testsuites>
 """)