Generate JUnit files using Jinja2 instead of junit-xml

teemtee · Sep 6, 2024 · 9ee1638 · 9ee1638
1 parent d83a790
commit 9ee1638
Show file tree

Hide file tree

Showing 14 changed files with 533 additions and 96 deletions.
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -44,7 +44,7 @@ repos:
           - "urllib3>=1.26.5, <2.0"  # 1.26.16 / 2.0.4
 
           # report-junit
-          - "junit_xml>=1.9"
+          - "lxml>=4.6.5"
 
           - "typing-extensions>=4.9.0; python_version < '3.13'"
           - "pytest"
@@ -62,6 +62,7 @@ repos:
           - "types-jinja2"
           - "types-babel"
           - "types-docutils"
+          - "types-lxml"
 
         pass_filenames: false
         args: [--config-file=pyproject.toml]
@@ -90,7 +91,7 @@ repos:
           - "urllib3>=1.26.5, <2.0"  # 1.26.16 / 2.0.4
 
           # report-junit
-          - "junit_xml>=1.9"
+          - "lxml>=4.6.5"
 
           - "typing-extensions>=4.9.0; python_version < '3.13'"
           - "pytest"
@@ -108,6 +109,7 @@ repos:
           - "types-jinja2"
           - "types-babel"
           - "types-docutils"
+          - "types-lxml"
 
   - repo: https://github.com/python-jsonschema/check-jsonschema
     rev: "0.29.1"

diff --git a/docs/releases.rst b/docs/releases.rst
@@ -4,6 +4,22 @@
     Releases
 ======================
 
+tmt-1.37
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The :ref:`/plugins/report/junit` report plugin now validates all the XML
+flavors against their respective XSD schemas and tries to prettify the final
+XML output. These functionalities are always disabled for ``custom`` flavors.
+The prettify functionality can be controlled for non-custom templates by
+``--prettify`` and ``--no-prettify`` arguments.
+
+The :ref:`/plugins/report/junit` report plugin now uses Jinja instead of
+``junit-xml`` library to generate the JUnit XMLs. It also adds support for a
+new ``--flavor`` argument. Using this argument the user can choose between a
+``default`` flavor, which keeps the current behavior untouched, and a
+``custom`` flavor where user must provide a custom template using a
+``--template-path`` argument.
+
 
 tmt-1.36
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

diff --git a/pyproject.toml b/pyproject.toml
@@ -61,7 +61,8 @@ provision-virtual = [
     ]
 provision-container = []
 report-junit = [
-    "junit_xml>=1.9",
+    # Required to support XML parsing and checking the XSD schemas.
+    "lxml>=4.6.5",
     ]
 report-polarion = [
     "tmt[report-junit]",
@@ -139,6 +140,7 @@ dependencies = [
     "types-jinja2",
     "types-babel",
     "types-docutils",
+    "types-lxml",
     ]
 features = ["all"]
 
@@ -216,7 +218,7 @@ module = [
     "guestfs.*",
     "html2text.*",
     "fmf.*",
-    "junit_xml.*",
+    "lxml.*",
     "libvirt.*",
     "nitrate.*",
     "pylero.*",

diff --git a/tests/report/junit/data/custom.xml.j2 b/tests/report/junit/data/custom.xml.j2
@@ -0,0 +1,6 @@
+<?xml version="1.0" ?>
+<tests>
+    {% for result in RESULTS %}
+        <test name="{{ result.name | e }}" value="{{ result.result.value | e }}"/>
+    {% endfor %}
+</tests>
diff --git a/tests/report/junit/data/main.fmf b/tests/report/junit/data/main.fmf
@@ -27,3 +27,5 @@
         /timeout:
             test: sleep 10
             duration: 2s
+        /escape"<speci&l>_chars:
+            test: "echo '<speci&l>\"chars'"
diff --git a/tests/report/junit/data/non-xml-custom.j2 b/tests/report/junit/data/non-xml-custom.j2
@@ -0,0 +1,3 @@
+{% for result in RESULTS %}
+name='{{ result.name | e }}',value='{{ result.result.value | e }}'
+{% endfor %}
diff --git a/tests/report/junit/test.sh b/tests/report/junit/test.sh
@@ -8,16 +8,60 @@ rlJournalStart
     rlPhaseEnd
 
     for method in tmt; do
-        rlPhaseStartTest "$method"
+        rlPhaseStartTest "[$method] Basic format checks"
             rlRun "tmt run -avr execute -h $method report -h junit --file junit.xml 2>&1 >/dev/null | tee output" 2
-            rlAssertGrep "2 tests passed, 2 tests failed and 2 errors" "output"
-            rlAssertGrep '<testsuite disabled="0" errors="2" failures="2" name="/plan" skipped="0" tests="6"' "junit.xml"
+            rlAssertGrep "3 tests passed, 2 tests failed and 2 errors" "output"
+            rlAssertGrep '00:00:00 pass /test/shell/escape"<speci&l>_chars (on default-0)' "output"
+            rlAssertGrep '<testsuite name="/plan" disabled="0" errors="2" failures="2" skipped="0" tests="7"' "junit.xml"
             rlAssertGrep 'fail</failure>' "junit.xml"
+
+            # Test the escape of special characters
+            rlAssertGrep '<testcase name="/test/shell/escape&quot;&lt;speci&amp;l&gt;_chars">' "junit.xml"
+            rlAssertGrep '<system-out>&lt;speci&amp;l&gt;"chars' "junit.xml"
+
+            # Check there is no schema problem reported
+            rlAssertNotGrep 'The generated XML output is not a valid XML file or it is not valid against the XSD schema\.' "output"
+        rlPhaseEnd
+
+        rlPhaseStartTest "[$method] Check the flavor argument is working"
+            rlRun "tmt run -avr execute -h $method report -h junit --file junit.xml --flavor default 2>&1 >/dev/null | tee output" 2
+            rlAssertGrep "3 tests passed, 2 tests failed and 2 errors" "output"
+
+            # Check there is no schema problem reported
+            rlAssertNotGrep 'The generated XML output is not a valid XML file or it is not valid against the XSD schema\.' "output"
+        rlPhaseEnd
+
+        rlPhaseStartTest "[$method] Check the mutually exclusive arguments"
+            rlRun "tmt run -avr execute -h $method report -h junit --file junit.xml --flavor custom 2>&1 >/dev/null | tee output" 2
+            rlAssertGrep "The 'custom' flavor requires the '--template-path' argument." "output"
+
+            rlRun "tmt run -avr execute -h $method report -h junit --file junit.xml --template-path custom.xml.j2 2>&1 >/dev/null | tee output" 2
+            rlAssertGrep "The '--template-path' can be used only with '--flavor=custom'." "output"
+
+        rlPhaseEnd
+
+        rlPhaseStartTest "[$method] Check the 'custom' flavor with a custom XML template"
+            rlRun "tmt run -avr execute -h $method report -h junit --file custom-template-out.xml --template-path custom.xml.j2 --flavor custom 2>&1 >/dev/null | tee output" 2
+
+            # There must not be a schema check when using a custom flavor
+            rlAssertGrep "The 'custom' JUnit flavor is used, you are solely responsible for the validity of the XML schema\." "output"
+
+            rlAssertGrep '<test name="/test/beakerlib/fail" value="fail"/>' "custom-template-out.xml"
+            rlAssertGrep '<test name="/test/beakerlib/pass" value="pass"/>' "custom-template-out.xml"
+            rlAssertGrep '<test name="/test/shell/pass" value="pass"/>' "custom-template-out.xml"
+            rlAssertGrep '<test name="/test/shell/timeout" value="error"/>' "custom-template-out.xml"
+            rlAssertGrep '<test name="/test/shell/escape&quot;&lt;speci&amp;l&gt;_chars" value="pass"/>' "custom-template-out.xml"
+        rlPhaseEnd
+
+        rlPhaseStartTest "[$method] The 'custom' flavor with a custom **non-XML** template must not work"
+            rlRun "tmt run -avr execute -h $method report -h junit --file custom-template-out.xml --template-path non-xml-custom.j2 --flavor custom 2>&1 >/dev/null | tee output" 2
+
+            rlAssertGrep 'The generated XML output is not a valid XML file.' "output"
         rlPhaseEnd
     done
 
     rlPhaseStartCleanup
-        rlRun "rm output junit.xml"
+        rlRun "rm output junit.xml custom-template-out.xml"
         rlRun "popd"
     rlPhaseEnd
 rlJournalEnd
diff --git a/tests/report/polarion/test.sh b/tests/report/polarion/test.sh
@@ -10,10 +10,11 @@ rlJournalStart
     rlPhaseStartTest
         rlRun "tmt run -avr execute report -h polarion --project-id RHELBASEOS --no-upload --planned-in RHEL-9.1.0 --file xunit.xml 2>&1 >/dev/null | tee output" 2
         rlAssertGrep "1 test passed, 1 test failed and 1 error" "output"
-        rlAssertGrep '<testsuite disabled="0" errors="1" failures="1" name="/plan" skipped="0" tests="3"' "xunit.xml"
+        rlAssertGrep '<testsuite name="/plan" disabled="0" errors="1" failures="1" skipped="0" tests="3"' "xunit.xml"
         rlAssertGrep '<property name="polarion-project-id" value="RHELBASEOS" />' "xunit.xml"
         rlAssertGrep '<property name="polarion-testcase-id" value="BASEOS-10914" />' "xunit.xml"
         rlAssertGrep '<property name="polarion-custom-plannedin" value="RHEL-9.1.0" />' "xunit.xml"
+        rlAssertGrep "Maximum test time '2s' exceeded." "xunit.xml"
     rlPhaseEnd
 
     rlPhaseStartCleanup

diff --git a/tests/unit/test_report_junit.py b/tests/unit/test_report_junit.py
@@ -120,11 +120,14 @@ def test_pass(self, report_fix):
 
         report.go()
 
-        assert_xml(out_file_path, """<?xml version="1.0" ?>
+        assert_xml(out_file_path, """<?xml version='1.0' encoding='utf-8'?>
 <testsuites disabled="0" errors="0" failures="0" tests="1" time="0.0">
-    <testsuite disabled="0" errors="0" failures="0" name="name" skipped="0" tests="1" time="0">
-        <testcase name="/pass"/>
-    </testsuite>
+  <testsuite name="name" disabled="0" errors="0" failures="0" skipped="0" tests="1" time="0.0">
+    <testcase name="/pass">
+
+
+                </testcase>
+  </testsuite>
 </testsuites>
 """)
 
@@ -133,13 +136,13 @@ def test_info(self, report_fix):
         results.append(Result(result=ResultOutcome.INFO, name="/info", serial_number=1))
         report.go()
 
-        assert_xml(out_file_path, """<?xml version="1.0" ?>
+        assert_xml(out_file_path, """<?xml version='1.0' encoding='utf-8'?>
 <testsuites disabled="0" errors="0" failures="0" tests="1" time="0.0">
-    <testsuite disabled="0" errors="0" failures="0" name="name" skipped="1" tests="1" time="0">
-        <testcase name="/info">
-            <skipped type="skipped" message="info"/>
-        </testcase>
-    </testsuite>
+  <testsuite name="name" disabled="0" errors="0" failures="0" skipped="1" tests="1" time="0.0">
+    <testcase name="/info">
+      <skipped type="skipped" message="info"/>
+    </testcase>
+  </testsuite>
 </testsuites>
 """)
 
@@ -148,13 +151,13 @@ def test_warn(self, report_fix):
         results.append(Result(result=ResultOutcome.WARN, name="/warn", serial_number=1))
         report.go()
 
-        assert_xml(out_file_path, """<?xml version="1.0" ?>
+        assert_xml(out_file_path, """<?xml version='1.0' encoding='utf-8'?>
 <testsuites disabled="0" errors="1" failures="0" tests="1" time="0.0">
-    <testsuite disabled="0" errors="1" failures="0" name="name" skipped="0" tests="1" time="0">
-        <testcase name="/warn">
-            <error type="error" message="warn"/>
-        </testcase>
-    </testsuite>
+  <testsuite name="name" disabled="0" errors="1" failures="0" skipped="0" tests="1" time="0.0">
+    <testcase name="/warn">
+      <error type="error" message="warn"/>
+    </testcase>
+  </testsuite>
 </testsuites>
 """)
 
@@ -163,13 +166,13 @@ def test_error(self, report_fix):
         results.append(Result(result=ResultOutcome.ERROR, name="/error", serial_number=1))
         report.go()
 
-        assert_xml(out_file_path, """<?xml version="1.0" ?>
+        assert_xml(out_file_path, """<?xml version='1.0' encoding='utf-8'?>
 <testsuites disabled="0" errors="1" failures="0" tests="1" time="0.0">
-    <testsuite disabled="0" errors="1" failures="0" name="name" skipped="0" tests="1" time="0">
-        <testcase name="/error">
-            <error type="error" message="error"/>
-        </testcase>
-    </testsuite>
+  <testsuite name="name" disabled="0" errors="1" failures="0" skipped="0" tests="1" time="0.0">
+    <testcase name="/error">
+      <error type="error" message="error"/>
+    </testcase>
+  </testsuite>
 </testsuites>
 """)
 
@@ -178,12 +181,12 @@ def test_fail(self, report_fix):
         results.append(Result(result=ResultOutcome.FAIL, name="/fail", serial_number=1))
         report.go()
 
-        assert_xml(out_file_path, """<?xml version="1.0" ?>
+        assert_xml(out_file_path, """<?xml version='1.0' encoding='utf-8'?>
 <testsuites disabled="0" errors="0" failures="1" tests="1" time="0.0">
-    <testsuite disabled="0" errors="0" failures="1" name="name" skipped="0" tests="1" time="0">
-        <testcase name="/fail">
-            <failure type="failure" message="fail"/>
-        </testcase>
-    </testsuite>
+  <testsuite name="name" disabled="0" errors="0" failures="1" skipped="0" tests="1" time="0.0">
+    <testcase name="/fail">
+      <failure type="failure" message="fail"/>
+    </testcase>
+  </testsuite>
 </testsuites>
 """)