From fd8a09795aa2ea2a091906efb92124eb85d49e05 Mon Sep 17 00:00:00 2001 From: Joey Vagedes Date: Thu, 27 Jun 2024 16:29:24 -0700 Subject: [PATCH] Host Based Unit Test updates (#837) ## Description Updates the host-based unit test runner to fail if a unit test executable returns successfully, but has no test results, or if a test suite generated from a unit test executable does not contain any tests. The issues above indicate configuration errors in the unit test source code itself and indicates to the developer that changes to the unit test need to be made. Updates the README.md file for the UnitTestFrameworkPkg to correct inaccurate information regarding code coverage and provide information on how to consolidate and generate unit test html reports. - [ ] Impacts functionality? - **Functionality** - Does the change ultimately impact how firmware functions? - Examples: Add a new library, publish a new PPI, update an algorithm, ... - [ ] Impacts security? - **Security** - Does the change have a direct security impact on an application, flow, or firmware? - Examples: Crypto algorithm change, buffer overflow fix, parameter validation improvement, ... - [x] Breaking change? - **Breaking change** - Will anyone consuming this change experience a break in build or boot behavior? - Examples: Add a new library class, move a module to a different repo, call a function in a new library class in a pre-existing module, ... - [ ] Includes tests? - **Tests** - Does the change include any explicit test code? - Examples: Unit tests, integration tests, robot tests, ... - [x] Includes documentation? - **Documentation** - Does the change contain explicit documentation additions outside direct code modifications (and comments)? - Examples: Update readme file, add feature readme file, link to documentation on an a separate Web page, ... ## How This Was Tested N/A ## Integration Instructions If any host based unit tests were written incorrectly, and there exists a test with no test suites, or a test suite with no tests, The host based unit test runner will now fail. These tests will need to be corrected. --------- Signed-off-by: Joey Vagedes --- .../HostBasedUnitTestRunner.py | 20 ++++++++++++++- UnitTestFrameworkPkg/ReadMe.md | 25 ++++++++++++++++--- .../UnitTestFrameworkPkg.ci.yaml | 3 ++- 3 files changed, 43 insertions(+), 5 deletions(-) diff --git a/BaseTools/Plugin/HostBasedUnitTestRunner/HostBasedUnitTestRunner.py b/BaseTools/Plugin/HostBasedUnitTestRunner/HostBasedUnitTestRunner.py index 8bd3e37c27..9d70dfbd54 100644 --- a/BaseTools/Plugin/HostBasedUnitTestRunner/HostBasedUnitTestRunner.py +++ b/BaseTools/Plugin/HostBasedUnitTestRunner/HostBasedUnitTestRunner.py @@ -99,6 +99,7 @@ def do_post_build(self, thebuilder): """).strip()) return 0 + error_messages = [] # MU_CHANGE- Check for invalid tests for test in testList: # Configure output name if test uses cmocka. shell_env.set_shell_var( @@ -120,7 +121,20 @@ def do_post_build(self, thebuilder): for xml_result_file in xml_results_list: root = xml.etree.ElementTree.parse( xml_result_file).getroot() + # MU_CHANGE [BEGIN] - Check for invalid tests + if len(root) == 0: + error_messages.append(f'{os.path.basename(test)} did not generate a test suite(s).') + error_messages.append(' Review source code to ensure Test suites are created and tests ' + ' are registered!') + failure_count += 1 for suite in root: + if len(suite) == 0: + error_messages.append(f'TestSuite [{suite.attrib["name"]}] for test {test} did not ' + 'contain a test case(s).') + error_messages.append(' Review source code to ensure test cases are registered to ' + 'the suite!') + failure_count += 1 + # MU_CHANGE [END] - Check for invalid tests for case in suite: for result in case: if result.tag == 'failure': @@ -130,7 +144,7 @@ def do_post_build(self, thebuilder): " %s - %s" % (case.attrib['name'], result.text)) failure_count += 1 - if thebuilder.env.GetValue("CODE_COVERAGE") != "FALSE": + if thebuilder.env.GetValue("CODE_COVERAGE", "FALSE") == "TRUE": # MU_CHANGE if thebuilder.env.GetValue("TOOL_CHAIN_TAG") == "GCC5": ret = self.gen_code_coverage_gcc(thebuilder) if ret != 0: @@ -151,6 +165,10 @@ def do_post_build(self, thebuilder): return -1 # MU_CHANGE end - reformat coverage data + # MU_CHANGE [BEGIN] - Check for invalid tests + for error in error_messages: + logging.error(error) + # MU_CHANGE [END] - Check for invalid tests return failure_count def gen_code_coverage_gcc(self, thebuilder): diff --git a/UnitTestFrameworkPkg/ReadMe.md b/UnitTestFrameworkPkg/ReadMe.md index dd497617a9..e534e8c6d5 100644 --- a/UnitTestFrameworkPkg/ReadMe.md +++ b/UnitTestFrameworkPkg/ReadMe.md @@ -1491,6 +1491,18 @@ GTEST_OUTPUT=xml: This mode is used by the test running plugin to aggregate the results for CI test status reporting in the web view. +### XML Reporting Test Consolidation + +There exists multiple tools for consolidating and generating consolidated test results from the +test xml files that are generated. The arguably most convenient tool available is the +`xunit-viewer` node package, installed via `npm install -g xunit-viewer`. This tool can +consolidate all generated xml reports and create an html or cli summary of test results. + +The following command will generate a consolidated report at `.html` and +also print summary overview of the test results to the command line: + +`xunit-viewer --results Build --output .html --console` + ### Code Coverage Code coverage can be enabled for Host based Unit Tests with `CODE_COVERAGE=TRUE`, which generates a cobertura report @@ -1498,10 +1510,17 @@ per package tested, and combined cobertura report for all packages tested. The p present at `Build//HostTest//_coverage.xml`. The overall cobertura report will be present at `Build/coverage.xml` -Code coverage generation has two config knobs: +Code coverage generation has three config knobs. Each can be turned on/off by setting it to TRUE +or FALSE e.g. `CC_REORGANIZE=TRUE`: + +1. `CC_REORGANIZE`: Controls if code coverage results are re-formatted into a "by-inf" folder + structure rather than the default "by-test" folder structure. Default: `TRUE` +1. `CC_FULL`: Generates zero'd out coverage data for untested source files in the package. + Default: `FALSE` +1. `CC_FLATTEN`: Groups all source files together, rather than by INF. Default: `FALSE` -1. `CC_FULL`: If set to `TRUE`, will generate zero'd out coverage data for untested source files in the package. -2. `CC_FLATTEN`: If Set to `TRUE`, will group all source files together, rather than by INF. +** NOTE: `CC_FULL` and `CC_FLATTEN` values only matter if `CC_REORGANIZE=TRUE`, as they only +effect how the coverage report is reorganized. **TIP: `CC_FLATTEN=TRUE/FALSE` will produce different coverage percentage results as `TRUE` de-duplicates source files that are consumed by multiple INFs. diff --git a/UnitTestFrameworkPkg/UnitTestFrameworkPkg.ci.yaml b/UnitTestFrameworkPkg/UnitTestFrameworkPkg.ci.yaml index d17f6e094e..4774c6cb32 100644 --- a/UnitTestFrameworkPkg/UnitTestFrameworkPkg.ci.yaml +++ b/UnitTestFrameworkPkg/UnitTestFrameworkPkg.ci.yaml @@ -115,7 +115,8 @@ "cobertura", # tool for code coverage "pycobertura", # tool for code coverage "loongarch", - "loongson" + "loongson", + "xunit", ], "IgnoreStandardPaths": [], # Standard Plugin defined paths that should be ignore "AdditionalIncludePaths": [] # Additional paths to spell check (wildcards supported)