From 33b72921b41e787941eff0e34e4b0b4cae93dcb8 Mon Sep 17 00:00:00 2001 From: Joey Vagedes Date: Mon, 1 Jul 2024 15:03:30 -0700 Subject: [PATCH] BaseTools: reformat HostBasedUnitTestRunner coverage results by inf (#616) (#688) Coverage results that are generated from HostBasedUnitTestRunner are inconsistent in terms of how the cobertura file is formatted. On Windows, cobertura results are grouped by test executable while on Linux, cobertura results are grouped by INF. This commit reformats coverage results to always be by INF and to additionally be package-path relative. An additional change this commit makes is to rename the package coverage file to include the package name. This is to make easily identify which coverage results come from which host test DSC and additionally prevent name conflicts when coverage results are uploaded during a PR gate. Overall coverage is still generated and at the root of the output directory (typically `$(ws)/Build/`) An additional change is made to how the `input_coverage` parameter is provided to `OpenCppCoverage` for msvc builds. `input_coverage` is now provided via a config file due to the command line argument limit being reached as the number of tests being run increases. The config file is written, the command is run, then the file is deleted. Integrates edk2-pytool-extensions 0.27.0 and edk2-pytool-library 0.20.0, which overhauls the database functionality to use an ORM for managing the database schema and access to the database. Updates the only plugin in MU_BASECORE that uses the database functionality, HostBasedUnitTestRunner. cherry-picked from 991a64e, pulled from 43ae607 in 202311 cherry-picked from 785fe3db51 - [ ] Impacts functionality? - [ ] Impacts security? - [x] Breaking change? - [ ] Includes tests? - [ ] Includes documentation?. --- .../HostBasedUnitTestRunner.py | 129 ++++++++++-------- UnitTestFrameworkPkg/ReadMe.md | 83 ++++++----- 2 files changed, 123 insertions(+), 89 deletions(-) diff --git a/BaseTools/Plugin/HostBasedUnitTestRunner/HostBasedUnitTestRunner.py b/BaseTools/Plugin/HostBasedUnitTestRunner/HostBasedUnitTestRunner.py index b2122b0dca..7b6c794d34 100644 --- a/BaseTools/Plugin/HostBasedUnitTestRunner/HostBasedUnitTestRunner.py +++ b/BaseTools/Plugin/HostBasedUnitTestRunner/HostBasedUnitTestRunner.py @@ -16,6 +16,8 @@ from edk2toolext.environment import shell_environment from edk2toollib.utility_functions import RunCmd from edk2toollib.utility_functions import GetHostInfo +from edk2toollib.database import Edk2DB # MU_CHANGE - reformat coverage data +from edk2toollib.database.tables import EnvironmentTable, SourceTable, PackageTable, InfTable # MU_CHANGE - reformat coverage data from textwrap import dedent @@ -139,6 +141,15 @@ def do_post_build(self, thebuilder): failure_count += 1 else: logging.info("Skipping code coverage. Currently, support GCC and MSVC compiler.") + return failure_count # MU_CHANGE - reformat coverage data + + # MU_CHANGE begin - reformat coverage data + if thebuilder.env.GetValue("CC_REORGANIZE", "TRUE") == "TRUE": + ret = self.organize_coverage(thebuilder) + if ret != 0: + logging.error("Failed to reorganize coverage data by INF.") + return -1 + # MU_CHANGE end - reformat coverage data return failure_count @@ -166,14 +177,11 @@ def gen_code_coverage_gcc(self, thebuilder): logging.error("UnitTest Coverage: Failed to aggregate coverage data.") return 1 - # Generate coverage XML - ret = RunCmd("lcov_cobertura",f"{buildOutputBase}/total-coverage.info -o {buildOutputBase}/compare.xml") - if ret != 0: - logging.error("UnitTest Coverage: Failed to generate coverage XML.") - return 1 - # Filter out auto-generated and test code - ret = RunCmd("lcov_cobertura",f"{buildOutputBase}/total-coverage.info --excludes ^.*UnitTest\|^.*MU\|^.*Mock\|^.*DEBUG -o {buildOutputBase}/coverage.xml") + # MU_CHANGE begin - reformat coverage data + file_out = thebuilder.env.GetValue("CI_PACKAGE_NAME", "") + "_coverage.xml" + ret = RunCmd("lcov_cobertura",f"{buildOutputBase}/total-coverage.info --excludes ^.*UnitTest\|^.*MU\|^.*Mock\|^.*DEBUG -o {buildOutputBase}/{file_out}") + # MU_CHANGE end - reformat coverage data if ret != 0: logging.error("UnitTest Coverage: Failed generate filtered coverage XML.") return 1 @@ -210,64 +218,73 @@ def gen_code_coverage_msvc(self, thebuilder): workspace = (workspace + os.sep) if workspace[-1] != os.sep else workspace workspaceBuild = os.path.join(workspace, 'Build') # Generate coverage file - coverageFile = "" - for testFile in testList: - ret = RunCmd("OpenCppCoverage", f"--source {workspace} --export_type binary:{testFile}.cov -- {testFile}", workingdir=f"{workspace}Build/") - if ret != 0: - logging.error("UnitTest Coverage: Failed to collect coverage data.") - return 1 - - coverageFile = f" --input_coverage={testFile}.cov" - totalCoverageFile = os.path.join(buildOutputBase, 'coverage.cov') - if os.path.isfile(totalCoverageFile): - coverageFile += f" --input_coverage={totalCoverageFile}" - ret = RunCmd( - "OpenCppCoverage", - f"--export_type binary:{totalCoverageFile} " + - f"--working_dir={workspaceBuild} " + - f"{coverageFile}" - ) - if ret != 0: - logging.error("UnitTest Coverage: Failed to collect coverage data.") - return 1 + # MU_CHANGE begin - reformat coverage data + pkg_cfg_file = os.path.join(buildOutputBase, "pkg-opencppcoverage.cfg") + if os.path.isfile(pkg_cfg_file): + os.remove(pkg_cfg_file) + + with open(pkg_cfg_file, "w") as f: + for testFile in testList: + ret = RunCmd("OpenCppCoverage", f"--source {workspace} --export_type binary:{testFile}.cov -- {testFile}", workingdir=f"{workspace}Build/") + f.write(f"input_coverage={testFile}.cov\n") + if ret != 0: + logging.error("UnitTest Coverage: Failed to collect coverage data.") + return 1 # Generate and XML file if requested.by each package - ret = RunCmd( - "OpenCppCoverage", - f"--export_type cobertura:{os.path.join(buildOutputBase, 'coverage.xml')} " + - f"--working_dir={workspaceBuild} " + - f"--input_coverage={totalCoverageFile} " - ) + + file_out = thebuilder.env.GetValue("CI_PACKAGE_NAME", "") + "_coverage.xml" + ret = RunCmd("OpenCppCoverage", f"--export_type cobertura:{os.path.join(buildOutputBase, file_out)} --config_file={pkg_cfg_file}", workingdir=f"{workspace}Build/") + os.remove(pkg_cfg_file) + if ret != 0: logging.error("UnitTest Coverage: Failed to generate cobertura format xml in single package.") return 1 # Generate total report XML file for all package - testCoverageList = glob.glob(os.path.join(workspace, "Build", "**", "*Test*.exe.cov"), recursive=True) - coverageFile = "" - totalCoverageFile = os.path.join(workspaceBuild, 'coverage.cov') - for testCoverage in testCoverageList: - coverageFile = f" --input_coverage={testCoverage}" - if os.path.isfile(totalCoverageFile): - coverageFile += f" --input_coverage={totalCoverageFile}" - ret = RunCmd( - "OpenCppCoverage", - f"--export_type binary:{totalCoverageFile} " + - f"--working_dir={workspaceBuild} " + - f"{coverageFile}" - ) - if ret != 0: - logging.error("UnitTest Coverage: Failed to collect coverage data.") - return 1 - - ret = RunCmd( - "OpenCppCoverage", - f"--export_type cobertura:{os.path.join(workspaceBuild, 'coverage.xml')} " + - f"--working_dir={workspaceBuild} " + - f"--input_coverage={totalCoverageFile}" - ) + testCoverageList = glob.glob(os.path.join(workspace, "Build", "**","*Test*.exe.cov"), recursive=True) + total_cfg_file = os.path.join(buildOutputBase, "total-opencppcoverage.cfg") + if os.path.isfile(total_cfg_file): + os.remove(total_cfg_file) + + with open(total_cfg_file, "w") as f: + for testCoverage in testCoverageList: + f.write(f"input_coverage={testCoverage}\n") + + ret = RunCmd("OpenCppCoverage", f"--export_type cobertura:{workspace}Build/coverage.xml --config_file={total_cfg_file}", workingdir=f"{workspace}Build/") + os.remove(total_cfg_file) + if ret != 0: logging.error("UnitTest Coverage: Failed to generate cobertura format xml.") return 1 return 0 + + def organize_coverage(self, thebuilder) -> int: + """Organize the generated coverage file by INF.""" + db_path = self.parse_workspace(thebuilder) + + workspace = thebuilder.env.GetValue("WORKSPACE") + buildOutputBase = thebuilder.env.GetValue("BUILD_OUTPUT_BASE") + package = thebuilder.env.GetValue("CI_PACKAGE_NAME", "") + file_out = package + "_coverage.xml" + cov_file = os.path.join(buildOutputBase, file_out) + + params = f"--database {db_path} coverage {cov_file} -o {cov_file} --by-package -ws {workspace}" + + params += f" -p {package}" * int(package != "") + params += " --full" * int(thebuilder.env.GetValue("CC_FULL", "FALSE") == "TRUE") + params += " --flatten" * int(thebuilder.env.GetValue("CC_FLATTEN", "FALSE") == "TRUE") + + return RunCmd("stuart_report", params) + + def parse_workspace(self, thebuilder) -> str: + """Parses the workspace with Edk2DB with the tables necessarty to run stuart_report.""" + db_path = os.path.join(thebuilder.env.GetValue("BUILD_OUTPUT_BASE"), "DATABASE.db") + db = Edk2DB(db_path, thebuilder.edk2path) + db.register(EnvironmentTable(), SourceTable(), PackageTable(), InfTable()) + env_dict = thebuilder.env.GetAllBuildKeyValues() | thebuilder.env.GetAllNonBuildKeyValues() + db.parse(env_dict) + + return db_path + # MU_CHANGE end - reformat coverage data diff --git a/UnitTestFrameworkPkg/ReadMe.md b/UnitTestFrameworkPkg/ReadMe.md index d28cb5cb0a..d3641f8854 100644 --- a/UnitTestFrameworkPkg/ReadMe.md +++ b/UnitTestFrameworkPkg/ReadMe.md @@ -1485,44 +1485,61 @@ This mode is used by the test running plugin to aggregate the results for CI tes ### Code Coverage -Host based Unit Tests will automatically enable coverage data. +Code coverage can be enabled for Host based Unit Tests with `CODE_COVERAGE=TRUE`, which generates a cobertura report +per package tested, and combined cobertura report for all packages tested. The per-package cobertura report will be +present at `Build//HostTest//_coverage.xml`. The overall cobertura report will be present +at `Build/coverage.xml` + +Code coverage generation has two config knobs: + +1. `CC_FULL`: If set to `TRUE`, will generate zero'd out coverage data for untested source files in the package. +2. `CC_FLATTEN`: If Set to `TRUE`, will group all source files together, rather than by INF. + +**TIP: `CC_FLATTEN=TRUE/FALSE` will produce different coverage percentage results as `TRUE` de-duplicates source files +that are consumed by multiple INFs. For Windows, this is primarily leveraged for pipeline builds, but this can be leveraged locally using the OpenCppCoverage windows tool to parse coverage data to cobertura xml format. -- Windows Prerequisite - ```bash - Download and install https://github.com/OpenCppCoverage/OpenCppCoverage/releases - python -m pip install --upgrade -r ./pip-requirements.txt - stuart_ci_build -c .pytool/CISettings.py -t NOOPT TOOL_CHAIN_TAG=VS2019 -p MdeModulePkg - Open Build/coverage.xml - ``` - - - How to see code coverage data on IDE Visual Studio - ``` - Open Visual Studio VS2019 or above version - Click "Tools" -> "OpenCppCoverage Settings" - Fill your execute file into "Program to run:" - Click "Tools" -> "Run OpenCppCoverage" - ``` - - -For Linux, this is primarily leveraged for pipeline builds, but this can be leveraged locally using the -lcov linux tool, and parsed using the lcov_cobertura python tool to parse it to cobertura xml format. - -- Linux Prerequisite - ```bash - sudo apt-get install -y lcov - python -m pip install --upgrade -r ./pip-requirements.txt - stuart_ci_build -c .pytool/CISettings.py -t NOOPT TOOL_CHAIN_TAG=GCC5 -p MdeModulePkg - Open Build/coverage.xml - ``` - - How to see code coverage data on IDE Visual Studio Code - ``` - Download plugin "Coverage Gutters" - Press Hot Key "Ctrl + Shift + P" and click option "Coverage Gutters: Display Coverage" - ``` +#### Prerequisites + +In addition to required prerequisites to build and test, there are additional requirements for calculating code +coverage files as noted below. + +* Windows Prerequisite + + 1. OpenCppCoverage: Download and install + +* Linux Prerequisite + + 1. lcov: sudo apt-get install -y lcov + +#### Examples + +```bash +stuart_ci_build -c .pytool/CISettings.py -t NOOPT TOOL_CHAIN_TAG=VS2019 -p MdeModulePkg CODE_COVERAGE=TRUE +stuart_ci_build -c .pytool/CISettings.py -t NOOPT TOOL_CHAIN_TAG=VS2019 CODE_COVERAGE=TRUE CC_FLATTEN=TRUE CC_FULL=FALSE +``` + +How to see code coverage data on IDE Visual Studio + +```text +Open Visual Studio VS2019 or above version +Click "Tools" -> "OpenCppCoverage Settings" +Fill your execute file into "Program to run:" +Click "Tools" -> "Run OpenCppCoverage" +``` + +#### Additional Tools + +There are a plethora of open source tools for generating reports from a Cobertura file, which is why it was selected as +the output file format. Tools such as pycobertura (`pip install pycobertura`) and [reportgenerator](https://www.nuget.org/packages/dotnet-reportgenerator-globaltool) +can be utilized to generate different report types, such as local html reports. VSCode Extensions such as [Coverage Gutters](https://marketplace.visualstudio.com/items?itemName=ryanluker.vscode-coverage-gutters) +can highlight coverage results directly in the file, and cloud tools such as [CodeCov](https://about.codecov.io/) can +consume cobertura files to provide PR checks and general code coverage statistics for the repository. +*** REMINDER: During CI builds, use the ``CODE_COVERAGE=TRUE` flag to generate the code coverage XML files, +and additionally use the `CC_FLATTEN=TRUE` or `CC_FULL=TRUE` flags to customize coverage results. ### Important Note