diff --git a/BaseTools/Plugin/HostBasedUnitTestRunner/HostBasedUnitTestRunner.py b/BaseTools/Plugin/HostBasedUnitTestRunner/HostBasedUnitTestRunner.py index 2b06e3c409..fc4801f0d0 100644 --- a/BaseTools/Plugin/HostBasedUnitTestRunner/HostBasedUnitTestRunner.py +++ b/BaseTools/Plugin/HostBasedUnitTestRunner/HostBasedUnitTestRunner.py @@ -12,10 +12,11 @@ import xml.etree.ElementTree from edk2toolext.environment.plugintypes.uefi_build_plugin import IUefiBuildPlugin from edk2toolext import edk2_logging -import edk2toollib.windows.locate_tools as locate_tools from edk2toolext.environment import shell_environment from edk2toollib.utility_functions import RunCmd from edk2toollib.utility_functions import GetHostInfo +from edk2toollib.database import Edk2DB # MU_CHANGE - reformat coverage data +from edk2toollib.database.tables import EnvironmentTable, SourceTable, PackageTable, InfTable # MU_CHANGE - reformat coverage data from textwrap import dedent @@ -140,6 +141,15 @@ def do_post_build(self, thebuilder): failure_count += 1 else: logging.info("Skipping code coverage. Currently, support GCC and MSVC compiler.") + return failure_count # MU_CHANGE - reformat coverage data + + # MU_CHANGE begin - reformat coverage data + if thebuilder.env.GetValue("CC_REORGANIZE", "TRUE") == "TRUE": + ret = self.organize_coverage(thebuilder) + if ret != 0: + logging.error("Failed to reorganize coverage data by INF.") + return -1 + # MU_CHANGE end - reformat coverage data return failure_count @@ -148,6 +158,9 @@ def gen_code_coverage_gcc(self, thebuilder): buildOutputBase = thebuilder.env.GetValue("BUILD_OUTPUT_BASE") workspace = thebuilder.env.GetValue("WORKSPACE") + # MU_CHANGE begin - regex string for exclude paths + regex_exclude = r"^.*UnitTest\|^.*MU\|^.*Mock\|^.*DEBUG" + # MU_CHANGE end - regex string for exclude paths # Generate base code coverage for all source files ret = RunCmd("lcov", f"--no-external --capture --initial --directory {buildOutputBase} --output-file {buildOutputBase}/cov-base.info --rc lcov_branch_coverage=1") @@ -167,14 +180,11 @@ def gen_code_coverage_gcc(self, thebuilder): logging.error("UnitTest Coverage: Failed to aggregate coverage data.") return 1 - # Generate coverage XML - ret = RunCmd("lcov_cobertura",f"{buildOutputBase}/total-coverage.info -o {buildOutputBase}/compare.xml") - if ret != 0: - logging.error("UnitTest Coverage: Failed to generate coverage XML.") - return 1 - # Filter out auto-generated and test code - ret = RunCmd("lcov_cobertura",f"{buildOutputBase}/total-coverage.info --excludes ^.*UnitTest\|^.*MU\|^.*Mock\|^.*DEBUG -o {buildOutputBase}/coverage.xml") + # MU_CHANGE begin - reformat coverage data + file_out = thebuilder.env.GetValue("CI_PACKAGE_NAME", "") + "_coverage.xml" + ret = RunCmd("lcov_cobertura",f"{buildOutputBase}/total-coverage.info --excludes {regex_exclude} -o {buildOutputBase}/{file_out}") + # MU_CHANGE end - reformat coverage data if ret != 0: logging.error("UnitTest Coverage: Failed generate filtered coverage XML.") return 1 @@ -193,7 +203,9 @@ def gen_code_coverage_gcc(self, thebuilder): # Generate and XML file if requested.for all package if os.path.isfile(f"{workspace}/Build/coverage.xml"): os.remove(f"{workspace}/Build/coverage.xml") - ret = RunCmd("lcov_cobertura",f"{workspace}/Build/all-coverage.info --excludes ^.*UnitTest\|^.*MU\|^.*Mock\|^.*DEBUG -o {workspace}/Build/coverage.xml") + # MU_CHANGE begin - regex string for exclude paths + ret = RunCmd("lcov_cobertura",f"{workspace}/Build/all-coverage.info --excludes {regex_exclude} -o {workspace}/Build/coverage.xml") + # MU_CHANGE end - regex string for exclude paths if ret != 0: logging.error("UnitTest Coverage: Failed generate all coverage XML.") return 1 @@ -211,64 +223,74 @@ def gen_code_coverage_msvc(self, thebuilder): workspace = (workspace + os.sep) if workspace[-1] != os.sep else workspace workspaceBuild = os.path.join(workspace, 'Build') # Generate coverage file - coverageFile = "" - for testFile in testList: - ret = RunCmd("OpenCppCoverage", f"--source {workspace} --export_type binary:{testFile}.cov -- {testFile}", workingdir=f"{workspace}Build/") - if ret != 0: - logging.error("UnitTest Coverage: Failed to collect coverage data.") - return 1 - - coverageFile = f" --input_coverage={testFile}.cov" - totalCoverageFile = os.path.join(buildOutputBase, 'coverage.cov') - if os.path.isfile(totalCoverageFile): - coverageFile += f" --input_coverage={totalCoverageFile}" - ret = RunCmd( - "OpenCppCoverage", - f"--export_type binary:{totalCoverageFile} " + - f"--working_dir={workspaceBuild} " + - f"{coverageFile}" - ) - if ret != 0: - logging.error("UnitTest Coverage: Failed to collect coverage data.") - return 1 + # MU_CHANGE begin - reformat coverage data + pkg_cfg_file = os.path.join(buildOutputBase, "pkg-opencppcoverage.cfg") + if os.path.isfile(pkg_cfg_file): + os.remove(pkg_cfg_file) + + with open(pkg_cfg_file, "w") as f: + for testFile in testList: + ret = RunCmd("OpenCppCoverage", f"--source {workspace} --export_type binary:{testFile}.cov -- {testFile}", workingdir=f"{workspace}Build/") + f.write(f"input_coverage={testFile}.cov\n") + if ret != 0: + logging.error("UnitTest Coverage: Failed to collect coverage data.") + return 1 # Generate and XML file if requested.by each package - ret = RunCmd( - "OpenCppCoverage", - f"--export_type cobertura:{os.path.join(buildOutputBase, 'coverage.xml')} " + - f"--working_dir={workspaceBuild} " + - f"--input_coverage={totalCoverageFile} " - ) + + file_out = thebuilder.env.GetValue("CI_PACKAGE_NAME", "") + "_coverage.xml" + ret = RunCmd("OpenCppCoverage", f"--export_type cobertura:{os.path.join(buildOutputBase, file_out)} --config_file={pkg_cfg_file}", workingdir=f"{workspace}Build/") + os.remove(pkg_cfg_file) + if ret != 0: logging.error("UnitTest Coverage: Failed to generate cobertura format xml in single package.") return 1 # Generate total report XML file for all package - testCoverageList = glob.glob(os.path.join(workspace, "Build", "**", "*Test*.exe.cov"), recursive=True) - coverageFile = "" - totalCoverageFile = os.path.join(workspaceBuild, 'coverage.cov') - for testCoverage in testCoverageList: - coverageFile = f" --input_coverage={testCoverage}" - if os.path.isfile(totalCoverageFile): - coverageFile += f" --input_coverage={totalCoverageFile}" - ret = RunCmd( - "OpenCppCoverage", - f"--export_type binary:{totalCoverageFile} " + - f"--working_dir={workspaceBuild} " + - f"{coverageFile}" - ) - if ret != 0: - logging.error("UnitTest Coverage: Failed to collect coverage data.") - return 1 - - ret = RunCmd( - "OpenCppCoverage", - f"--export_type cobertura:{os.path.join(workspaceBuild, 'coverage.xml')} " + - f"--working_dir={workspaceBuild} " + - f"--input_coverage={totalCoverageFile}" - ) + testCoverageList = glob.glob(os.path.join(workspace, "Build", "**","*Test*.exe.cov"), recursive=True) + total_cfg_file = os.path.join(buildOutputBase, "total-opencppcoverage.cfg") + if os.path.isfile(total_cfg_file): + os.remove(total_cfg_file) + + with open(total_cfg_file, "w") as f: + for testCoverage in testCoverageList: + f.write(f"input_coverage={testCoverage}\n") + + ret = RunCmd("OpenCppCoverage", f"--export_type cobertura:{workspace}Build/coverage.xml --config_file={total_cfg_file}", workingdir=f"{workspace}Build/") + os.remove(total_cfg_file) + if ret != 0: logging.error("UnitTest Coverage: Failed to generate cobertura format xml.") return 1 return 0 + + def organize_coverage(self, thebuilder) -> int: + """Organize the generated coverage file by INF.""" + db_path = self.parse_workspace(thebuilder) + + workspace = thebuilder.env.GetValue("WORKSPACE") + buildOutputBase = thebuilder.env.GetValue("BUILD_OUTPUT_BASE") + package = thebuilder.env.GetValue("CI_PACKAGE_NAME", "") + file_out = package + "_coverage.xml" + cov_file = os.path.join(buildOutputBase, file_out) + exclude = thebuilder.env.GetValue("CC_EXCLUDE", "*NULL*,*Null*,*null*") + + params = f"--database {db_path} coverage {cov_file} -o {cov_file} --by-package -ws {workspace}" + + params += f" -p {package}" * int(package != "") + params += " --full" * int(thebuilder.env.GetValue("CC_FULL", "FALSE") == "TRUE") + params += " --flatten" * int(thebuilder.env.GetValue("CC_FLATTEN", "FALSE") == "TRUE") + params += f" --exclude {exclude}" * int(exclude != "") + return RunCmd("stuart_report", params) + + def parse_workspace(self, thebuilder) -> str: + """Parses the workspace with Edk2DB with the tables necessarty to run stuart_report.""" + db_path = os.path.join(thebuilder.env.GetValue("BUILD_OUTPUT_BASE"), "DATABASE.db") + db = Edk2DB(db_path, thebuilder.edk2path) + db.register(EnvironmentTable(), SourceTable(), PackageTable(), InfTable()) + env_dict = thebuilder.env.GetAllBuildKeyValues() | thebuilder.env.GetAllNonBuildKeyValues() + db.parse(env_dict) + + return db_path + # MU_CHANGE end - reformat coverage data diff --git a/UnitTestFrameworkPkg/ReadMe.md b/UnitTestFrameworkPkg/ReadMe.md index 6a67ca23a1..f359746523 100644 --- a/UnitTestFrameworkPkg/ReadMe.md +++ b/UnitTestFrameworkPkg/ReadMe.md @@ -1364,6 +1364,9 @@ After that, the following commands will set up the build and run the host-based # stuart_setup -c ./.pytool/CISettings.py TOOL_CHAIN_TAG= stuart_setup -c ./.pytool/CISettings.py TOOL_CHAIN_TAG=VS2019 +# Mu specific step to clone mu repos required for ci check +# stuart_ci_setup -c ./.pytool/CISettings.py TOOL_CHAIN_TAG= +stuart_ci_setup -c ./.pytool/CISettings.py TOOL_CHAIN_TAG=VS2019 # Update all binary dependencies # stuart_update -c ./.pytool/CISettings.py TOOL_CHAIN_TAG= stuart_update -c ./.pytool/CISettings.py TOOL_CHAIN_TAG=VS2019 @@ -1485,44 +1488,70 @@ This mode is used by the test running plugin to aggregate the results for CI tes ### Code Coverage -Host based Unit Tests will automatically enable coverage data. +Code coverage can be enabled for Host based Unit Tests with `CODE_COVERAGE=TRUE`, which generates a cobertura report +per package tested, and combined cobertura report for all packages tested. The per-package cobertura report will be +present at `Build//HostTest//_coverage.xml`. The overall cobertura report will be present +at `Build/coverage.xml` + +Code coverage generation has three config knobs. Each can be turned on/off by setting it to TRUE +or FALSE e.g. `CC_REORGANIZE=TRUE`: + +1. `CC_REORGANIZE`: Controls if code coverage results are re-formatted into a "by-inf" folder + structure rather than the default "by-test" folder structure. Default: `TRUE` +1. `CC_FULL`: Generates zero'd out coverage data for untested source files in the package. + Default: `FALSE` +1. `CC_FLATTEN`: Groups all source files together, rather than by INF. Default: `FALSE` +1. `CC_EXCLUDE`: Comma separated list of fnmatch expressions to exclude from results. + Default: \*NULL\*,\*Null\*,\*null\* + +** NOTE: `CC_FULL` and `CC_FLATTEN` and `CC_EXCLUDE` values only matter if `CC_REORGANIZE=TRUE`, as they only +effect how the coverage report is reorganized. + +**TIP: `CC_FLATTEN=TRUE/FALSE` will produce different coverage percentage results as `TRUE` de-duplicates source files +that are consumed by multiple INFs. For Windows, this is primarily leveraged for pipeline builds, but this can be leveraged locally using the OpenCppCoverage windows tool to parse coverage data to cobertura xml format. -- Windows Prerequisite - ```bash - Download and install https://github.com/OpenCppCoverage/OpenCppCoverage/releases - python -m pip install --upgrade -r ./pip-requirements.txt - stuart_ci_build -c .pytool/CISettings.py -t NOOPT TOOL_CHAIN_TAG=VS2019 -p MdeModulePkg - Open Build/coverage.xml - ``` - - - How to see code coverage data on IDE Visual Studio - ``` - Open Visual Studio VS2019 or above version - Click "Tools" -> "OpenCppCoverage Settings" - Fill your execute file into "Program to run:" - Click "Tools" -> "Run OpenCppCoverage" - ``` - - -For Linux, this is primarily leveraged for pipeline builds, but this can be leveraged locally using the -lcov linux tool, and parsed using the lcov_cobertura python tool to parse it to cobertura xml format. - -- Linux Prerequisite - ```bash - sudo apt-get install -y lcov - python -m pip install --upgrade -r ./pip-requirements.txt - stuart_ci_build -c .pytool/CISettings.py -t NOOPT TOOL_CHAIN_TAG=GCC5 -p MdeModulePkg - Open Build/coverage.xml - ``` - - How to see code coverage data on IDE Visual Studio Code - ``` - Download plugin "Coverage Gutters" - Press Hot Key "Ctrl + Shift + P" and click option "Coverage Gutters: Display Coverage" - ``` +#### Prerequisites + +In addition to required prerequisites to build and test, there are additional requirements for calculating code +coverage files as noted below. + +* Windows Prerequisite + + 1. OpenCppCoverage: Download and install + +* Linux Prerequisite + + 1. lcov: sudo apt-get install -y lcov + +#### Examples + +```bash +stuart_ci_build -c .pytool/CISettings.py -t NOOPT TOOL_CHAIN_TAG=VS2019 -p MdeModulePkg CODE_COVERAGE=TRUE +stuart_ci_build -c .pytool/CISettings.py -t NOOPT TOOL_CHAIN_TAG=VS2019 CODE_COVERAGE=TRUE CC_FLATTEN=TRUE CC_FULL=FALSE +``` + +How to see code coverage data on IDE Visual Studio + +```text +Open Visual Studio VS2019 or above version +Click "Tools" -> "OpenCppCoverage Settings" +Fill your execute file into "Program to run:" +Click "Tools" -> "Run OpenCppCoverage" +``` + +#### Additional Tools + +There are a plethora of open source tools for generating reports from a Cobertura file, which is why it was selected as +the output file format. Tools such as pycobertura (`pip install pycobertura`) and [reportgenerator](https://www.nuget.org/packages/dotnet-reportgenerator-globaltool) +can be utilized to generate different report types, such as local html reports. VSCode Extensions such as [Coverage Gutters](https://marketplace.visualstudio.com/items?itemName=ryanluker.vscode-coverage-gutters) +can highlight coverage results directly in the file, and cloud tools such as [CodeCov](https://about.codecov.io/) can +consume cobertura files to provide PR checks and general code coverage statistics for the repository. +*** REMINDER: During CI builds, use the ``CODE_COVERAGE=TRUE` flag to generate the code coverage XML files, +and additionally use the `CC_FLATTEN=TRUE` or `CC_FULL=TRUE` flags to customize coverage results. ### Important Note