diff --git a/.azurepipelines/Ubuntu-GCC5.yml b/.azurepipelines/Ubuntu-GCC5.yml index f557099d64b..16490ff9fd8 100644 --- a/.azurepipelines/Ubuntu-GCC5.yml +++ b/.azurepipelines/Ubuntu-GCC5.yml @@ -27,6 +27,7 @@ extends: do_non_ci_setup: true do_pr_eval: true calculate_code_coverage: true + coverage_publish_target: 'codecov' container_build: true os_type: Linux build_matrix: diff --git a/.azurepipelines/Windows-VS.yml b/.azurepipelines/Windows-VS.yml index 64410899624..bff2bd54ad3 100644 --- a/.azurepipelines/Windows-VS.yml +++ b/.azurepipelines/Windows-VS.yml @@ -27,6 +27,7 @@ extends: do_non_ci_setup: true do_pr_eval: true calculate_code_coverage: true + coverage_publish_target: 'codecov' os_type: Windows_NT build_matrix: TARGET_MDE_CPU: diff --git a/.github/codecov.yml b/.github/codecov.yml new file mode 100644 index 00000000000..deaeb11c763 --- /dev/null +++ b/.github/codecov.yml @@ -0,0 +1,30 @@ +## @file +# codecov upload configuration file to carryforward coverage results of +# packages that do not upload coverage results for a given pull request. +## +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: BSD-2-Clause-Patent +## +flags: + CryptoPkg: + carryforward: true + MdeModulePkg: + carryforward: true + MdePkg: + carryforward: true + NetworkPkg: + carryforward: true + PcAtChipsetPkg: + carryforward: true + PerformancePkg: + carryforward: true + PolicyServicePkg: + carryforward: true + ShellPkg: + carryforward: true + StandaloneMmPkg: + carryforward: true + UefiCpuPkg: + carryforward: true + UnitTestFrameworkPkg: + carryforward: true diff --git a/.pytool/Plugin/HostUnitTestCompilerPlugin/HostUnitTestCompilerPlugin.py b/.pytool/Plugin/HostUnitTestCompilerPlugin/HostUnitTestCompilerPlugin.py index a99ce4ee14b..ec2063a0b46 100644 --- a/.pytool/Plugin/HostUnitTestCompilerPlugin/HostUnitTestCompilerPlugin.py +++ b/.pytool/Plugin/HostUnitTestCompilerPlugin/HostUnitTestCompilerPlugin.py @@ -87,6 +87,7 @@ def __GetHostUnitTestArch(self, environment): def RunBuildPlugin(self, packagename, Edk2pathObj, pkgconfig, environment, PLM, PLMHelper, tc, output_stream=None): self._env = environment environment.SetValue("CI_BUILD_TYPE", "host_unit_test", "Set in HostUnitTestCompilerPlugin") + environment.SetValue("CI_PACKAGE_NAME", packagename, "Set in HostUnitTestCompilerPlugin") # Parse the config for required DscPath element if "DscPath" not in pkgconfig: diff --git a/BaseTools/Plugin/HostBasedUnitTestRunner/HostBasedUnitTestRunner.py b/BaseTools/Plugin/HostBasedUnitTestRunner/HostBasedUnitTestRunner.py index b2122b0dca8..9f3e89f284c 100644 --- a/BaseTools/Plugin/HostBasedUnitTestRunner/HostBasedUnitTestRunner.py +++ b/BaseTools/Plugin/HostBasedUnitTestRunner/HostBasedUnitTestRunner.py @@ -16,6 +16,8 @@ from edk2toolext.environment import shell_environment from edk2toollib.utility_functions import RunCmd from edk2toollib.utility_functions import GetHostInfo +from edk2toollib.database import Edk2DB # MU_CHANGE - reformat coverage data +from edk2toollib.database.tables import SourceTable, PackageTable, InfTable # MU_CHANGE - reformat coverage data from textwrap import dedent @@ -139,6 +141,15 @@ def do_post_build(self, thebuilder): failure_count += 1 else: logging.info("Skipping code coverage. Currently, support GCC and MSVC compiler.") + return failure_count # MU_CHANGE - reformat coverage data + + # MU_CHANGE begin - reformat coverage data + if thebuilder.env.GetValue("CC_REORGANIZE", "TRUE") == "TRUE": + ret = self.organize_coverage(thebuilder) + if ret != 0: + logging.error("Failed to reorganize coverage data by INF.") + return -1 + # MU_CHANGE end - reformat coverage data return failure_count @@ -166,14 +177,11 @@ def gen_code_coverage_gcc(self, thebuilder): logging.error("UnitTest Coverage: Failed to aggregate coverage data.") return 1 - # Generate coverage XML - ret = RunCmd("lcov_cobertura",f"{buildOutputBase}/total-coverage.info -o {buildOutputBase}/compare.xml") - if ret != 0: - logging.error("UnitTest Coverage: Failed to generate coverage XML.") - return 1 - # Filter out auto-generated and test code - ret = RunCmd("lcov_cobertura",f"{buildOutputBase}/total-coverage.info --excludes ^.*UnitTest\|^.*MU\|^.*Mock\|^.*DEBUG -o {buildOutputBase}/coverage.xml") + # MU_CHANGE begin - reformat coverage data + file_out = thebuilder.env.GetValue("CI_PACKAGE_NAME", "") + "_coverage.xml" + ret = RunCmd("lcov_cobertura",f"{buildOutputBase}/total-coverage.info --excludes ^.*UnitTest\|^.*MU\|^.*Mock\|^.*DEBUG -o {buildOutputBase}/{file_out}") + # MU_CHANGE end - reformat coverage data if ret != 0: logging.error("UnitTest Coverage: Failed generate filtered coverage XML.") return 1 @@ -208,66 +216,74 @@ def gen_code_coverage_msvc(self, thebuilder): testList = glob.glob(os.path.join(buildOutputBase, "**","*Test*.exe"), recursive=True) workspace = thebuilder.env.GetValue("WORKSPACE") workspace = (workspace + os.sep) if workspace[-1] != os.sep else workspace - workspaceBuild = os.path.join(workspace, 'Build') # Generate coverage file - coverageFile = "" - for testFile in testList: - ret = RunCmd("OpenCppCoverage", f"--source {workspace} --export_type binary:{testFile}.cov -- {testFile}", workingdir=f"{workspace}Build/") - if ret != 0: - logging.error("UnitTest Coverage: Failed to collect coverage data.") - return 1 - - coverageFile = f" --input_coverage={testFile}.cov" - totalCoverageFile = os.path.join(buildOutputBase, 'coverage.cov') - if os.path.isfile(totalCoverageFile): - coverageFile += f" --input_coverage={totalCoverageFile}" - ret = RunCmd( - "OpenCppCoverage", - f"--export_type binary:{totalCoverageFile} " + - f"--working_dir={workspaceBuild} " + - f"{coverageFile}" - ) - if ret != 0: - logging.error("UnitTest Coverage: Failed to collect coverage data.") - return 1 + # MU_CHANGE begin - reformat coverage data + pkg_cfg_file = os.path.join(buildOutputBase, "pkg-opencppcoverage.cfg") + if os.path.isfile(pkg_cfg_file): + os.remove(pkg_cfg_file) + + with open(pkg_cfg_file, "w") as f: + for testFile in testList: + ret = RunCmd("OpenCppCoverage", f"--source {workspace} --export_type binary:{testFile}.cov -- {testFile}", workingdir=f"{workspace}Build/") + f.write(f"input_coverage={testFile}.cov\n") + if ret != 0: + logging.error("UnitTest Coverage: Failed to collect coverage data.") + return 1 # Generate and XML file if requested.by each package - ret = RunCmd( - "OpenCppCoverage", - f"--export_type cobertura:{os.path.join(buildOutputBase, 'coverage.xml')} " + - f"--working_dir={workspaceBuild} " + - f"--input_coverage={totalCoverageFile} " - ) + + file_out = thebuilder.env.GetValue("CI_PACKAGE_NAME", "") + "_coverage.xml" + ret = RunCmd("OpenCppCoverage", f"--export_type cobertura:{os.path.join(buildOutputBase, file_out)} --config_file={pkg_cfg_file}", workingdir=f"{workspace}Build/") + os.remove(pkg_cfg_file) + if ret != 0: logging.error("UnitTest Coverage: Failed to generate cobertura format xml in single package.") return 1 # Generate total report XML file for all package - testCoverageList = glob.glob(os.path.join(workspace, "Build", "**", "*Test*.exe.cov"), recursive=True) - coverageFile = "" - totalCoverageFile = os.path.join(workspaceBuild, 'coverage.cov') - for testCoverage in testCoverageList: - coverageFile = f" --input_coverage={testCoverage}" - if os.path.isfile(totalCoverageFile): - coverageFile += f" --input_coverage={totalCoverageFile}" - ret = RunCmd( - "OpenCppCoverage", - f"--export_type binary:{totalCoverageFile} " + - f"--working_dir={workspaceBuild} " + - f"{coverageFile}" - ) - if ret != 0: - logging.error("UnitTest Coverage: Failed to collect coverage data.") - return 1 - - ret = RunCmd( - "OpenCppCoverage", - f"--export_type cobertura:{os.path.join(workspaceBuild, 'coverage.xml')} " + - f"--working_dir={workspaceBuild} " + - f"--input_coverage={totalCoverageFile}" - ) + testCoverageList = glob.glob(os.path.join(workspace, "Build", "**","*Test*.exe.cov"), recursive=True) + total_cfg_file = os.path.join(buildOutputBase, "total-opencppcoverage.cfg") + if os.path.isfile(total_cfg_file): + os.remove(total_cfg_file) + + with open(total_cfg_file, "w") as f: + for testCoverage in testCoverageList: + f.write(f"input_coverage={testCoverage}\n") + + ret = RunCmd("OpenCppCoverage", f"--export_type cobertura:{workspace}Build/coverage.xml --config_file={total_cfg_file}", workingdir=f"{workspace}Build/") + os.remove(total_cfg_file) + if ret != 0: logging.error("UnitTest Coverage: Failed to generate cobertura format xml.") return 1 return 0 + + def organize_coverage(self, thebuilder) -> int: + """Organize the generated coverage file by INF.""" + db_path = self.parse_workspace(thebuilder) + + workspace = thebuilder.env.GetValue("WORKSPACE") + buildOutputBase = thebuilder.env.GetValue("BUILD_OUTPUT_BASE") + package = thebuilder.env.GetValue("CI_PACKAGE_NAME", "") + file_out = package + "_coverage.xml" + cov_file = os.path.join(buildOutputBase, file_out) + + params = f"--database {db_path} coverage {cov_file} -o {cov_file} --by-package -ws {workspace}" + + params += f" -p {package}" * int(package != "") + params += " --full" * int(thebuilder.env.GetValue("CC_FULL", "FALSE") == "TRUE") + params += " --flatten" * int(thebuilder.env.GetValue("CC_FLATTEN", "FALSE") == "TRUE") + + return RunCmd("stuart_report", params) + + def parse_workspace(self, thebuilder) -> str: + """Parses the workspace with Edk2DB with the tables necessarty to run stuart_report.""" + db_path = os.path.join(thebuilder.env.GetValue("BUILD_OUTPUT_BASE"), "DATABASE.db") + with Edk2DB(db_path, thebuilder.edk2path) as db: + db.register(SourceTable(), PackageTable(), InfTable()) + env_dict = thebuilder.env.GetAllBuildKeyValues() | thebuilder.env.GetAllNonBuildKeyValues() + db.parse(env_dict) + + return db_path + # MU_CHANGE end - reformat coverage data diff --git a/UnitTestFrameworkPkg/ReadMe.md b/UnitTestFrameworkPkg/ReadMe.md index bde6bff7356..c2111aaa46c 100644 --- a/UnitTestFrameworkPkg/ReadMe.md +++ b/UnitTestFrameworkPkg/ReadMe.md @@ -1493,47 +1493,61 @@ This mode is used by the test running plugin to aggregate the results for CI tes ### Code Coverage -Host based Unit Tests will automatically enable coverage data. +Code coverage can be enabled for Host based Unit Tests with `CODE_COVERAGE=TRUE`, which generates a cobertura report +per package tested, and combined cobertura report for all packages tested. The per-package cobertura report will be +present at `Build//HostTest//_coverage.xml`. The overall cobertura report will be present +at `Build/coverage.xml` + +Code coverage generation has two config knobs: + +1. `CC_FULL`: If set to `TRUE`, will generate zero'd out coverage data for untested source files in the package. +2. `CC_FLATTEN`: If Set to `TRUE`, will group all source files together, rather than by INF. + +**TIP: `CC_FLATTEN=TRUE/FALSE` will produce different coverage percentage results as `TRUE` de-duplicates source files +that are consumed by multiple INFs. For Windows, this is primarily leveraged for pipeline builds, but this can be leveraged locally using the OpenCppCoverage windows tool to parse coverage data to cobertura xml format. +#### Prerequisites + +In addition to required prerequisites to build and test, there are additional requirements for calculating code +coverage files as noted below. + * Windows Prerequisite - ```text - Download and install https://github.com/OpenCppCoverage/OpenCppCoverage/releases - python -m pip install --upgrade -r ./pip-requirements.txt - stuart_ci_build -c .pytool/CISettings.py -t NOOPT TOOL_CHAIN_TAG=VS2019 -p MdeModulePkg - Open Build/coverage.xml - ``` + 1. OpenCppCoverage: Download and install - * How to see code coverage data on IDE Visual Studio +* Linux Prerequisite - ```text - Open Visual Studio VS2019 or above version - Click "Tools" -> "OpenCppCoverage Settings" - Fill your execute file into "Program to run:" - Click "Tools" -> "Run OpenCppCoverage" - ``` + 1. lcov: sudo apt-get install -y lcov -For Linux, this is primarily leveraged for pipeline builds, but this can be leveraged locally using the -lcov linux tool, and parsed using the lcov_cobertura python tool to parse it to cobertura xml format. +#### Examples -* Linux Prerequisite +```bash +stuart_ci_build -c .pytool/CISettings.py -t NOOPT TOOL_CHAIN_TAG=VS2019 -p MdeModulePkg CODE_COVERAGE=TRUE +stuart_ci_build -c .pytool/CISettings.py -t NOOPT TOOL_CHAIN_TAG=VS2019 CODE_COVERAGE=TRUE CC_FLATTEN=TRUE CC_FULL=FALSE +``` + +How to see code coverage data on IDE Visual Studio + +```text +Open Visual Studio VS2019 or above version +Click "Tools" -> "OpenCppCoverage Settings" +Fill your execute file into "Program to run:" +Click "Tools" -> "Run OpenCppCoverage" +``` - ```bash - sudo apt-get install -y lcov - python -m pip install --upgrade -r ./pip-requirements.txt - stuart_ci_build -c .pytool/CISettings.py -t NOOPT TOOL_CHAIN_TAG=GCC5 -p MdeModulePkg - Open Build/coverage.xml - ``` +#### Additional Tools - * How to see code coverage data on IDE Visual Studio Code +There are a plethora of open source tools for generating reports from a Cobertura file, which is why it was selected as +the output file format. Tools such as pycobertura (`pip install pycobertura`) and [reportgenerator](https://www.nuget.org/packages/dotnet-reportgenerator-globaltool) +can be utilized to generate different report types, such as local html reports. VSCode Extensions such as [Coverage Gutters](https://marketplace.visualstudio.com/items?itemName=ryanluker.vscode-coverage-gutters) +can highlight coverage results directly in the file, and cloud tools such as [CodeCov](https://about.codecov.io/) can +consume cobertura files to provide PR checks and general code coverage statistics for the repository. - ```bash - Download plugin "Coverage Gutters" - Press Hot Key "Ctrl + Shift + P" and click option "Coverage Gutters: Display Coverage" - ``` +*** REMINDER: During CI builds, use the ``CODE_COVERAGE=TRUE` flag to generate the code coverage XML files, +and additionally use the `CC_FLATTEN=TRUE` or `CC_FULL=TRUE` flags to customize coverage results. ### Important Note diff --git a/pip-requirements.txt b/pip-requirements.txt index 5d542093ed1..1848841d51c 100644 --- a/pip-requirements.txt +++ b/pip-requirements.txt @@ -13,9 +13,10 @@ ## edk2-pytool-library~=0.19.4 # MU_CHANGE -edk2-pytool-extensions~=0.26.0 # MU_CHANGE +edk2-pytool-extensions~=0.26.2 # MU_CHANGE edk2-basetools==0.1.49 antlr4-python3-runtime==4.13.1 +regex lcov-cobertura==2.0.2 -regex==2023.8.8 +pygount==1.6.1 # MU_CHANGE toml==0.10.2 # MU_CHANGE