From 070418b9a246fb77d764238545c1e657f9b81c59 Mon Sep 17 00:00:00 2001 From: Joey Vagedes Date: Wed, 5 Jul 2023 08:39:11 -0700 Subject: [PATCH] CI: Add Rust Host Unit Test CI check (#481) Adds a CI plugin, RustHostUnitTestPlugin, with the scope `rust-ci` that runs all tests, ensuring they pass. If they pass, code coverage is calculated, which must meet the requirements specified in a package's ci.yaml file (default is 75% code coverage). Will generate a coverage.xml file in the Build directory. - [ ] Impacts functionality? - **Functionality** - Does the change ultimately impact how firmware functions? - Examples: Add a new library, publish a new PPI, update an algorithm, ... - [ ] Impacts security? - **Security** - Does the change have a direct security impact on an application, flow, or firmware? - Examples: Crypto algorithm change, buffer overflow fix, parameter validation improvement, ... - [ ] Breaking change? - **Breaking change** - Will anyone consuming this change experience a break in build or boot behavior? - Examples: Add a new library class, move a module to a different repo, call a function in a new library class in a pre-existing module, ... - [ ] Includes tests? - **Tests** - Does the change include any explicit test code? - Examples: Unit tests, integration tests, robot tests, ... - [x] Includes documentation? - **Documentation** - Does the change contain explicit documentation additions outside direct code modifications (and comments)? - Examples: Update readme file, add feature readme file, link to documentation on an a separate Web page, ... Ensured Check can run successfully Add the `rust-ci` scope to your settings python file for stuart_ci_build. add `toml` to your pip-requirements.txt file. Signed-off-by: Joey Vagedes --- .../Plugin/RustHostUnitTestPlugin/Readme.md | 21 +++ .../RustHostUnitTestPlugin.py | 106 +++++++++++++++ .../RustHostUnitTest_plug_in.yaml | 12 ++ .../RustPackageHelper/RustPackageHelper.py | 128 ++++++++++++++++++ .../RustPackageHelper_plug_in.yaml | 12 ++ Docs/rust_build.md | 7 +- Makefile.toml | 7 +- pip-requirements.txt | 1 + 8 files changed, 288 insertions(+), 6 deletions(-) create mode 100644 .pytool/Plugin/RustHostUnitTestPlugin/Readme.md create mode 100644 .pytool/Plugin/RustHostUnitTestPlugin/RustHostUnitTestPlugin.py create mode 100644 .pytool/Plugin/RustHostUnitTestPlugin/RustHostUnitTest_plug_in.yaml create mode 100644 .pytool/Plugin/RustPackageHelper/RustPackageHelper.py create mode 100644 .pytool/Plugin/RustPackageHelper/RustPackageHelper_plug_in.yaml diff --git a/.pytool/Plugin/RustHostUnitTestPlugin/Readme.md b/.pytool/Plugin/RustHostUnitTestPlugin/Readme.md new file mode 100644 index 0000000000..85252f40eb --- /dev/null +++ b/.pytool/Plugin/RustHostUnitTestPlugin/Readme.md @@ -0,0 +1,21 @@ +# Rust Host Unit Test Plugin + +This CI plugin runs all unit tests with coverage enabled, calculating coverage results on a per package basis. It filters results to only calculate coverage on files within the package. + +This CI plugin will also calculate coverage for the entire workspace. + +## Plugin Customizations + +As a default, this plugin requires 75% coverage, though this can be configured within a packages ci.yaml file by adding the entry `RustCoverageCheck`. The required coverage percent can also be customized on a per (rust) package bases. + +### Example ci settings + +``` yaml +"RustHostUnitTestPlugin": { + "Coverage": 1, + "CoverageOverrides": { + "DxeRust": 0.0, + "UefiEventLib": 0.0, + } +} +``` diff --git a/.pytool/Plugin/RustHostUnitTestPlugin/RustHostUnitTestPlugin.py b/.pytool/Plugin/RustHostUnitTestPlugin/RustHostUnitTestPlugin.py new file mode 100644 index 0000000000..758730e538 --- /dev/null +++ b/.pytool/Plugin/RustHostUnitTestPlugin/RustHostUnitTestPlugin.py @@ -0,0 +1,106 @@ +# @file RustHostUnitTestPlugin.py +# CiBuildPlugin used to run cargo tarpaulin for all host based tests. +# Ensures that all host based tests pass and meet code coverage requirements. +## +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: BSD-2-Clause-Patent +## +from edk2toolext.environment.plugintypes.ci_build_plugin import ICiBuildPlugin +from typing import List +from edk2toolext.environment.repo_resolver import repo_details +from pathlib import Path +import re +import logging + +class RustHostUnitTestPlugin(ICiBuildPlugin): + def GetTestName(self, packagename: str, environment: object) -> tuple[str, str]: + return (f'Host Unit Tests in {packagename}', f'{packagename}.RustHostUnitTestPlugin') + + def RunsOnTargetList(self) -> List[str]: + return ["NO-TARGET"] + + def RunBuildPlugin(self, packagename, Edk2pathObj, pkgconfig, environment, PLM, PLMHelper, tc, output_stream): + + ws = Edk2pathObj.WorkspacePath + rust_ws = PLMHelper.RustWorkspace(ws) # .pytool/Plugin/RustPackageHelper + + # Build list of packages that are in the EDK2 package we are running CI on + pp = Path(Edk2pathObj.GetAbsolutePathOnThisSystemFromEdk2RelativePath(packagename)) + package_name_list = [pkg.name for pkg in filter(lambda pkg: Path(pkg.path).is_relative_to(pp), rust_ws.members)] + package_path_list = [pkg.path for pkg in filter(lambda pkg: Path(pkg.path).is_relative_to(pp), rust_ws.members)] + logging.debug(f"Rust Packages to test: {' '.join(package_name_list)}") + + # Build a list of paths to ignore when computing results. This includes: + # 1. Any tests folder in a rust package + # 2. Everything in a submodule + # 3. Everything in an EDK2 package not being tested. + ignore_list = [str(Path("**", "tests", "*"))] + ignore_list.extend([str(Path(s, "**", "*")) for s in repo_details(ws)["Submodules"]]) + ignore_list.extend(list(set([pkg.path for pkg in rust_ws.members]) - set(package_path_list))) + logging.debug(f"Paths to ignore when computing coverage: {' '.join(ignore_list)}") + + # Run tests and evaluate results + results = rust_ws.coverage(package_name_list, ignore_list = ignore_list, report_type = "xml") + + # Evaluate unit test results + failed = 0 + for test in results["pass"]: + tc.LogStdOut(f'{test} ... PASS') + + for test in results["fail"]: + tc.LogStdError(f'{test} ... FAIL') + failed += 1 + + # If we failed a unit test, we have no coverage data to evaluate + if failed > 0: + tc.SetFailed(f'Host unit tests failed. Failures {failed}', "CHECK_FAILED") + return failed + + # Calculate coverage + coverage = {} + for file, cov in results["coverage"].items(): + try: + package = next(pkg.name for pkg in rust_ws.members if Path(ws,file).is_relative_to(pkg.path)) + except StopIteration: + continue + covered, total = cov.split("/") + if package in coverage: + coverage[package]["cov"] += int(covered) + coverage[package]["total"] += int(total) + else: + coverage[package] = {"cov": int(covered), "total": int(total)} + + # Evaluate coverage results + default_cov = pkgconfig.get("coverage", 0.75) + for pkg, cov in coverage.items(): + required_cov = pkgconfig.get("CoverageOverrides", {pkg: default_cov}).get(pkg, default_cov) + + calc_cov = round(cov["cov"] / cov["total"], 2) + if calc_cov >= required_cov: + tc.LogStdOut(f'coverage::{pkg}: {calc_cov} greater than {required_cov} ... PASS') + else: + tc.LogStdError(f'coverage::{pkg}: {calc_cov} less than {required_cov} ... FAIL') + failed += 1 + + # Move coverage.xml to Build Directory + xml = Path(rust_ws.path) / "target" / "cobertura.xml" + out = Path(rust_ws.path) / "Build" + + if (out / "coverage.xml").exists(): + (out / "coverage.xml").unlink() + xml = xml.rename(out / "coverage.xml") + + with open(xml, 'r') as f: + contents = f.read() + contents = re.sub(r'(.*?)', r'.', contents) + + with open (xml, "w") as f: + f.write(contents) + + # Return + if failed > 0: + tc.SetFailed(f'Coverage requirements not met. Failures {failed}', "CHECK_FAILED") + else: + tc.SetSuccess() + + return failed diff --git a/.pytool/Plugin/RustHostUnitTestPlugin/RustHostUnitTest_plug_in.yaml b/.pytool/Plugin/RustHostUnitTestPlugin/RustHostUnitTest_plug_in.yaml new file mode 100644 index 0000000000..badb0eac9f --- /dev/null +++ b/.pytool/Plugin/RustHostUnitTestPlugin/RustHostUnitTest_plug_in.yaml @@ -0,0 +1,12 @@ +## @file RustHostUnitTest_plug_in.yaml +# IUefiBuildPlugin used to compile and run any rust unit tests. +# Will also calculate code coverage. +# +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: BSD-2-Clause-Patent +## +{ + "scope": "rust-ci", + "name": "Rust Host-Based Unit Test Runner", + "module": "RustHostUnitTestPlugin" +} diff --git a/.pytool/Plugin/RustPackageHelper/RustPackageHelper.py b/.pytool/Plugin/RustPackageHelper/RustPackageHelper.py new file mode 100644 index 0000000000..0b7af0c2cd --- /dev/null +++ b/.pytool/Plugin/RustPackageHelper/RustPackageHelper.py @@ -0,0 +1,128 @@ +# @file RustPackageHelper.py +# HelperFucntion used to share the RustPackage +# class to the rest of the build system. +## +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: BSD-2-Clause-Patent +## +import io +import os +import toml +from typing import Union +from pathlib import Path +from edk2toolext.environment.plugintypes.uefi_helper_plugin import IUefiHelperPlugin +from edk2toollib.utility_functions import RunCmd + + +class RustPackage: + def __init__(self, path: Path): + self.path = path + self.name = path.name + + +class RustWorkspace: + def __init__(self, path: Union[Path, str]): + self.path: Path = Path(path) + self.toml: dict = {} + self.members: list[RustPackage] = [] + + self.__load_toml() + self.__set_members() + + def __load_toml(self): + """Loads the repositories Cargo.toml file as a dictionary.""" + try: + self.toml = toml.load(self.path / "Cargo.toml") + except Exception: + raise Exception(f"Failed to load Cargo.toml from {self.path}") + + def __set_members(self): + """Finds all members of the workspace.""" + workspace = self.toml.get("workspace") + members = set() + + # Grab all members specifically specified in the workspace + for member in workspace["members"]: + members.add(RustPackage(self.path / member)) + + # Build a dep list that only contains dependencies with a path. These are workspace + # members. + dep_list = workspace["dependencies"] + dep_list = [dep_list[dep] for dep in dep_list if type(dep_list[dep]) != str and dep_list[dep].get("path")] + + for dep in dep_list: + members.add(RustPackage(self.path / dep["path"])) + + self.members = list(members) + + def coverage(self, pkg_list = None, ignore_list = None, report_type: str = "html" ): + """Runs coverage at the workspace level. + + Generates a single report that provides coverage information for all + packages in the workspace. + """ + if pkg_list is None: + pkg_list = [pkg.name for pkg in self.members] + + # Set up the command + command = "cargo" + params = "make" + if ignore_list: + params += f' -e COV_FLAGS="--out {report_type} --exclude-files {",".join(ignore_list)}"' + else: + params += f' -e COV_FLAGS="--out {report_type}"' + params += f" coverage {','.join(pkg_list)}" + + # Run the command + output = io.StringIO() + RunCmd(command, params, workingdir=self.path, outstream=output) + output.seek(0) + lines = output.readlines() + + result = { + "pass": [], + "fail": [], + "coverage": {} + } + + # Determine passed and failed tests + for line in lines: + line = line.strip().strip("\n") + + if line.startswith("test result:"): + continue + + if line.startswith("test "): + line = line.replace("test ", "") + if line.endswith("... ok"): + result["pass"].append(line.replace(" ... ok", "")) + else: + result["fail"].append(line.replace(" ... FAILED", "")) + continue + + if len(result["fail"]) > 0: + return result + + # Determine coverage if all tests passed + for line in lines: + line = line.strip().strip("\n") + if line.startswith("|| Tested/Total Lines"): + continue + + if line == "||": + continue + + if line.startswith("||"): + line = line.replace("|| ", "") + path, cov = line.split(":") + cov = cov.split()[0] + result["coverage"][path] = cov + + return result + + +class RustPackageHelper(IUefiHelperPlugin): + def RegisterHelpers(self, obj): + fp = os.path.abspath(__file__) + obj.Register("RustPackage", RustPackage, fp) + obj.Register("RustWorkspace", RustWorkspace, fp) diff --git a/.pytool/Plugin/RustPackageHelper/RustPackageHelper_plug_in.yaml b/.pytool/Plugin/RustPackageHelper/RustPackageHelper_plug_in.yaml new file mode 100644 index 0000000000..8f53b042ff --- /dev/null +++ b/.pytool/Plugin/RustPackageHelper/RustPackageHelper_plug_in.yaml @@ -0,0 +1,12 @@ +## @file RustPackageHelper_plug_in.yaml +# HelperFucntion used to share the RustPackage +# class to the rest of the build system. +# +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: BSD-2-Clause-Patent +## +{ + "scope": "global", + "name": "Rust Package Helper", + "module": "RustPackageHelper" +} diff --git a/Docs/rust_build.md b/Docs/rust_build.md index fa60cc36fb..15dce583dc 100644 --- a/Docs/rust_build.md +++ b/Docs/rust_build.md @@ -94,7 +94,12 @@ The following command line options are available: cargo make test ``` -- Note: If a package is not specified, all packages will be tested. +```cmd +cargo make coverage +``` + +- Note: If a package is not specified, all packages will be tested. Multiple packages can be provided, comma separated. +- cargo make coverage to generate coverage results on top of testing. ### Supported Build Combinations diff --git a/Makefile.toml b/Makefile.toml index 46bb15bb32..a091f29379 100644 --- a/Makefile.toml +++ b/Makefile.toml @@ -10,7 +10,7 @@ PACKAGE_TARGET = {value = "-p ${CARGO_MAKE_TASK_ARGS}",condition = { env_true = BUILD_FLAGS = "--profile ${RUSTC_PROFILE} --target ${TARGET_TRIPLE} -Zbuild-std=core,compiler_builtins,alloc -Zbuild-std-features=compiler-builtins-mem -Zunstable-options --timings=html" TEST_FLAGS = { value = "", condition = { env_not_set = ["TEST_FLAGS"] } } -COV_FLAGS = { value = "--out Html", condition = { env_not_set = ["COV_FLAGS"] } } +COV_FLAGS = { value = "--out Html --exclude-files **/tests/*", condition = { env_not_set = ["COV_FLAGS"] } } [env.development] RUSTC_PROFILE = "dev" @@ -43,10 +43,7 @@ command = "cargo" args = ["test", "@@split(PACKAGE_TARGET, )", "@@split(TEST_FLAGS, )"] [tasks.coverage] -disabled = true - -[tasks.cov] description = "Build and run all tests and calculate coverage." clear = true command = "cargo" -args = ["tarpaulin", "@@split(PACKAGE_TARGET, )", "@@split(COV_FLAGS, )", "--exclude-files", "**/tests/*", "--output-dir", "${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/target"] +args = ["tarpaulin", "@@split(PACKAGE_TARGET, )", "@@split(COV_FLAGS, )", "--output-dir", "${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/target"] diff --git a/pip-requirements.txt b/pip-requirements.txt index 5175ca568b..c8047ba7b8 100644 --- a/pip-requirements.txt +++ b/pip-requirements.txt @@ -18,3 +18,4 @@ edk2-basetools==0.1.48 antlr4-python3-runtime==4.13.0 lcov-cobertura==2.0.2 regex==2023.8.8 +toml==0.10.2 # MU_CHANGE