From 2f1d69bac195739535fe8b6156716dd16fda3393 Mon Sep 17 00:00:00 2001 From: Matthew Brady Date: Thu, 1 Aug 2024 15:41:27 +0100 Subject: [PATCH] Modify README & project name --- README.md | 92 +++++++++++++++++++--------- pyproject.toml | 10 +-- yocto_import_sbom/BBClass.py | 2 +- yocto_import_sbom/ConfigClass.py | 8 +-- yocto_import_sbom/OEClass.py | 33 ++++++---- yocto_import_sbom/RecipeListClass.py | 21 ++++++- yocto_import_sbom/SBOMClass.py | 6 +- yocto_import_sbom/main.py | 24 +++++--- 8 files changed, 129 insertions(+), 67 deletions(-) diff --git a/README.md b/README.md index 3612033..238e70f 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Synopsys Scan Yocto Script - bd_yocto_import_sbom.py v1.0 +# Synopsys Scan Yocto Script - bd_scan_yocto_via_sbom.py v1.0.2 # PROVISION OF THIS SCRIPT This script is provided under the MIT license (see LICENSE file). @@ -8,44 +8,68 @@ It does not represent any extension of licensed functionality of Synopsys softwa If you have comments or issues, please raise a GitHub issue here. Synopsys support is not able to respond to support tickets for this OSS utility. Users of this pilot project commit to engage properly with the authors to address any identified issues. # INTRODUCTION -## OVERVIEW OF BD_YOCTO_IMPORT_SBOM +## OVERVIEW OF BD_SCAN_YOCTO_VIA_SBOM This utility is intended to: -- Scan a Yocto project to create an SPDX SBOM file which is uploaded to the specified Black Duck server to create a project version -- Optionally filter recipes using data from the OpenEmbedded APIs to 'fix-up' recipes moved to new layers or with different versions/revisions -- Signature scan packages/downloaded archives +- Scan a Yocto project to generate an SPDX SBOM file which can be uploaded to the specified Black Duck server to create a project version +- Filter recipes using data from the OpenEmbedded APIs to 'fix-up' recipes moved to new layers or with different versions/revisions +- Signature scan packages/downloaded archives (not from OE) - Apply patches for locally patched CVEs identified from `cve_patch` if this data is available This utility has some benefits over alternative Black Duck Yocto scan processes (including [Synopsys Detect](https://detect.synopsys.com/doc) and [bd-scan-yocto](https://github.com/matthewb66/bd_scan_yocto)). -There are several important options to modify the behaviour including: -- Skip use of bitbake command to locate project files by specifying license.manifest and bitbake-layers output file (use --skip_bitbake) -- Use data from the OpenEmbedded API to verify recipes, layers, version to ensure they match known components in the BD KB (use --get_oe_date) -- Cache OE data in JSON files in a local folder to remove need to download on every run (use --oe_data_folder) -- Specify semantic version distance for matching recipes against OE data (default distance is '0.0.0' - use --max_oe_version_distance) -- Create SPDX output file and do not upload to Black Duck to create project (use --output_file) -- Specify license.manifest, machine, target, download_dir, package_dir, image_package_type to override values extracted from Bitbake environment -- Skip Signature scan of downloaded archives and packages (use --skip_sig_scan) - Note that, from Black Duck version 2024.7 onwards, the use of SPDX SBOM upload provides for the optional, automatic creation of custom components for recipes not matched in the BD KB. This would enable the creation of a complete SBOM including 3rd party or local, custom components. +See the RECOMMENDATIONS section below for guidance on optimising Yocto project scans. + +## OPTIONAL BEHAVIOUR + +There are several important options to modify the behaviour of this utility including: +- Skip use of bitbake command to locate project files by specifying license.manifest and bitbake-layers output file (use `--skip_bitbake`) +- Use data from the OpenEmbedded API to verify recipes, layers, version to ensure they match known components in the BD KB (use `--skip_oe_data` to skip) +- Cache OE data in JSON files in a local folder to remove need to download on every run (use `--oe_data_folder FOLDER`) +- Specify semantic version distance for matching recipes against OE data (default distance is '0.0.0' - use `--max_oe_version_distance X.X.X`) +- Create SPDX output file and do not upload to Black Duck to create project (use `--output_file OUTPUT`) +- Specify license.manifest, machine, target, download_dir, package_dir, image_package_type to override values extracted from Bitbake environment +- Skip Signature scan of downloaded archives and packages (use `--skip_sig_scan`) + ## INSTALLATION -1. Clone the repository +1. Create virtualenv +2. Run `pip3 install bd_scan_yocto_via_sbom` + +Alternatively, if you want to manage the repository locally: + +1. clone the repository 2. Create virtualenv -3. Install required dependencies: - 1. Run `pip3 install blackduck` +3. Build the utility `python -m build` +4. Install the package `pip3 install dist/bd_scan_yocto_via_sbom-1.0.X-py3-none-any.whl` ## EXECUTION -Run the utility using python: +Run the utility as a package: + +1. Set the Bitbake environment (for example `source oe-init-build-env`) +2. Run `bd-scan-yocto-via-sbom OPTIONS` - python3 run.py OPTIONS +Alternatively, if you have installed the repository locally: + +1. Set the Bitbake environment (for example `source oe-init-build-env`) +2. Run `python3 run.py OPTIONS` + +## RECOMMENDATIONS + +For optimal Yocto scan results, consider the following: + +1. The utility will call Bitbake to extract the environment and layer information. You can override values (license.manifest, machine, target, download_dir, package_dir, image_package_type) extracted from the environment using command line options. +2. Use the `--oe_data_folder FOLDER` option to cache the downloaded OE data (~300MB on every run) noting that the data does not change frequently. +3. Add the `cve_check` class to the local.conf to ensure patched CVEs are identified and check that PHASE 6 picks up the cve-check file (see CVE PATCHING below). +4. Use the `--max_oe_version_distance X.X.X` option to specify fuzzy matching for OE recipes (values in the range '0.0.1' to '0.0.10' are recommended). ## USAGE - usage: bd-yocto-import-sbom [-h] [--blackduck_url BLACKDUCK_URL] [--blackduck_api_token BLACKDUCK_API_TOKEN] [--blackduck_trust_cert] [-p PROJECT] [-v VERSION] [-l LICENSE_MANIFEST] + usage: bd-scan-yocto-via-sbom [-h] [--blackduck_url BLACKDUCK_URL] [--blackduck_api_token BLACKDUCK_API_TOKEN] [--blackduck_trust_cert] [-p PROJECT] [-v VERSION] [-l LICENSE_MANIFEST] [-b BITBAKE_LAYERS] [-c CVE_CHECK_FILE] [-o OUTPUT] [--debug] [--logfile LOGFILE] Create BD Yocto project from license.manifest @@ -79,7 +103,7 @@ Run the utility using python: Specify output SBOM SPDX file for manual upload (if specified then BD project will not be created automatically and CVE patching not supported) - --get_oe_data Download and use OE data to check layers, versions & + --skip_oe_data Download and use OE data to check layers, versions & revisions --oe_data_folder OE_DATA_FOLDER Folder to contain OE data files - if files do not @@ -130,11 +154,11 @@ process CVE patch status; not required if `--output` specified. : Optionally specify yocto license.manifest file created by Bitbake. Usually located in the folder `tmp/deploy/licenses/-/license.manifest`. Should be determined from Bitbake environment -by default (unless --skip_bitbake used). +by default (unless `--skip_bitbake` used). --target TARGET (also -t): -: Bitbake target (e.g. `core-image-sato`) - required unless --skip_bitbake used. +: Bitbake target (e.g. `core-image-sato`) - required unless `--skip_bitbake` used. --skip_bitbake: @@ -144,19 +168,19 @@ output to be specified. --bitbake_layers_file BITBAKE_OUTPUT_FILE (also -b): : Optionally specify output of the command `bitbake-layers show-recipes` stored in the specified file. Should be determined from Bitbake environment -by default (unless --skip_bitbake used). +by default (unless `--skip_bitbake` used). --cve_checkfile CVE_CHECK_FILE: : Optionally specify output file from run of the `cve_check` custom class which generates a list of patched CVEs. Usually located in the -folder `build/tmp/deploy/images/XXX`. Should be determined from Bitbake environment by default (unless --skip_bitbake used) +folder `build/tmp/deploy/images/XXX`. Should be determined from Bitbake environment by default (unless `--skip_bitbake` used) ---get_oe_data: +--skip_oe_data: -: Download layers/recipes/layers from layers.openembedded.org APIs to review origin layers and revisions used in recipes to +: Do not download layers/recipes/layers from layers.openembedded.org APIs to review origin layers and revisions used in recipes to ensure more accurate matching and complete BOMs. ---oe_data_folder: +--oe_data_folder FOLDER: : Create OE data files in the specified folder if not already existing. If OE data files exist already in this folder, use them to review layers and revisions to ensure more accurate matching and complete BOMs. Allows offline usage @@ -164,8 +188,8 @@ of OE data or reduction of large data transfers if script is run frequently. --skip_sig_scan: -: Do not send identified package and downloaded archives for Signature scanning. By default, if --get_oe_data is used, only recipes -not matched against OE data will be scanned, use --scan_all_files to scan all recipes. +: Do not send identified package and downloaded archives for Signature scanning. By default, only recipes +not matched against OE data will be scanned, use `--scan_all_files` to scan all recipes. --max_oe_version_distance MAJOR.MINOR.PATCH: @@ -183,3 +207,11 @@ value should probably be in the range 0.0.1 to 0.0.10). - Recipe version is 3.2.4 - closest previous OE recipe version is 3.2.1: Distance value would need to be minimum 0.0.3 - Recipe version is 3.2.4 - closest previous OE recipe version is 3.0.1: Distance value would need to be minimum 0.2.0 - Recipe version is 3.2.4 - closest previous OE recipe version is 2.0.1: Distance value would need to be minimum 1.0.0 + +## CVE PATCHING + +For patched CVE remediation in the Black Duck project, you will need to add the `cve_check` bbclass to the Yocto build configuration to generate the CVE check log output. Add the following line to the `build/conf/local.conf` file: + + INHERIT += "cve-check" + +Then rebuild the project (using for example `bitbake core-image-sato`) to run the CVE check action and generate the required CVE log files without a full rebuild. diff --git a/pyproject.toml b/pyproject.toml index 054a83e..050102b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,8 +3,8 @@ requires = ["setuptools>=67.0"] build-backend = "setuptools.build_meta" [project] -name = "bd_yocto_import_sbom" -version = "1.0.1" +name = "bd_scan_yocto_via_sbom" +version = "1.0.2" authors = [ { name="Matthew Brady", email="mbrad@synopsys.com" }, ] @@ -23,8 +23,8 @@ dependencies = [ ] [project.urls] -Homepage = "https://github.com/matthewb66/bd_yocto_import_sbom" -Issues = "https://github.com/matthewb66/bd_yocto_import_sbom/issues" +Homepage = "https://github.com/matthewb66/bd_scan_yocto_via_sbom" +Issues = "https://github.com/matthewb66/bd_scan_yocto_via_sbom/issues" [project.scripts] -bd-yocto-import = "yocto_import_sbom:main.main" \ No newline at end of file +bd-scan-yocto-via-sbom = "yocto_import_sbom:main.main" \ No newline at end of file diff --git a/yocto_import_sbom/BBClass.py b/yocto_import_sbom/BBClass.py index 4bb9915..a6feb4a 100644 --- a/yocto_import_sbom/BBClass.py +++ b/yocto_import_sbom/BBClass.py @@ -130,7 +130,7 @@ def process_bitbake_env(self, conf): if not conf.package_dir and conf.deploy_dir: temppath = os.path.join(conf.deploy_dir, conf.image_pkgtype) if os.path.isdir(temppath): - conf.pkg_dir = temppath + conf.package_dir = temppath @staticmethod def run_cmd(command): diff --git a/yocto_import_sbom/ConfigClass.py b/yocto_import_sbom/ConfigClass.py index b0bf896..48e13b2 100644 --- a/yocto_import_sbom/ConfigClass.py +++ b/yocto_import_sbom/ConfigClass.py @@ -43,7 +43,7 @@ def __init__(self): help="OPTIONAL Specify output SBOM SPDX file for manual upload (if specified then BD " "project will not be created automatically and CVE patching not supported)", default="") - parser.add_argument("--get_oe_data", + parser.add_argument("--skip_oe_data", help="OPTIONAL Download and use OE data to check layers, versions & revisions", action='store_true') parser.add_argument("--oe_data_folder", @@ -95,7 +95,7 @@ def __init__(self): self.machine = args.machine self.bitbake_layers_file = '' self.cve_check_file = '' - self.get_oe_data = False + self.skip_oe_data = False self.max_oe_version_distance = '' self.oe_data_folder = args.oe_data_folder self.package_dir = '' @@ -198,8 +198,8 @@ def __init__(self): logging.error("Black Duck URL/API and output file not specified - nothing to do") terminate = True - if args.get_oe_data: - self.get_oe_data = True + if args.skip_oe_data: + self.skip_oe_data = True distarr = OE.calc_specified_version_distance(args.max_oe_version_distance) if distarr[0] == -1: diff --git a/yocto_import_sbom/OEClass.py b/yocto_import_sbom/OEClass.py index ba1e90c..6b42246 100644 --- a/yocto_import_sbom/OEClass.py +++ b/yocto_import_sbom/OEClass.py @@ -240,6 +240,11 @@ def get_branch_by_layerbranchid(self, id): return {} def compare_recipes(self, conf, recipe, oe_recipe, best_oe_recipe): + # Returns: + # - Bool - Match found + # - Bool - Exact version match + oe_ver_equal = False + try: if best_oe_recipe != {}: logging.debug(f"Comparing {recipe.name}/{recipe.version} to {oe_recipe['pn']}/{oe_recipe['pv']} " @@ -247,7 +252,6 @@ def compare_recipes(self, conf, recipe, oe_recipe, best_oe_recipe): else: logging.debug(f"Comparing {recipe.name}/{recipe.version} to {oe_recipe['pn']}/{oe_recipe['pv']}") - oe_ver_equal = False pref = False ver = Recipe.filter_version_string(recipe.version) @@ -307,10 +311,10 @@ def compare_recipes(self, conf, recipe, oe_recipe, best_oe_recipe): pref = True if pref: - return True + return True, oe_ver_equal except Exception as e: logging.error(f"Error in compare_recipes(): {e}") - return False + return False, False @staticmethod def get_branch_priority(branch): @@ -323,12 +327,20 @@ def get_branch_priority(branch): def get_recipe(self, conf, recipe): # need to look for closest version match + # Return: + # - OE Recipe + # - OE Layer + # - Bool exact version match + # - Bool exact layer match try: best_recipe = {} + exact_ver = False + exact_layer = False if recipe.name in self.recipename_dict.keys(): for oe_recipe in self.recipename_dict[recipe.name]: - if self.compare_recipes(conf, recipe, oe_recipe, best_recipe): + match, exact_ver = self.compare_recipes(conf, recipe, oe_recipe, best_recipe) + if match: best_recipe = oe_recipe recipe.matched_oe = True @@ -345,22 +357,19 @@ def get_recipe(self, conf, recipe): else: logging.debug(f"Recipe {recipe.name}: {recipe.layer}/{recipe.name}/{recipe_ver} - " f"No close (previous) OE version match found") - return {}, {} - - # logging.debug(f"Recipe {recipe.name}: {recipe.layer}/{recipe.name}/{recipe_ver} - " - # f"Recipe {best_layer['name']}/{best_recipe['pn']}/{best_ver}-{best_recipe['pr']} " - # f"exists in OE data but distance {best_distance} exceeds specified max version " - # f"distance {global_values.max_oe_version_distance}") + return {}, {}, False, False best_layer = self.get_layer_by_layerbranchid(best_recipe['layerbranch']) logging.debug(f"Recipe {recipe.name}: {recipe.layer}/{recipe.name}/{recipe_ver} - OE near match " f"{best_layer['name']}/{best_recipe['pn']}/{best_ver}-{best_recipe['pr']}") - return best_recipe, best_layer + if recipe.layer == best_layer['name']: + exact_layer = True + return best_recipe, best_layer, exact_ver, exact_layer except KeyError as e: logging.warning(f"Error getting nearest OE recipe - {e}") - return {}, {} + return {}, {}, False, False @staticmethod def coerce_version(version: str): diff --git a/yocto_import_sbom/RecipeListClass.py b/yocto_import_sbom/RecipeListClass.py index a44d5b8..0174a91 100644 --- a/yocto_import_sbom/RecipeListClass.py +++ b/yocto_import_sbom/RecipeListClass.py @@ -51,11 +51,28 @@ def get_layers(self): def check_recipes_in_oe(self, conf, oe): recipes_in_oe = 0 + exact_recipes_in_oe = 0 + changed_layers = 0 + exact_layers = 0 for recipe in self.recipes: - recipe.oe_recipe, recipe.oe_layer = oe.get_recipe(conf, recipe) + recipe.oe_recipe, recipe.oe_layer, exact_ver, exact_layer = oe.get_recipe(conf, recipe) if recipe.oe_recipe != {}: recipes_in_oe += 1 - logging.info(f"- {recipes_in_oe} out of {self.count()} total recipes found in OE data") + if not exact_layer: + changed_layers += 1 + else: + exact_layers += 1 + if exact_ver: + exact_recipes_in_oe += 1 + + # logging.info(f"- {recipes_in_oe} out of {self.count()} total recipes found in OE data ({exact_recipes_in_oe} " + # f"exact version matches and {changed_layers} recipe layers modified)") + logging.info(f"SUMMARY OE MATCH DATA:") + logging.info(f"- {self.count()} Total Recipes") + logging.info(f"- {recipes_in_oe} Recipes found in OE Data of which:") + logging.info(f" - {exact_recipes_in_oe} have exact version match") + logging.info(f" - {exact_layers} have the same layer as OE") + logging.info(f" - {changed_layers} exist in different OE layer") def scan_pkg_download_files(self, conf, bom): all_pkg_files = BB.get_pkg_files(conf) diff --git a/yocto_import_sbom/SBOMClass.py b/yocto_import_sbom/SBOMClass.py index 253b1d1..6755010 100644 --- a/yocto_import_sbom/SBOMClass.py +++ b/yocto_import_sbom/SBOMClass.py @@ -79,7 +79,7 @@ def add_package(self, recipe): if recipe.oe_recipe == {}: recipe_layer = recipe.layer recipe_name = recipe.name - if recipe.epochbitbake_layers_file: + if recipe.epoch: recipe_version = f"{recipe.epoch}:{recipe.version}" else: recipe_version = recipe.version @@ -142,10 +142,10 @@ def output(self, output_file): except Exception as e: logging.error('Unable to create output SPDX file \n' + str(e)) - sys.exit(3) + return False self.file = output_file - return + return True @staticmethod def filter_special_chars(val): diff --git a/yocto_import_sbom/main.py b/yocto_import_sbom/main.py index b7fd9db..600a8e5 100644 --- a/yocto_import_sbom/main.py +++ b/yocto_import_sbom/main.py @@ -20,20 +20,21 @@ def main(): logging.info("") logging.info("--- PHASE 2 - GET OE DATA ------------------------------------------------") - if conf.get_oe_data: - + if not conf.skip_oe_data: oe_class = OE(conf) reclist.check_recipes_in_oe(conf, oe_class) logging.info("Done processing OE data") else: logging.info("Skipping connection to OE APIs to verify origin layers and revisions " - "(use --get_oe_data to enable)") + "(remove --skip_oe_data to enable)") logging.info("") logging.info("--- PHASE 3 - WRITE SBOM -------------------------------------------------") sbom = SBOM(conf.bd_project, conf.bd_version) sbom.process_recipes(reclist.recipes) - sbom.output(conf.output_file) + if not sbom.output(conf.output_file): + logging.error("Unable to create SBOM file") + sys.exit(2) if conf.output_file: # Create SBOM and terminate @@ -42,6 +43,7 @@ def main(): logging.info("Done") sys.exit(0) + logging.info("Done creating SBOM file") logging.info("") logging.info("--- PHASE 4 - UPLOAD SBOM ------------------------------------------------") bom = BOM(conf) @@ -54,12 +56,14 @@ def main(): logging.info("") logging.info("--- PHASE 5 - SIGNATURE SCAN PACKAGES ------------------------------------") - if not conf.skip_sig_scan and conf.package_dir and conf.download_dir: - - if reclist.scan_pkg_download_files(conf, bom): - logging.error(f"Unable to scan package and download files") - sys.exit(2) - logging.info("Done") + if not conf.skip_sig_scan: + if conf.package_dir and conf.download_dir: + if reclist.scan_pkg_download_files(conf, bom): + logging.error(f"Unable to Signature scan package and download files") + sys.exit(2) + logging.info("Done") + else: + logging.info("Skipped (package_dir or download_dir not identified)") else: logging.info("Skipped (--skip_sig_scan specified)")