Add SPDX generation using spdx-tools

ci/test_files_touched.py

@@ -98,17 +98,29 @@
         'tern report -f json -i photon:3.0',
         'tern report -f spdxtagvalue -i photon:3.0',
         'tern report -f spdxjson -i photon:3.0',
+        'tern report -f spdxyaml -i photon:3.0',
+        'tern report -f spdxxml -i photon:3.0',
+        'tern report -f spdxrdf -i photon:3.0',
         'tern report -d samples/alpine_python/Dockerfile',
         'tern report -f html -i photon:3.0',
         'tern report -f cyclonedxjson -i photon:3.0'],
     # tern/formats/spdx
     re.compile('tern/formats/spdx'): [
-        'tern report -f spdxtagvalue -i photon:3.0 -o spdx.spdx && ' \
-        'java -jar tools-java/target/tools-java-*-jar-with-dependencies.jar '\
+        'tern report -f spdxtagvalue -i photon:3.0 -o spdx.spdx && '
+        'java -jar tools-java/target/tools-java-*-jar-with-dependencies.jar '
         'Verify spdx.spdx',
-        'tern report -f spdxjson -i photon:3.0 -o spdx.json && ' \
-        'java -jar tools-java/target/tools-java-*-jar-with-dependencies.jar '\
-        'Verify spdx.json'],
+        'tern report -f spdxjson -i photon:3.0 -o spdx.json && '
+        'java -jar tools-java/target/tools-java-*-jar-with-dependencies.jar '
+        'Verify spdx.json',
+        'tern report -f spdxyaml -i photon:3.0 -o spdx.yaml && '
+        'java -jar tools-java/target/tools-java-*-jar-with-dependencies.jar '
+        'Verify spdx.yaml',
+        'tern report -f spdxxml -i photon:3.0 -o spdx.xml && '
+        'java -jar tools-java/target/tools-java-*-jar-with-dependencies.jar '
+        'Verify spdx.xml',
+        'tern report -f spdxrdf -i photon:3.0 -o spdx.rdf.xml && '
+        'java -jar tools-java/target/tools-java-*-jar-with-dependencies.jar '
+        'Verify spdx.rdf.xml'],
     # tern/tools
     re.compile('tern/tools'):
     ['tern report -i golang:alpine'],

requirements.txt

@@ -18,4 +18,5 @@ GitPython~=3.1
 prettytable~=3.8
 packageurl-python>=0.11.1
 license-expression>=30.1
+spdx-tools>=0.8.1
 

setup.cfg

@@ -44,14 +44,19 @@ include_package_data = True
 [entry_points]
 tern.formats =
     default = tern.formats.default.generator:Default
-    spdxtagvalue = tern.formats.spdx.spdxtagvalue.generator:SpdxTagValue
-    spdxjson = tern.formats.spdx.spdxjson.generator:SpdxJSON
-    spdxjsonc = tern.formats.spdx.spdxjson.consumer:SpdxJSON
     json = tern.formats.json.generator:JSON
     jsonc = tern.formats.json.consumer:JSON
     yaml = tern.formats.yaml.generator:YAML
     html = tern.formats.html.generator:HTML
     cyclonedxjson = tern.formats.cyclonedx.cyclonedxjson.generator:CycloneDXJSON
+    spdxjson = tern.formats.spdx.spdxjson.generator:SpdxJSON
+    spdxyaml = tern.formats.spdx.spdxyaml.generator:SpdxYAML
+    spdxxml = tern.formats.spdx.spdxxml.generator:SpdxXML
+    spdxrdf = tern.formats.spdx.spdxrdf.generator:SpdxRDF
+    spdxtagvalue = tern.formats.spdx.spdxtagvalue.generator:SpdxTagValue
+    spdxtagvalue_legacy = tern.formats.spdx_legacy.spdxtagvalue.generator:SpdxTagValue
+    spdxjson_legacy = tern.formats.spdx_legacy.spdxjson.generator:SpdxJSON
+    spdxjsonc = tern.formats.spdx_legacy.spdxjson.consumer:SpdxJSON
 tern.extensions =
     cve_bin_tool = tern.extensions.cve_bin_tool.executor:CveBinTool
     scancode = tern.extensions.scancode.executor:Scancode

tern/__main__.py

@@ -214,8 +214,13 @@ def main():
                                metavar='REPORT_FORMAT',
                                help="Format the report using one of the "
                                "available formats: "
-                               "spdxtagvalue, spdxjson, cyclonedxjson, json, "
-                               "yaml, html")
+                               "spdxtagvalue, spdxjson, spdxyaml, spdxxml, spdxrdf, "
+                               "cyclonedxjson, json, yaml, html")
+    parser_report.add_argument('-sv', '--spdx-version',
+                               metavar='SPDX_VERSION',
+                               help="Specify the version of the SPDX report format."
+                                    "This will be ignored for all non-SPDX formats."
+                                    "Possible values are '2.2' (default) or '2.3'.")
     parser_report.add_argument('-o', '--output-file', default=None,
                                metavar='FILE',
                                help="Write the report to a file. "

tern/formats/cyclonedx/cyclonedxjson/generator.py

@@ -50,12 +50,14 @@ def get_document_dict(image_obj_list):
 
 
 class CycloneDXJSON(generator.Generate):
-    def generate(self, image_obj_list, print_inclusive=False):
+    def generate(self, image_obj_list, spdx_version, print_inclusive=False):
         ''' Generate a CycloneDX document
         The whole document should be stored in a dictionary which can be
         converted to JSON and dumped to a file using the write_report function
         in report.py. '''
         logger.debug('Generating CycloneDX JSON document...')
+        if spdx_version is not None:
+            logger.warning("The SPDX version parameter is not supported for CycloneDX JSON.")
 
         report = get_document_dict(image_obj_list)
 

tern/formats/default/generator.py

@@ -140,8 +140,11 @@ def print_licenses_only(image_obj_list):
 
 
 class Default(generator.Generate):
-    def generate(self, image_obj_list, print_inclusive=False):
+    def generate(self, image_obj_list, spdx_version: str, print_inclusive=False):
         '''Generate a default report'''
+        if spdx_version is not None:
+            logger.warning("The SPDX version parameter is not supported for the default format.")
+
         report = formats.disclaimer.format(
             version_info=content.get_tool_version())
         logger.debug('Creating a detailed report of components in image...')
@@ -154,8 +157,11 @@ def generate(self, image_obj_list, print_inclusive=False):
             return report
         return report + print_licenses_only(image_obj_list)
 
-    def generate_layer(self, layer):
+    def generate_layer(self, layer, spdx_version: str):
         """Generate a default report for one layer object"""
+        if spdx_version is not None:
+            logger.warning("The SPDX version parameter is not supported for the default format.")
+
         report = formats.disclaimer.format(
             version_info=content.get_tool_version())
         logger.debug("Generating summary report for layer...")

tern/formats/generator.py

@@ -10,6 +10,6 @@
 class Generate(metaclass=ABCMeta):
     '''Base class for report plugins'''
     @abstractmethod
-    def generate(self, image_obj_list, print_inclusive=False):
+    def generate(self, image_obj_list, spdx_version, print_inclusive=False):
         '''Format the report according to the plugin style.
         Each subclass is responsible for their own formatting.'''

tern/formats/html/generator.py

@@ -321,16 +321,22 @@ def get_report_dict(image_obj_list):
 
 
 class HTML(generator.Generate):
-    def generate(self, image_obj_list, print_inclusive=False):
+    def generate(self, image_obj_list, spdx_version: str, print_inclusive=False):
         '''Given a list of image objects, create a html report
         for the images'''
+        if spdx_version is not None:
+            logger.warning("The SPDX version parameter is not supported for HTML.")
+
         report_dict = get_report_dict(image_obj_list)
         report = create_html_report(report_dict, image_obj_list)
         return report
 
-    def generate_layer(self, layer):
+    def generate_layer(self, layer, spdx_version: str):
         """Given a layer object, create a html report for the layer"""
         logger.debug("Creating HTML report...")
+        if spdx_version is not None:
+            logger.warning("The SPDX version parameter is not supported for HTML.")
+
         report = ""
         report = report + '\n' + head_layer % (css, get_tool_version())
         report = report + '\n' + report_dict_to_html(layer.to_dict())

tern/formats/json/generator.py

@@ -8,18 +8,28 @@
 """
 
 import json
+import logging
+
 from tern.formats import generator
+from tern.utils import constants
 
+logger = logging.getLogger(constants.logger_name)
 
 class JSON(generator.Generate):
-    def generate(self, image_obj_list, print_inclusive=False):
+    def generate(self, image_obj_list, spdx_version: str, print_inclusive=False):
         '''Given a list of image objects, create a json object string'''
         image_list = []
+        if spdx_version is not None:
+            logger.warning("The SPDX version parameter is not supported for JSON.")
+
         for image in image_obj_list:
             image_list.append({'image': image.to_dict()})
         image_dict = {'images': image_list}
         return json.dumps(image_dict)
 
-    def generate_layer(self, layer):
+    def generate_layer(self, layer, spdx_version: str):
         """Create a json object for one layer"""
+        if spdx_version is not None:
+            logger.warning("The SPDX version parameter is not supported for JSON.")
+
         return json.dumps(layer.to_dict())

tern/formats/spdx/__init__.py

@@ -1,4 +0,0 @@
-# -*- coding: utf-8 -*-
-#
-# Copyright (c) 2019 VMware, Inc. All Rights Reserved.
-# SPDX-License-Identifier: BSD-2-Clause

tern/formats/spdx/constants.py

@@ -0,0 +1,15 @@
+from spdx_tools.spdx.model import Version
+
+DOCUMENT_ID = 'SPDXRef-DOCUMENT'
+DOCUMENT_NAME = 'Tern report for {image_name}'
+SPDX_VERSION = 'SPDX-2.2'
+DATA_LICENSE = 'CC0-1.0'
+DOCUMENT_COMMENT = 'This document was generated by ' \
+    'the Tern Project: https://github.com/tern-tools/tern'
+DOCUMENT_NAMESPACE = 'https://spdx.org/spdxdocs/tern-' \
+    'report-{version}-{image}-{uuid}'
+LICENSE_LIST_VERSION = Version(3, 20)
+CREATOR_NAME = 'tern-{version}'
+DOCUMENT_NAME_SNAPSHOT = 'Tern SPDX SBoM'
+DOCUMENT_NAMESPACE_SNAPSHOT = 'https://spdx.org/spdxdocs/tern-report-' \
+    '{timestamp}-{uuid}'

tern/formats/spdx/file_helpers.py

@@ -0,0 +1,134 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2023 VMWare, Inc. All Rights Reserved.
+# SPDX-License-Identifier: BSD-2-Clause
+
+"""
+File level helpers for SPDX document generator
+"""
+import logging
+from datetime import datetime
+from typing import List
+
+from spdx_tools.spdx.model import File as SpdxFile, SpdxNone, SpdxNoAssertion, Checksum, ChecksumAlgorithm, FileType
+
+from tern.classes.file_data import FileData
+from tern.classes.image import Image
+from tern.classes.image_layer import ImageLayer
+from tern.classes.template import Template
+from tern.formats.spdx.layer_helpers import get_layer_checksum
+from tern.formats.spdx.general_helpers import get_package_license_declared, get_file_spdxref
+from tern.utils import constants
+
+logger = logging.getLogger(constants.logger_name)
+
+
+def get_spdx_file_list_from_layer(layer_obj: ImageLayer, template: Template, timestamp: datetime, spdx_version: str) -> List[SpdxFile]:
+    """Given a layer object and the SPDX template mapping, return a list
+    of SPDX Files for each file in the layer"""
+    spdx_files: List[SpdxFile] = []
+    file_refs = set()
+    for filedata in layer_obj.files:
+        # we do not know the layer's id, so we will use the timestamp instead
+        file_ref = get_file_spdxref(filedata, str(timestamp))
+        if file_ref not in file_refs:
+            spdx_files.append(get_spdx_file_from_filedata(filedata, template, str(timestamp), spdx_version))
+            file_refs.add(file_ref)
+    return spdx_files
+
+
+def get_spdx_file_list_from_image(image_obj: Image, template: Template, spdx_version: str) -> List[SpdxFile]:
+    """Given an image_obj object, and the SPDX template mapping, return a list
+    of SPDX Files for each file in each layer of the image."""
+    spdx_files: List[SpdxFile] = []
+
+    # use file refs to keep track of duplicate files that may be located
+    # in different places in the filesystem
+    file_refs = set()
+    for layer in image_obj.layers:
+        if layer.files_analyzed:
+            layer_checksum_value = get_layer_checksum(layer).value
+            for filedata in layer.files:
+                # we use the layer checksum as the layer id
+                file_ref = get_file_spdxref(filedata, layer_checksum_value)
+                if file_ref not in file_refs:
+                    spdx_files.append(get_spdx_file_from_filedata(filedata, template, layer_checksum_value, spdx_version))
+                    file_refs.add(file_ref)
+    return spdx_files
+
+
+def get_spdx_file_from_filedata(filedata: FileData, template: Template, layer_id: str, spdx_version: str) -> SpdxFile:
+    """Given a FileData object and its SPDX template mapping, return an
+    SPDX representation of the file. A layer_id is used to
+    distinguish copies of the same file occurring in different places in the
+    image"""
+    mapping = filedata.to_dict(template)
+
+    if filedata.licenses:
+        # Add the license expression to the list if it is a valid SPDX
+        # identifier; otherwise, add the LicenseRef
+        license_info_in_file = [get_package_license_declared(lic) for lic in set(filedata.licenses)]
+    else:
+        license_info_in_file = [SpdxNone()]
+
+    file_notice = get_file_notice(filedata)
+    file_comment = get_file_comment(filedata)
+    file_contributors = get_file_contributors(filedata)
+
+    file_types = None
+    if mapping['FileType']:
+        file_types = [FileType[mapping['FileType'].upper()]]
+
+    return SpdxFile(
+        spdx_id=get_file_spdxref(filedata, layer_id),
+        name=mapping['FileName'],
+        checksums=[get_file_checksum(filedata)],
+        license_concluded=SpdxNoAssertion() if spdx_version == "SPDX-2.2" else None,  # we don't provide this
+        copyright_text=SpdxNoAssertion() if spdx_version == "SPDX-2.2" else None,     # we don't know this
+        file_types=file_types,
+        license_info_in_file=license_info_in_file,
+        notice=file_notice if file_notice else None,
+        comment=file_comment if file_comment else None,
+        contributors=file_contributors if file_contributors else None,
+    )
+
+
+def get_file_checksum(filedata: FileData) -> Checksum:
+    """Given a FileData object, return the checksum required by SPDX.
+    Currently, the spec requires a SHA1 checksum"""
+    checksum = filedata.get_checksum('sha1')
+    if not checksum:
+        logger.error("No SHA1 checksum found in file. Resorting to empty file checksum.")
+        checksum = "da39a3ee5e6b4b0d3255bfef95601890afd80709"
+    return Checksum(ChecksumAlgorithm.SHA1, checksum)
+
+
+def get_file_notice(filedata: FileData) -> str:
+    """Return a formatted string with all copyrights found in a file. Return
+    an empty string if there are no copyrights"""
+    notice = ''
+    for cp in filedata.copyrights:
+        notice = notice + cp + '\n'
+    return notice
+
+
+def get_file_comment(filedata: FileData) -> str:
+    """Return a formatted comment string with all file level notices. Return
+    an empty string if no notices are present"""
+    comment = ''
+    for origin in filedata.origins.origins:
+        comment = comment + f'{origin.origin_str}:' + '\n'
+        for notice in origin.notices:
+            comment = comment + \
+                f'{notice.level}: {notice.message}' + '\n'
+    return comment
+
+
+def get_file_contributors(filedata: FileData) -> List[str]:
+    """The SPDX spec allows for an optional list of file contributors.
+    If there are any authors found in the file, return a list of authors.
+    If empty, return an empty list"""
+    contributors = []
+    for author in filedata.authors:
+        contributors.append(author)
+    return contributors