Add ProductDependency model to support relating packages in products #…

…138 (#147)

Signed-off-by: tdruez <[email protected]>

CHANGELOG.rst

@@ -54,6 +54,10 @@ Release notes
 - Use the declared_license_expression_spdx value in SPDX outputs.
   https://github.com/nexB/dejacode/issues/63
 
+- Add new ProductDependency model to support relating Packages in the context of a
+  Product.
+  https://github.com/nexB/dejacode/issues/138
+
 ### Version 5.1.0
 
 - Upgrade Python version to 3.12 and Django to 5.0.x

Makefile

@@ -50,10 +50,10 @@ doc8:
 	  --ignore-path docs/installation_and_sysadmin/ --quiet docs/
 
 valid:
-	@echo "-> Run Ruff linter"
-	@${ACTIVATE} ruff check --fix
 	@echo "-> Run Ruff format"
 	@${ACTIVATE} ruff format
+	@echo "-> Run Ruff linter"
+	@${ACTIVATE} ruff check --fix
 
 check:
 	@echo "-> Run Ruff linter validation (pycodestyle, bandit, isort, and more)"

component_catalog/license_expression_dje.py

@@ -14,6 +14,7 @@
 from django.forms import widgets
 from django.urls import reverse
 from django.utils.html import format_html
+from django.utils.safestring import mark_safe
 
 from boolean.boolean import PARSE_ERRORS
 from license_expression import ExpressionError
@@ -432,7 +433,10 @@ def render_expression_as_html(expression, dataspace):
     licensing = get_dataspace_licensing(dataspace)
 
     formatted_expression = get_formatted_expression(licensing, expression, show_policy)
-    return format_html(formatted_expression)
+    return format_html(
+        '<span class="license-expression">{}</span>',
+        mark_safe(formatted_expression),
+    )
 
 
 def get_expression_as_spdx(expression, dataspace):

component_catalog/models.py

@@ -48,6 +48,7 @@
 from component_catalog.license_expression_dje import get_expression_as_spdx
 from component_catalog.license_expression_dje import get_license_objects
 from component_catalog.license_expression_dje import parse_expression
+from component_catalog.license_expression_dje import render_expression_as_html
 from dejacode_toolkit import spdx
 from dejacode_toolkit.download import DataCollectionException
 from dejacode_toolkit.download import collect_package_data
@@ -228,6 +229,11 @@ def get_expression_as_spdx(self, expression):
     def concluded_license_expression_spdx(self):
         return self.get_expression_as_spdx(self.license_expression)
 
+    @property
+    def license_expression_html(self):
+        if self.license_expression:
+            return render_expression_as_html(self.license_expression, self.dataspace)
+
     def save(self, *args, **kwargs):
         """
         Call the handle_assigned_licenses method on save, except during copy.
@@ -1627,6 +1633,30 @@ def annotate_sortable_identifier(self):
             sortable_identifier=Concat(*PACKAGE_URL_FIELDS, "filename", output_field=CharField())
         )
 
+    def only_rendering_fields(self):
+        """Minimum requirements to render a Package element in the UI."""
+        return self.only(
+            "uuid",
+            *PACKAGE_URL_FIELDS,
+            "filename",
+            "license_expression",
+            "dataspace__name",
+            "dataspace__show_usage_policy_in_user_views",
+        )
+
+    def declared_dependencies_count(self, product):
+        """
+        Annotate the QuerySet with this each Package declared_dependencies count.
+        A ``product`` context need to be provided to get the proper counts as
+        dependencies are always scoped to a Product.
+        """
+        return self.annotate(
+            declared_dependencies_count=models.Count(
+                "declared_dependencies",
+                filter=models.Q(declared_dependencies__product=product),
+            )
+        )
+
 
 class Package(
     ExternalReferenceMixin,

component_catalog/templates/component_catalog/includes/hierarchy_instance_box.html

@@ -28,6 +28,12 @@
             <button type="button" data-bs-toggle="tooltip" title="Edit" class="btn btn-link p-0" aria-label="Edit object"><i class="far fa-edit fa-sm"></i></button>
           </span>
         {% endif %}
+        {% if relation.package_id and relation.package.declared_dependencies.all %}
+          <a class="btn badge text-bg-primary rounded-pill ms-1"
+             href="{{ product.get_absolute_url }}?dependencies-for_package__uuid={{ relation.package.uuid }}#dependencies" class="ms-1" data-bs-toggle="tooltip" title="Dependencies" aria-label="Dependencies">
+            {{ relation.package.declared_dependencies.all|length }}<i class="fa-solid fa-share-nodes ms-1"></i>
+          </a>
+        {% endif %}
       {% elif instance.is_active or is_product %}
         <a href="{{ instance.get_absolute_url }}#hierarchy">{{ instance }}</a>
         {% if relation.component_id and has_edit_productcomponent %}

component_catalog/tests/test_license_expression_dje.py

@@ -309,11 +309,16 @@ def test_get_dataspace_licensing(self):
 
     def test_render_expression_as_html(self):
         expression_as_html = render_expression_as_html(str(self.license1.key), self.dataspace)
-        expected = '<a href="/licenses/Starship/apache-2.0/" title="Apache 2.0">apache-2.0</a>'
+        expected = (
+            '<span class="license-expression">'
+            '<a href="/licenses/Starship/apache-2.0/" title="Apache 2.0">apache-2.0</a>'
+            "</span>"
+        )
         self.assertEqual(expected, expression_as_html)
 
         expression_as_html = render_expression_as_html("unknown", self.dataspace)
-        self.assertEqual("unknown", expression_as_html)
+        expected = '<span class="license-expression">unknown</span>'
+        self.assertEqual(expected, expression_as_html)
 
 
 def _print_sequence_diff(left, right):

component_catalog/tests/test_views.py

@@ -716,27 +716,27 @@ def test_component_catalog_list_view_filters_breadcrumbs(self):
 
         expected = f"""
         <div class="my-1">
-            <a href="{href1}" class="text-decoration-none">
+            <a href="{href1}" class="text-decoration-none me-1">
               <span class="badge text-bg-secondary rounded-pill">
                 Type: "not_a_valid_entry" <i class="fas fa-times-circle"></i>
                 </span>
             </a>
-            <a href="{href2}" class="text-decoration-none">
+            <a href="{href2}" class="text-decoration-none me-1">
               <span class="badge text-bg-secondary rounded-pill">
                 License: "license1" <i class="fas fa-times-circle"></i>
                 </span>
             </a>
-            <a href="{href3}" class="text-decoration-none">
+            <a href="{href3}" class="text-decoration-none me-1">
               <span class="badge text-bg-secondary rounded-pill">
                 License: "license2" <i class="fas fa-times-circle"></i>
                 </span>
             </a>
-            <a href="{href4}" class="text-decoration-none">
+            <a href="{href4}" class="text-decoration-none me-1">
               <span class="badge text-bg-secondary rounded-pill">
                 Search: "a" <i class="fas fa-times-circle"></i>
                 </span>
             </a>
-            <a href="{href5}" class="text-decoration-none">
+            <a href="{href5}" class="text-decoration-none me-1">
               <span class="badge text-bg-secondary rounded-pill">
                 Sort: "name" <i class="fas fa-times-circle"></i>
                 </span>

component_catalog/views.py

@@ -1682,9 +1682,9 @@ def send_scan_data_as_file_view(request, project_uuid, filename):
         raise Http404
 
     scancodeio = ScanCodeIO(dataspace)
-    scan_results_url = scancodeio.get_scan_results_url(project_uuid)
+    scan_results_url = scancodeio.get_scan_action_url(project_uuid, "results")
     scan_results = scancodeio.fetch_scan_data(scan_results_url)
-    scan_summary_url = scancodeio.get_scan_summary_url(project_uuid)
+    scan_summary_url = scancodeio.get_scan_action_url(project_uuid, "summary")
     scan_summary = scancodeio.fetch_scan_data(scan_summary_url)
 
     in_memory_zip = io.BytesIO()

dejacode/urls.py

@@ -48,6 +48,7 @@
 from policy.api import UsagePolicyViewSet
 from product_portfolio.api import CodebaseResourceViewSet
 from product_portfolio.api import ProductComponentViewSet
+from product_portfolio.api import ProductDependencyViewSet
 from product_portfolio.api import ProductPackageViewSet
 from product_portfolio.api import ProductViewSet
 from reporting.api import ReportViewSet
@@ -69,6 +70,7 @@
 api_router.register("packages", PackageViewSet)
 api_router.register("products", ProductViewSet)
 api_router.register("product_components", ProductComponentViewSet)
+api_router.register("product_dependencies", ProductDependencyViewSet)
 api_router.register("product_packages", ProductPackageViewSet)
 api_router.register("codebase_resources", CodebaseResourceViewSet)
 api_router.register("request_templates", RequestTemplateViewSet)

dejacode_toolkit/scancodeio.py

@@ -35,16 +35,9 @@ def __init__(self, *args, **kwargs):
     def get_scan_detail_url(self, project_uuid):
         return f"{self.project_api_url}{project_uuid}/"
 
-    def get_scan_results_url(self, project_uuid):
+    def get_scan_action_url(self, project_uuid, action_name):
         detail_url = self.get_scan_detail_url(project_uuid)
-        return f"{detail_url}results/"
-
-    def get_scan_summary_url(self, project_uuid):
-        detail_url = self.get_scan_detail_url(project_uuid)
-        return f"{detail_url}summary/"
-
-    def get_project_packages_url(self, project_uuid):
-        return f"{self.project_api_url}{project_uuid}/packages/"
+        return f"{detail_url}{action_name}/"
 
     def get_scan_results(self, download_url, dataspace):
         scan_info = self.fetch_scan_info(uri=download_url, dataspace=dataspace)
@@ -231,22 +224,30 @@ def update_from_scan(self, package, user):
 
         return updated_fields
 
-    def fetch_project_packages(self, project_uuid):
-        """Return the list of packages for the provided `project_uuid`."""
-        project_packages_url = self.get_project_packages_url(project_uuid)
-        packages = []
+    def fetch_results(self, api_url):
+        results = []
 
-        next_url = project_packages_url
+        next_url = api_url
         while next_url:
-            logger.debug(f"{self.label}: fetch packages from project_packages_url={next_url}")
+            logger.debug(f"{self.label}: fetch results from api_url={next_url}")
             response = self.request_get(url=next_url)
             if not response:
-                raise Exception("Error fetching project packages")
+                raise Exception("Error fetching results")
 
-            packages.extend(response["results"])
+            results.extend(response["results"])
             next_url = response["next"]
 
-        return packages
+        return results
+
+    def fetch_project_packages(self, project_uuid):
+        """Return the list of packages for the provided `project_uuid`."""
+        api_url = self.get_scan_action_url(project_uuid, "packages")
+        return self.fetch_results(api_url)
+
+    def fetch_project_dependencies(self, project_uuid):
+        """Return the list of dependencies for the provided `project_uuid`."""
+        api_url = self.get_scan_action_url(project_uuid, "dependencies")
+        return self.fetch_results(api_url)
 
     # (label, scan_field, model_field, input_type)
     SCAN_SUMMARY_FIELDS = [

dje/filters.py

@@ -57,16 +57,22 @@ def is_active(self):
     def get_query_no_sort(self):
         return remove_field_from_query_dict(self.data, "sort")
 
+    def get_filter_breadcrumb(self, field_name, data_field_name, value):
+        return {
+            "label": self.filters[field_name].label,
+            "value": value,
+            "remove_url": remove_field_from_query_dict(self.data, data_field_name, value),
+        }
+
     def get_filters_breadcrumbs(self):
-        return [
-            {
-                "label": self.filters[field_name].label,
-                "value": value,
-                "remove_url": remove_field_from_query_dict(self.data, field_name, value),
-            }
-            for field_name in self.form.changed_data
-            for value in self.data.getlist(field_name)
-        ]
+        breadcrumbs = []
+
+        for field_name in self.form.changed_data:
+            data_field_name = f"{self.form_prefix}-{field_name}" if self.form_prefix else field_name
+            for value in self.data.getlist(data_field_name):
+                breadcrumbs.append(self.get_filter_breadcrumb(field_name, data_field_name, value))
+
+        return breadcrumbs
 
 
 class DataspacedFilterSet(FilterSetUtilsMixin, django_filters.FilterSet):

dje/tasks.py

@@ -206,15 +206,22 @@ def pull_project_data_from_scancodeio(scancodeproject_uuid):
         return
 
     scancode_project.status = ScanCodeProject.Status.SUCCESS
-    msg = f"- Imported {len(created)} package{pluralize(created)}."
-    scancode_project.append_to_log(msg)
 
-    if existing:
-        msg = f"- {len(existing)} package(s) was/were already available in the Dataspace."
+    for object_type, values in created.items():
+        object_type_plural = f"{object_type}{pluralize(values)}"
+        object_type_plural = object_type_plural.replace("dependencys", "dependencies")
+        msg = f"- Imported {len(values)} {object_type_plural}."
         scancode_project.append_to_log(msg)
 
-    if errors:
-        scancode_project.append_to_log(f"- {len(errors)} errors occurred during import.")
+    for object_type, values in existing.items():
+        msg = (
+            f"- {len(values)} {object_type}{pluralize(values)} already available in the Dataspace."
+        )
+        scancode_project.append_to_log(msg)
+
+    for object_type, values in errors.items():
+        msg = f"- {len(values)} {object_type} error{pluralize(values)} " f"occurred during import."
+        scancode_project.append_to_log(msg)
 
     scancode_project.save()
     description = "\n".join(scancode_project.import_log)