Refactor the whole results system #138

Signed-off-by: tdruez <[email protected]>

dje/tasks.py

@@ -206,15 +206,21 @@ def pull_project_data_from_scancodeio(scancodeproject_uuid):
         return
 
     scancode_project.status = ScanCodeProject.Status.SUCCESS
-    msg = f"- Imported {len(created)} package{pluralize(created)}."
-    scancode_project.append_to_log(msg)
 
-    if existing:
-        msg = f"- {len(existing)} package(s) was/were already available in the Dataspace."
+    for object_type, values in created.items():
+        msg = f"- Imported {len(values)} {object_type}{pluralize(values)}."
         scancode_project.append_to_log(msg)
 
-    if errors:
-        scancode_project.append_to_log(f"- {len(errors)} errors occurred during import.")
+    for object_type, values in existing.items():
+        msg = (
+            f"- {len(values)} {object_type}{pluralize(values)} were already "
+            f"available in the Dataspace."
+        )
+        scancode_project.append_to_log(msg)
+
+    for object_type, values in errors.items():
+        msg = f"- {len(values)} {object_type} error{pluralize(values)} " f"occurred during import."
+        scancode_project.append_to_log(msg)
 
     scancode_project.save()
     description = "\n".join(scancode_project.import_log)

product_portfolio/importers.py

@@ -442,8 +442,9 @@ def validate_headers(self):
             self.validate_toolkit_options(scan_options)
 
         elif tool_name == "scanpipe":
-            runs = header.get("runs", [])
+            pass
             # TODO: Reconsider the value of this, as if there's packages data we should
+            # runs = header.get("runs", [])
             # accept the input anyway
             # self.validate_pipeline_runs(runs)
 
@@ -476,7 +477,8 @@ def validate_toolkit_options(scan_options):
     #         "scan_single_package",
     #     )
     #
-    #     has_a_valid_pipeline = [True for run in runs if run.get("pipeline_name") in valid_pipelines]
+    #     has_a_valid_pipeline = [True for run in runs if run.get("pipeline_name") in
+    #     valid_pipelines]
     #
     #     if not has_a_valid_pipeline:
     #         raise ValidationError(
@@ -639,9 +641,9 @@ class ImportPackageFromScanCodeIO:
 
     def __init__(self, user, project_uuid, product, update_existing=False, scan_all_packages=False):
         self.licensing = Licensing()
-        self.created = []
-        self.existing = []
-        self.errors = []
+        self.created = defaultdict(list)
+        self.existing = defaultdict(list)
+        self.errors = defaultdict(list)
         # Use to assign dependencies to the correct Package instance
         self.package_uid_mapping = {}
 
@@ -664,7 +666,7 @@ def save(self):
         if self.scan_all_packages:
             transaction.on_commit(lambda: self.product.scan_all_packages_task(self.user))
 
-        return self.created, self.existing, self.errors
+        return dict(self.created), dict(self.existing), dict(self.errors)
 
     def import_packages(self):
         for package_data in self.packages:
@@ -684,7 +686,7 @@ def import_package(self, package_data):
         # Check if the Package already exists in the local Dataspace
         try:
             package = Package.objects.scope(self.user.dataspace).get(**unique_together_lookups)
-            self.existing.append(package)
+            self.existing["package"].append(str(package))
         except (ObjectDoesNotExist, MultipleObjectsReturned):
             package = None
 
@@ -697,9 +699,9 @@ def import_package(self, package_data):
                 reference_object = qs.first()
                 try:
                     package = copy_object(reference_object, user_dataspace, self.user, update=False)
-                    self.created.append(package)
+                    self.created["package"].append(str(package))
                 except IntegrityError as error:
-                    self.errors.append(error)
+                    self.errors["package"].append(str(error))
 
         if license_expression := package_data.get("declared_license_expression"):
             license_expression = str(self.licensing.dedup(license_expression))
@@ -712,9 +714,9 @@ def import_package(self, package_data):
             try:
                 package = Package.create_from_data(self.user, package_data, validate=True)
             except ValidationError as errors:
-                self.errors.append(errors)
+                self.errors["package"].append(str(errors))
                 return
-            self.created.append(package)
+            self.created["package"].append(str(package))
 
         ProductPackage.objects.get_or_create(
             product=self.product,
@@ -735,6 +737,7 @@ def import_dependency(self, dependency_data):
 
         dependency_qs = ProductDependency.objects.scope(self.user.dataspace)
         if dependency_qs.filter(product=self.product, dependency_uid=dependency_uid).exists():
+            self.existing["dependency"].append(str(dependency_uid))
             return
 
         dependency_data["product"] = self.product
@@ -746,7 +749,13 @@ def import_dependency(self, dependency_data):
             )
 
         try:
-            ProductDependency.create_from_data(self.user, dependency_data, validate=True)
+            dependency = ProductDependency.create_from_data(
+                user=self.user,
+                data=dependency_data,
+                validate=True,
+            )
         except ValidationError as errors:
-            # self.errors.append(errors)
+            self.errors["dependency"].append(str(errors))
             return
+
+        self.created["dependency"].append(str(dependency))

product_portfolio/models.py

@@ -1299,9 +1299,9 @@ def import_data_from_scancodeio(self):
         created, existing, errors = importer.save()
 
         self.results = {
-            "created": [str(package) for package in created],
-            "existing": [str(package) for package in existing],
-            "errors": [str(error) for error in errors],
+            "created": created,
+            "existing": existing,
+            "errors": errors,
         }
         self.save()
 

product_portfolio/templates/product_portfolio/scancodeio_project_status.html

@@ -1,30 +1,36 @@
 {% load humanize %}
 
-{% if scancode_project.results.errors %}
-  <h5 class="text-danger">{{ scancode_project.results.errors|length }} errors:</h5>
-  <ul class="mb-1">
-    {% for error in scancode_project.results.errors %}
-      <li>{{ error }}</li>
-    {% endfor %}
-  </ul>
+{% if results.errors %}
+  {% for object_type, values in results.errors.items %}
+    <h5 class="text-danger">{{ values|length }} {{ object_type }} errors:</h5>
+    <ul class="mb-1">
+      {% for entry in values %}
+        <li>{{ entry }}</li>
+      {% endfor %}
+    </ul>
+  {% endfor %}
   <hr>
 {% endif %}
-{% if scancode_project.results.created %}
-  <h5>{{ scancode_project.results.created|length }} packages created:</h5>
-  <ul class="mb-1">
-    {% for package in scancode_project.results.created %}
-      <li>{{ package }}</li>
-    {% endfor %}
-  </ul>
+{% if results.created %}
+  {% for object_type, values in results.created.items %}
+    <h5>{{ values|length }} {{ object_type }} created:</h5>
+    <ul class="mb-1">
+      {% for entry in values %}
+        <li>{{ entry }}</li>
+      {% endfor %}
+    </ul>
+  {% endfor %}
   <hr>
 {% endif %}
-{% if scancode_project.results.existing %}
-  <h5>{{ scancode_project.results.existing|length }} packages updated/existing:</h5>
-  <ul class="mb-1">
-    {% for package in scancode_project.results.existing %}
-      <li>{{ package }}</li>
-    {% endfor %}
-  </ul>
+{% if results.existing %}
+  {% for object_type, values in results.created.items %}
+    <h5>{{ values|length }} {{ object_type }} updated/existing:</h5>
+    <ul class="mb-1">
+      {% for entry in values %}
+        <li>{{ entry }}</li>
+      {% endfor %}
+    </ul>
+  {% endfor %}
   <hr>
 {% endif %}
 

product_portfolio/views.py

@@ -2170,8 +2170,16 @@ def scancodeio_project_status_view(request, scancodeproject_uuid):
     scan_detail_url = scancodeio.get_scan_detail_url(scancode_project.project_uuid)
     scan_data = scancodeio.fetch_scan_data(scan_detail_url)
 
+    results = scancode_project.results
+
+    # Backward compatibility
+    is_old_results_format = any(isinstance(entry, list) for entry in results.values())
+    if is_old_results_format:
+        results = {key: {"package": value} for key, value in results.items() if value}
+
     context = {
         "scancode_project": scancode_project,
+        "results": results,
         "scan_data": scan_data,
     }