diff --git a/dje/tasks.py b/dje/tasks.py index 1783463f..a49ef1e3 100644 --- a/dje/tasks.py +++ b/dje/tasks.py @@ -206,15 +206,21 @@ def pull_project_data_from_scancodeio(scancodeproject_uuid): return scancode_project.status = ScanCodeProject.Status.SUCCESS - msg = f"- Imported {len(created)} package{pluralize(created)}." - scancode_project.append_to_log(msg) - if existing: - msg = f"- {len(existing)} package(s) was/were already available in the Dataspace." + for object_type, values in created.items(): + msg = f"- Imported {len(values)} {object_type}{pluralize(values)}." scancode_project.append_to_log(msg) - if errors: - scancode_project.append_to_log(f"- {len(errors)} errors occurred during import.") + for object_type, values in existing.items(): + msg = ( + f"- {len(values)} {object_type}{pluralize(values)} were already " + f"available in the Dataspace." + ) + scancode_project.append_to_log(msg) + + for object_type, values in errors.items(): + msg = f"- {len(values)} {object_type} error{pluralize(values)} " f"occurred during import." + scancode_project.append_to_log(msg) scancode_project.save() description = "\n".join(scancode_project.import_log) diff --git a/product_portfolio/importers.py b/product_portfolio/importers.py index aadaa2e6..98e08550 100644 --- a/product_portfolio/importers.py +++ b/product_portfolio/importers.py @@ -442,8 +442,9 @@ def validate_headers(self): self.validate_toolkit_options(scan_options) elif tool_name == "scanpipe": - runs = header.get("runs", []) + pass # TODO: Reconsider the value of this, as if there's packages data we should + # runs = header.get("runs", []) # accept the input anyway # self.validate_pipeline_runs(runs) @@ -476,7 +477,8 @@ def validate_toolkit_options(scan_options): # "scan_single_package", # ) # - # has_a_valid_pipeline = [True for run in runs if run.get("pipeline_name") in valid_pipelines] + # has_a_valid_pipeline = [True for run in runs if run.get("pipeline_name") in + # valid_pipelines] # # if not has_a_valid_pipeline: # raise ValidationError( @@ -639,9 +641,9 @@ class ImportPackageFromScanCodeIO: def __init__(self, user, project_uuid, product, update_existing=False, scan_all_packages=False): self.licensing = Licensing() - self.created = [] - self.existing = [] - self.errors = [] + self.created = defaultdict(list) + self.existing = defaultdict(list) + self.errors = defaultdict(list) # Use to assign dependencies to the correct Package instance self.package_uid_mapping = {} @@ -664,7 +666,7 @@ def save(self): if self.scan_all_packages: transaction.on_commit(lambda: self.product.scan_all_packages_task(self.user)) - return self.created, self.existing, self.errors + return dict(self.created), dict(self.existing), dict(self.errors) def import_packages(self): for package_data in self.packages: @@ -684,7 +686,7 @@ def import_package(self, package_data): # Check if the Package already exists in the local Dataspace try: package = Package.objects.scope(self.user.dataspace).get(**unique_together_lookups) - self.existing.append(package) + self.existing["package"].append(str(package)) except (ObjectDoesNotExist, MultipleObjectsReturned): package = None @@ -697,9 +699,9 @@ def import_package(self, package_data): reference_object = qs.first() try: package = copy_object(reference_object, user_dataspace, self.user, update=False) - self.created.append(package) + self.created["package"].append(str(package)) except IntegrityError as error: - self.errors.append(error) + self.errors["package"].append(str(error)) if license_expression := package_data.get("declared_license_expression"): license_expression = str(self.licensing.dedup(license_expression)) @@ -712,9 +714,9 @@ def import_package(self, package_data): try: package = Package.create_from_data(self.user, package_data, validate=True) except ValidationError as errors: - self.errors.append(errors) + self.errors["package"].append(str(errors)) return - self.created.append(package) + self.created["package"].append(str(package)) ProductPackage.objects.get_or_create( product=self.product, @@ -735,6 +737,7 @@ def import_dependency(self, dependency_data): dependency_qs = ProductDependency.objects.scope(self.user.dataspace) if dependency_qs.filter(product=self.product, dependency_uid=dependency_uid).exists(): + self.existing["dependency"].append(str(dependency_uid)) return dependency_data["product"] = self.product @@ -746,7 +749,13 @@ def import_dependency(self, dependency_data): ) try: - ProductDependency.create_from_data(self.user, dependency_data, validate=True) + dependency = ProductDependency.create_from_data( + user=self.user, + data=dependency_data, + validate=True, + ) except ValidationError as errors: - # self.errors.append(errors) + self.errors["dependency"].append(str(errors)) return + + self.created["dependency"].append(str(dependency)) diff --git a/product_portfolio/models.py b/product_portfolio/models.py index f605c6d1..075ca1b0 100644 --- a/product_portfolio/models.py +++ b/product_portfolio/models.py @@ -1299,9 +1299,9 @@ def import_data_from_scancodeio(self): created, existing, errors = importer.save() self.results = { - "created": [str(package) for package in created], - "existing": [str(package) for package in existing], - "errors": [str(error) for error in errors], + "created": created, + "existing": existing, + "errors": errors, } self.save() diff --git a/product_portfolio/templates/product_portfolio/scancodeio_project_status.html b/product_portfolio/templates/product_portfolio/scancodeio_project_status.html index e3e03309..b40ee07b 100644 --- a/product_portfolio/templates/product_portfolio/scancodeio_project_status.html +++ b/product_portfolio/templates/product_portfolio/scancodeio_project_status.html @@ -1,30 +1,36 @@ {% load humanize %} -{% if scancode_project.results.errors %} -