Skip to content

Commit

Permalink
Add prototype for Dependency model and view #138
Browse files Browse the repository at this point in the history
Signed-off-by: tdruez <[email protected]>
  • Loading branch information
tdruez committed Jul 5, 2024
1 parent 59725d2 commit 5b7b5cf
Show file tree
Hide file tree
Showing 9 changed files with 604 additions and 2 deletions.
22 changes: 22 additions & 0 deletions dejacode_toolkit/scancodeio.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,10 @@ def get_scan_summary_url(self, project_uuid):
def get_project_packages_url(self, project_uuid):
return f"{self.project_api_url}{project_uuid}/packages/"

# TODO: Remove duplication with get_project_packages_url
def get_project_dependencies_url(self, project_uuid):
return f"{self.project_api_url}{project_uuid}/dependencies/"

def get_scan_results(self, download_url, dataspace):
scan_info = self.fetch_scan_info(uri=download_url, dataspace=dataspace)

Expand Down Expand Up @@ -248,6 +252,24 @@ def fetch_project_packages(self, project_uuid):

return packages

# TODO: Remove duplication with fetch_project_packages
def fetch_project_dependencies(self, project_uuid):
"""Return the list of dependencies for the provided `project_uuid`."""
api_url = self.get_project_dependencies_url(project_uuid)
dependencies = []

next_url = api_url
while next_url:
logger.debug(f"{self.label}: fetch dependencies from project_packages_url={next_url}")
response = self.request_get(url=next_url)
if not response:
raise Exception("Error fetching project dependencies")

dependencies.extend(response["results"])
next_url = response["next"]

return dependencies

# (label, scan_field, model_field, input_type)
SCAN_SUMMARY_FIELDS = [
(
Expand Down
33 changes: 33 additions & 0 deletions product_portfolio/filters.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
from product_portfolio.models import CodebaseResource
from product_portfolio.models import Product
from product_portfolio.models import ProductComponent
from product_portfolio.models import ProductDependency
from product_portfolio.models import ProductPackage
from product_portfolio.models import ProductStatus

Expand Down Expand Up @@ -314,3 +315,35 @@ class Meta:
"related_deployed_from",
"related_deployed_to",
]


class DependencyFilterSet(DataspacedFilterSet):
q = SearchFilter(
label=_("Search"),
search_fields=[
"for_package__filename",
"for_package__type",
"for_package__namespace",
"for_package__name",
"for_package__version",
"resolved_to_package__filename",
"resolved_to_package__type",
"resolved_to_package__namespace",
"resolved_to_package__name",
"resolved_to_package__version",
],
)
is_deployment_path = BooleanChoiceFilter(
widget=DropDownWidget(anchor="#codebase"),
)

class Meta:
model = ProductDependency
fields = [
"scope",
"datasource_id",
"is_runtime",
"is_optional",
"is_resolved",
"is_direct",
]
4 changes: 2 additions & 2 deletions product_portfolio/forms.py
Original file line number Diff line number Diff line change
Expand Up @@ -599,7 +599,7 @@ class BaseProductImportFormView(forms.Form):
initial=False,
help_text=_(
"If checked, the discovered packages from the manifest that are already "
"existing in your Dataspace will be updated with ScanCode data"
"existing in your Dataspace will be updated with ScanCode data. "
"Note that only the empty fields will be updated. "
"By default (un-checked), existing packages will be assign to the product "
"without any modification."
Expand Down Expand Up @@ -894,7 +894,7 @@ class PullProjectDataForm(forms.Form):
initial=False,
help_text=_(
"If checked, the discovered packages from the Project that are already "
"existing in your Dataspace will be updated with ScanCode.io data."
"existing in your Dataspace will be updated with ScanCode.io data. "
"Note that only the empty fields will be updated. "
"By default (un-checked), existing packages will be assign to the product "
"without any modification."
Expand Down
37 changes: 37 additions & 0 deletions product_portfolio/importers.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@
from product_portfolio.models import CodebaseResourceUsage
from product_portfolio.models import Product
from product_portfolio.models import ProductComponent
from product_portfolio.models import ProductDependency
from product_portfolio.models import ProductItemPurpose
from product_portfolio.models import ProductPackage
from product_portfolio.models import ProductRelationStatus
Expand Down Expand Up @@ -639,6 +640,8 @@ def __init__(self, user, project_uuid, product, update_existing=False, scan_all_
self.created = []
self.existing = []
self.errors = []
# Use to assign dependencies to the correct Package instance
self.package_uid_mapping = {}

self.user = user
self.project_uuid = project_uuid
Expand All @@ -650,9 +653,11 @@ def __init__(self, user, project_uuid, product, update_existing=False, scan_all_
self.packages = scancodeio.fetch_project_packages(self.project_uuid)
if not self.packages:
raise Exception("Packages could not be fetched from ScanCode.io")
self.dependencies = scancodeio.fetch_project_dependencies(self.project_uuid)

def save(self):
self.import_packages()
self.import_dependencies()

if self.scan_all_packages:
transaction.on_commit(lambda: self.product.scan_all_packages_task(self.user))
Expand All @@ -663,7 +668,12 @@ def import_packages(self):
for package_data in self.packages:
self.import_package(package_data)

def import_dependencies(self):
for dependency_data in self.dependencies:
self.import_dependency(dependency_data)

def import_package(self, package_data):
package_uid = package_data.get("package_uid")
unique_together_lookups = {
field: value
for field in self.unique_together_fields
Expand Down Expand Up @@ -715,3 +725,30 @@ def import_package(self, package_data):
"created_by": self.user,
},
)

self.package_uid_mapping[package_uid] = package

def import_dependency(self, dependency_data):
dependency = None
# TODO: Check if the Dependency already exists in the local Dataspace
# try:
# dependency = ProductDependency.objects.scope(self.user.dataspace)
# .get(**unique_together_lookups)
# self.existing.append(package)
# except (ObjectDoesNotExist, MultipleObjectsReturned):
# dependency = None

dependency_data["product"] = self.product
for_package_uid = dependency_data["for_package_uid"]
dependency_data["for_package"] = self.package_uid_mapping.get(for_package_uid)
resolved_to_package_uid = dependency_data["resolved_to_package_uid"]
dependency_data["resolved_to_package"] = self.package_uid_mapping.get(
resolved_to_package_uid
)

if not dependency:
try:
ProductDependency.create_from_data(self.user, dependency_data, validate=True)
except ValidationError as errors:
print(errors)
return
219 changes: 219 additions & 0 deletions product_portfolio/migrations/0006_productdependency_and_more.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,219 @@
# Generated by Django 5.0.6 on 2024-07-05 12:49

import django.db.models.deletion
import dje.fields
import uuid
from django.conf import settings
from django.db import migrations, models


class Migration(migrations.Migration):

dependencies = [
("component_catalog", "0005_remove_component_concluded_license_and_more"),
("dje", "0003_alter_dejacodeuser_homepage_layout"),
("product_portfolio", "0005_alter_product_license_expression_and_more"),
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
]

operations = [
migrations.CreateModel(
name="ProductDependency",
fields=[
(
"id",
models.AutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
(
"uuid",
models.UUIDField(
default=uuid.uuid4, editable=False, verbose_name="UUID"
),
),
(
"created_date",
models.DateTimeField(
auto_now_add=True,
db_index=True,
help_text="The date and time the object was created.",
),
),
(
"last_modified_date",
models.DateTimeField(
auto_now=True,
db_index=True,
help_text="The date and time the object was last modified.",
),
),
(
"dependency_uid",
models.CharField(
help_text="The unique identifier of this dependency.",
max_length=1024,
),
),
(
"extracted_requirement",
models.CharField(
blank=True,
help_text="The version requirements of this dependency.",
max_length=256,
),
),
(
"scope",
models.CharField(
blank=True,
help_text="The scope of this dependency, how it is used in a project.",
max_length=64,
),
),
(
"datasource_id",
models.CharField(
blank=True,
help_text="The identifier for the datafile handler used to obtain this dependency.",
max_length=64,
),
),
(
"is_runtime",
models.BooleanField(
default=False,
help_text="True if this dependency is a runtime dependency.",
),
),
(
"is_optional",
models.BooleanField(
default=False,
help_text="True if this dependency is an optional dependency",
),
),
(
"is_resolved",
models.BooleanField(
default=False,
help_text="True if this dependency version requirement has been pinned and this dependency points to an exact version.",
),
),
(
"is_direct",
models.BooleanField(
default=False,
help_text="True if this is a direct, first-level dependency relationship for a package.",
),
),
(
"created_by",
models.ForeignKey(
editable=False,
help_text="The application user who created the object.",
null=True,
on_delete=django.db.models.deletion.PROTECT,
related_name="created_%(class)ss",
serialize=False,
to=settings.AUTH_USER_MODEL,
),
),
(
"dataspace",
models.ForeignKey(
editable=False,
help_text="A Dataspace is an independent, exclusive set of DejaCode data, which can be either nexB master reference data or installation-specific data.",
on_delete=django.db.models.deletion.PROTECT,
to="dje.dataspace",
),
),
(
"for_package",
models.ForeignKey(
blank=True,
editable=False,
help_text="The package that declares this dependency.",
null=True,
on_delete=django.db.models.deletion.CASCADE,
related_name="declared_dependencies",
to="component_catalog.package",
),
),
(
"last_modified_by",
dje.fields.LastModifiedByField(
editable=False,
help_text="The application user who last modified the object.",
null=True,
on_delete=django.db.models.deletion.PROTECT,
related_name="modified_%(class)ss",
serialize=False,
to=settings.AUTH_USER_MODEL,
),
),
(
"product",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
related_name="dependencies",
to="product_portfolio.product",
),
),
(
"resolved_to_package",
models.ForeignKey(
blank=True,
editable=False,
help_text="The resolved package for this dependency. If empty, it indicates the dependency is unresolved.",
null=True,
on_delete=django.db.models.deletion.SET_NULL,
related_name="resolved_from_dependencies",
to="component_catalog.package",
),
),
],
options={
"verbose_name": "product dependency",
"verbose_name_plural": "product dependencies",
"ordering": ["dependency_uid"],
"indexes": [
models.Index(fields=["scope"], name="product_por_scope_eb3e33_idx"),
models.Index(
fields=["is_runtime"], name="product_por_is_runt_3476a9_idx"
),
models.Index(
fields=["is_optional"], name="product_por_is_opti_9b7ae0_idx"
),
models.Index(
fields=["is_resolved"], name="product_por_is_reso_368f34_idx"
),
models.Index(
fields=["is_direct"], name="product_por_is_dire_3abb9d_idx"
),
],
},
),
migrations.AddConstraint(
model_name="productdependency",
constraint=models.UniqueConstraint(
condition=models.Q(("dependency_uid", ""), _negated=True),
fields=("product", "dependency_uid"),
name="product_portfolio_productdependency_unique_dependency_uid_within_product",
),
),
migrations.AddConstraint(
model_name="productdependency",
constraint=models.UniqueConstraint(
fields=("dataspace", "uuid"),
name="product_portfolio_productdependency_unique_uuid_within_dataspace",
),
),
migrations.AlterUniqueTogether(
name="productdependency",
unique_together={("dataspace", "uuid"), ("product", "dependency_uid")},
),
]
Loading

0 comments on commit 5b7b5cf

Please sign in to comment.