From e2507da9d25be668f0ec57ee92a483322b74b9fa Mon Sep 17 00:00:00 2001 From: tdruez <489057+tdruez@users.noreply.github.com> Date: Wed, 7 Aug 2024 16:40:22 +0400 Subject: [PATCH] Enhance the Package search logic #160 (#161) Signed-off-by: tdruez --- component_catalog/filters.py | 13 ++- component_catalog/tests/__init__.py | 18 ++++ component_catalog/tests/test_filters.py | 101 +++++++++--------- .../testfiles/search/component_dataset.json | 20 ++-- dje/filters.py | 2 +- 5 files changed, 93 insertions(+), 61 deletions(-) diff --git a/component_catalog/filters.py b/component_catalog/filters.py index 353df10b..bdec1f33 100644 --- a/component_catalog/filters.py +++ b/component_catalog/filters.py @@ -159,13 +159,22 @@ def filter(self, qs, value): class PackageFilterSet(DataspacedFilterSet): q = PackageSearchFilter( label=_("Search"), - match_order_fields=["filename"], - search_fields=[ + match_order_fields=[ + "type", + "namespace", + "name", + "version", "filename", + "download_url", + "sha1", + "md5", + ], + search_fields=[ "type", "namespace", "name", "version", + "filename", "download_url", "sha1", "md5", diff --git a/component_catalog/tests/__init__.py b/component_catalog/tests/__init__.py index 7aa35de2..ea856412 100644 --- a/component_catalog/tests/__init__.py +++ b/component_catalog/tests/__init__.py @@ -5,3 +5,21 @@ # See https://github.com/nexB/dejacode for support or download. # See https://aboutcode.org for more information about AboutCode FOSS projects. # + +from component_catalog.models import Component +from component_catalog.models import Package + + +def make_package(dataspace, package_url=None, **data): + package = Package(dataspace=dataspace, **data) + if package_url: + package.set_package_url(package_url) + package.save() + return package + + +def make_component(dataspace, **data): + return Component.objects.create( + dataspace=dataspace, + **data, + ) diff --git a/component_catalog/tests/test_filters.py b/component_catalog/tests/test_filters.py index 4439ed87..5406718c 100644 --- a/component_catalog/tests/test_filters.py +++ b/component_catalog/tests/test_filters.py @@ -17,7 +17,8 @@ from component_catalog.models import Component from component_catalog.models import ComponentKeyword from component_catalog.models import ComponentType -from component_catalog.models import Package +from component_catalog.tests import make_component +from component_catalog.tests import make_package from dje.models import Dataspace from dje.tests import create_superuser from dje.tests import create_user @@ -28,7 +29,7 @@ class ComponentFilterSetTest(TestCase): def setUp(self): - self.dataspace = Dataspace.objects.create(name="nexB") + self.dataspace = Dataspace.objects.create(name="Reference") self.other_dataspace = Dataspace.objects.create(name="Other") self.nexb_user = create_superuser("nexb_user", self.dataspace) self.basic_user = create_user("basic_user", self.dataspace) @@ -123,10 +124,8 @@ def test_component_filterset_primary_language_filter(self): filterset = ComponentFilterSet(dataspace=self.dataspace) self.assertEqual([], list(filterset.filters["primary_language"].field.choices)) - c1 = Component.objects.create( - name="c1", dataspace=self.dataspace, primary_language="Python" - ) - c2 = Component.objects.create(name="c2", dataspace=self.dataspace, primary_language="Java") + c1 = make_component(self.dataspace, name="c1", primary_language="Python") + c2 = make_component(self.dataspace, name="c2", primary_language="Java") filterset = ComponentFilterSet(dataspace=self.dataspace) self.assertEqual([c1, c2], list(filterset.qs)) @@ -146,13 +145,13 @@ def test_component_filterset_related_only_values_filter(self): ["licenses", "primary_language", "usage_policy"], ComponentFilterSet.related_only ) - c1 = Component.objects.create( + c1 = make_component( + self.dataspace, name="c1", - dataspace=self.dataspace, license_expression=self.license1.key, primary_language="Python", ) - c2 = Component.objects.create(name="c2", dataspace=self.dataspace, primary_language="Java") + c2 = make_component(self.dataspace, name="c2", primary_language="Java") filterset = ComponentFilterSet(dataspace=self.dataspace) self.assertEqual([c1, c2], list(filterset.qs)) @@ -210,9 +209,9 @@ class ComponentFilterSearchTestCase(TestCase): fixtures = [join(testfiles_location, "search", "component_dataset.json")] def test_component_filterset_search_filter(self): - nexb_dataspace = Dataspace.objects.get(name="nexB") + dataspace = Dataspace.objects.get(name="Reference") data = {"q": ""} - component_filterset = ComponentFilterSet(dataspace=nexb_dataspace, data=data) + component_filterset = ComponentFilterSet(dataspace=dataspace, data=data) expected = [ "jblogbackup 1.0", "jblogbackup 1.1", @@ -227,7 +226,7 @@ def test_component_filterset_search_filter(self): self.assertEqual(expected, [str(component) for component in component_filterset.qs]) data = {"q": "logback"} - component_filterset = ComponentFilterSet(dataspace=nexb_dataspace, data=data) + component_filterset = ComponentFilterSet(dataspace=dataspace, data=data) expected = [ "logback 0.9.9", "logback 1.0.0", @@ -241,57 +240,57 @@ def test_component_filterset_search_filter(self): self.assertEqual(expected, [str(component) for component in component_filterset.qs]) def test_component_filterset_sort_keeps_default_ordering_from_model(self): - nexb_dataspace = Dataspace.objects.get(name="nexB") + dataspace = Dataspace.objects.get(name="Reference") data = {} - component_filterset = ComponentFilterSet(dataspace=nexb_dataspace, data=data) + component_filterset = ComponentFilterSet(dataspace=dataspace, data=data) self.assertEqual((), component_filterset.qs.query.order_by) data = {"sort": ""} - component_filterset = ComponentFilterSet(dataspace=nexb_dataspace, data=data) + component_filterset = ComponentFilterSet(dataspace=dataspace, data=data) self.assertEqual((), component_filterset.qs.query.order_by) data = {"sort": "invalid"} - component_filterset = ComponentFilterSet(dataspace=nexb_dataspace, data=data) + component_filterset = ComponentFilterSet(dataspace=dataspace, data=data) self.assertEqual((), component_filterset.qs.query.order_by) data = {"sort": "name"} - component_filterset = ComponentFilterSet(dataspace=nexb_dataspace, data=data) + component_filterset = ComponentFilterSet(dataspace=dataspace, data=data) self.assertEqual(("name", "version"), component_filterset.qs.query.order_by) data = {"sort": "version"} - component_filterset = ComponentFilterSet(dataspace=nexb_dataspace, data=data) + component_filterset = ComponentFilterSet(dataspace=dataspace, data=data) self.assertEqual(("version", "name"), component_filterset.qs.query.order_by) data = {"sort": "primary_language"} - component_filterset = ComponentFilterSet(dataspace=nexb_dataspace, data=data) + component_filterset = ComponentFilterSet(dataspace=dataspace, data=data) self.assertEqual( ("primary_language", "name", "version"), component_filterset.qs.query.order_by ) class PackageFilterSearchTestCase(TestCase): - def setUp(self): - self.nexb_dataspace = Dataspace.objects.create(name="nexB") + def sorted_results(self, qs): + return sorted([str(package) for package in qs]) - def create_package(**kwargs): - return Package.objects.create(**kwargs, dataspace=self.nexb_dataspace) + def setUp(self): + self.dataspace = Dataspace.objects.create(name="Reference") filename = { "filename": "setup.exe", } - create_package(**filename) + make_package(self.dataspace, **filename) simple_purl = { "type": "deb", "name": "curl", } - create_package(**simple_purl) + make_package(self.dataspace, **simple_purl) simple_purl2 = { **simple_purl, "type": "git", } - create_package(**simple_purl2) + make_package(self.dataspace, **simple_purl2) complete_purl = { "type": "deb", @@ -301,60 +300,66 @@ def create_package(**kwargs): "qualifiers": "arch=i386", "subpath": "googleapis/api/annotations", } - create_package(**complete_purl) + make_package(self.dataspace, **complete_purl) def test_package_filterset_search_filter(self): - def sorted_results(qs): - return sorted([str(package) for package in qs]) - data = {"q": ""} - filterset = PackageFilterSet(dataspace=self.nexb_dataspace, data=data) + filterset = PackageFilterSet(dataspace=self.dataspace, data=data) expected = [ "pkg:deb/debian/curl@7.50.3-1?arch=i386#googleapis/api/annotations", "pkg:git/curl", "pkg:deb/curl", "setup.exe", ] - self.assertEqual(sorted(expected), sorted_results(filterset.qs)) + self.assertEqual(sorted(expected), self.sorted_results(filterset.qs)) data = {"q": "deb/curl"} - filterset = PackageFilterSet(dataspace=self.nexb_dataspace, data=data) + filterset = PackageFilterSet(dataspace=self.dataspace, data=data) expected = [ "pkg:deb/curl", "pkg:deb/debian/curl@7.50.3-1?arch=i386#googleapis/api/annotations", ] - self.assertEqual(sorted(expected), sorted_results(filterset.qs)) + self.assertEqual(sorted(expected), self.sorted_results(filterset.qs)) data = {"q": "pkg:deb/curl"} - filterset = PackageFilterSet(dataspace=self.nexb_dataspace, data=data) - self.assertEqual(sorted(expected), sorted_results(filterset.qs)) + filterset = PackageFilterSet(dataspace=self.dataspace, data=data) + self.assertEqual(sorted(expected), self.sorted_results(filterset.qs)) data = {"q": "deb/debian/curl@7.50.3-1"} - filterset = PackageFilterSet(dataspace=self.nexb_dataspace, data=data) + filterset = PackageFilterSet(dataspace=self.dataspace, data=data) expected = [ "pkg:deb/debian/curl@7.50.3-1?arch=i386#googleapis/api/annotations", ] - self.assertEqual(sorted(expected), sorted_results(filterset.qs)) + self.assertEqual(sorted(expected), self.sorted_results(filterset.qs)) data = {"q": "pkg:deb/debian/curl@7.50.3-1"} - filterset = PackageFilterSet(dataspace=self.nexb_dataspace, data=data) - self.assertEqual(sorted(expected), sorted_results(filterset.qs)) + filterset = PackageFilterSet(dataspace=self.dataspace, data=data) + self.assertEqual(sorted(expected), self.sorted_results(filterset.qs)) data = {"q": "git/curl"} - filterset = PackageFilterSet(dataspace=self.nexb_dataspace, data=data) + filterset = PackageFilterSet(dataspace=self.dataspace, data=data) expected = [ "pkg:git/curl", ] - self.assertEqual(sorted(expected), sorted_results(filterset.qs)) + self.assertEqual(sorted(expected), self.sorted_results(filterset.qs)) data = {"q": "pkg:git/curl"} - filterset = PackageFilterSet(dataspace=self.nexb_dataspace, data=data) - self.assertEqual(sorted(expected), sorted_results(filterset.qs)) + filterset = PackageFilterSet(dataspace=self.dataspace, data=data) + self.assertEqual(sorted(expected), self.sorted_results(filterset.qs)) data = {"q": "setup.exe"} - filterset = PackageFilterSet(dataspace=self.nexb_dataspace, data=data) + filterset = PackageFilterSet(dataspace=self.dataspace, data=data) expected = [ "setup.exe", ] - self.assertEqual(sorted(expected), sorted_results(filterset.qs)) + self.assertEqual(sorted(expected), self.sorted_results(filterset.qs)) data = {"q": "pkg:setup.exe"} - filterset = PackageFilterSet(dataspace=self.nexb_dataspace, data=data) - self.assertEqual([], sorted_results(filterset.qs)) + filterset = PackageFilterSet(dataspace=self.dataspace, data=data) + self.assertEqual([], self.sorted_results(filterset.qs)) + + def test_package_filterset_search_match_order_on_purl_fields(self): + make_package(self.dataspace, package_url="pkg:pypi/django@5.0") + make_package(self.dataspace, package_url="pkg:pypi/django@4.0", filename="Django-4.0.zip") + + data = {"q": "django"} + filterset = PackageFilterSet(dataspace=self.dataspace, data=data) + expected = ["pkg:pypi/django@4.0", "pkg:pypi/django@5.0"] + self.assertEqual(sorted(expected), self.sorted_results(filterset.qs)) diff --git a/component_catalog/tests/testfiles/search/component_dataset.json b/component_catalog/tests/testfiles/search/component_dataset.json index 9cd869c8..e8a72078 100644 --- a/component_catalog/tests/testfiles/search/component_dataset.json +++ b/component_catalog/tests/testfiles/search/component_dataset.json @@ -3,7 +3,7 @@ "model": "dje.dataspace", "fields": { "uuid": "f26d21dc-d781-4727-84c3-1f7edc14b919", - "name": "nexB", + "name": "Reference", "homepage_url": "http://www.nexb.com/", "contact_info": "http://www.nexb.com/", "notes": "Creators/maintainers of the system software", @@ -25,7 +25,7 @@ "model": "component_catalog.component", "fields": { "dataspace": [ - "nexB" + "Reference" ], "uuid": "53fa7229-617b-4dfa-9eee-8c09e9380046", "created_date": "2014-07-01T00:00:00Z", @@ -65,7 +65,7 @@ "model": "component_catalog.component", "fields": { "dataspace": [ - "nexB" + "Reference" ], "uuid": "03df48a9-60df-43b6-9c04-4a4c9015f309", "created_date": "2014-07-01T00:00:00Z", @@ -105,7 +105,7 @@ "model": "component_catalog.component", "fields": { "dataspace": [ - "nexB" + "Reference" ], "uuid": "fed79fab-5d00-4994-a2d1-ad35164b736a", "created_date": "2014-07-01T00:00:00Z", @@ -145,7 +145,7 @@ "model": "component_catalog.component", "fields": { "dataspace": [ - "nexB" + "Reference" ], "uuid": "4a037d63-f479-41f7-985e-436b8039160a", "created_date": "2014-07-01T00:00:00Z", @@ -185,7 +185,7 @@ "model": "component_catalog.component", "fields": { "dataspace": [ - "nexB" + "Reference" ], "uuid": "663fe571-6042-48cd-95ec-6cf57316a591", "created_date": "2015-08-04T15:08:45.738Z", @@ -225,7 +225,7 @@ "model": "component_catalog.component", "fields": { "dataspace": [ - "nexB" + "Reference" ], "uuid": "d47238fd-09aa-45e9-aafa-9d8c3fe10b93", "created_date": "2014-07-01T00:00:00Z", @@ -265,7 +265,7 @@ "model": "component_catalog.component", "fields": { "dataspace": [ - "nexB" + "Reference" ], "uuid": "31368d10-b832-4c77-87d3-2d300989d30e", "created_date": "2014-07-01T00:00:00Z", @@ -305,7 +305,7 @@ "model": "component_catalog.component", "fields": { "dataspace": [ - "nexB" + "Reference" ], "uuid": "0a8bfdd1-7c48-4ac7-99f1-9e6c46e1bb81", "created_date": "2014-07-01T00:00:00Z", @@ -345,7 +345,7 @@ "model": "component_catalog.component", "fields": { "dataspace": [ - "nexB" + "Reference" ], "uuid": "9b1c732d-081b-44ae-a366-ac0182281422", "created_date": "2017-06-13T10:05:03.727Z", diff --git a/dje/filters.py b/dje/filters.py index ae97ca2c..43839925 100644 --- a/dje/filters.py +++ b/dje/filters.py @@ -210,7 +210,7 @@ def filter(self, qs, value): class MatchOrderedSearchFilter(SearchRankFilter): """ - Start with a case-insensitive containment search on the `name` field, + Start with a case-insensitive containment search on the `match_order_fields` fields, ordering based on the match type using annotations. If that simple search Return nothing, fallback to the SearchRankFilter