Enhance the Package search logic #160 (#161)

Signed-off-by: tdruez <[email protected]>

component_catalog/filters.py

@@ -159,13 +159,22 @@ def filter(self, qs, value):
 class PackageFilterSet(DataspacedFilterSet):
     q = PackageSearchFilter(
         label=_("Search"),
-        match_order_fields=["filename"],
-        search_fields=[
+        match_order_fields=[
+            "type",
+            "namespace",
+            "name",
+            "version",
             "filename",
+            "download_url",
+            "sha1",
+            "md5",
+        ],
+        search_fields=[
             "type",
             "namespace",
             "name",
             "version",
+            "filename",
             "download_url",
             "sha1",
             "md5",

component_catalog/tests/__init__.py

@@ -5,3 +5,21 @@
 # See https://github.com/nexB/dejacode for support or download.
 # See https://aboutcode.org for more information about AboutCode FOSS projects.
 #
+
+from component_catalog.models import Component
+from component_catalog.models import Package
+
+
+def make_package(dataspace, package_url=None, **data):
+    package = Package(dataspace=dataspace, **data)
+    if package_url:
+        package.set_package_url(package_url)
+    package.save()
+    return package
+
+
+def make_component(dataspace, **data):
+    return Component.objects.create(
+        dataspace=dataspace,
+        **data,
+    )

component_catalog/tests/test_filters.py

@@ -17,7 +17,8 @@
 from component_catalog.models import Component
 from component_catalog.models import ComponentKeyword
 from component_catalog.models import ComponentType
-from component_catalog.models import Package
+from component_catalog.tests import make_component
+from component_catalog.tests import make_package
 from dje.models import Dataspace
 from dje.tests import create_superuser
 from dje.tests import create_user
@@ -28,7 +29,7 @@
 
 class ComponentFilterSetTest(TestCase):
     def setUp(self):
-        self.dataspace = Dataspace.objects.create(name="nexB")
+        self.dataspace = Dataspace.objects.create(name="Reference")
         self.other_dataspace = Dataspace.objects.create(name="Other")
         self.nexb_user = create_superuser("nexb_user", self.dataspace)
         self.basic_user = create_user("basic_user", self.dataspace)
@@ -123,10 +124,8 @@ def test_component_filterset_primary_language_filter(self):
         filterset = ComponentFilterSet(dataspace=self.dataspace)
         self.assertEqual([], list(filterset.filters["primary_language"].field.choices))
 
-        c1 = Component.objects.create(
-            name="c1", dataspace=self.dataspace, primary_language="Python"
-        )
-        c2 = Component.objects.create(name="c2", dataspace=self.dataspace, primary_language="Java")
+        c1 = make_component(self.dataspace, name="c1", primary_language="Python")
+        c2 = make_component(self.dataspace, name="c2", primary_language="Java")
 
         filterset = ComponentFilterSet(dataspace=self.dataspace)
         self.assertEqual([c1, c2], list(filterset.qs))
@@ -146,13 +145,13 @@ def test_component_filterset_related_only_values_filter(self):
             ["licenses", "primary_language", "usage_policy"], ComponentFilterSet.related_only
         )
 
-        c1 = Component.objects.create(
+        c1 = make_component(
+            self.dataspace,
             name="c1",
-            dataspace=self.dataspace,
             license_expression=self.license1.key,
             primary_language="Python",
         )
-        c2 = Component.objects.create(name="c2", dataspace=self.dataspace, primary_language="Java")
+        c2 = make_component(self.dataspace, name="c2", primary_language="Java")
 
         filterset = ComponentFilterSet(dataspace=self.dataspace)
         self.assertEqual([c1, c2], list(filterset.qs))
@@ -210,9 +209,9 @@ class ComponentFilterSearchTestCase(TestCase):
     fixtures = [join(testfiles_location, "search", "component_dataset.json")]
 
     def test_component_filterset_search_filter(self):
-        nexb_dataspace = Dataspace.objects.get(name="nexB")
+        dataspace = Dataspace.objects.get(name="Reference")
         data = {"q": ""}
-        component_filterset = ComponentFilterSet(dataspace=nexb_dataspace, data=data)
+        component_filterset = ComponentFilterSet(dataspace=dataspace, data=data)
         expected = [
             "jblogbackup 1.0",
             "jblogbackup 1.1",
@@ -227,7 +226,7 @@ def test_component_filterset_search_filter(self):
         self.assertEqual(expected, [str(component) for component in component_filterset.qs])
 
         data = {"q": "logback"}
-        component_filterset = ComponentFilterSet(dataspace=nexb_dataspace, data=data)
+        component_filterset = ComponentFilterSet(dataspace=dataspace, data=data)
         expected = [
             "logback 0.9.9",
             "logback 1.0.0",
@@ -241,57 +240,57 @@ def test_component_filterset_search_filter(self):
         self.assertEqual(expected, [str(component) for component in component_filterset.qs])
 
     def test_component_filterset_sort_keeps_default_ordering_from_model(self):
-        nexb_dataspace = Dataspace.objects.get(name="nexB")
+        dataspace = Dataspace.objects.get(name="Reference")
         data = {}
-        component_filterset = ComponentFilterSet(dataspace=nexb_dataspace, data=data)
+        component_filterset = ComponentFilterSet(dataspace=dataspace, data=data)
         self.assertEqual((), component_filterset.qs.query.order_by)
 
         data = {"sort": ""}
-        component_filterset = ComponentFilterSet(dataspace=nexb_dataspace, data=data)
+        component_filterset = ComponentFilterSet(dataspace=dataspace, data=data)
         self.assertEqual((), component_filterset.qs.query.order_by)
 
         data = {"sort": "invalid"}
-        component_filterset = ComponentFilterSet(dataspace=nexb_dataspace, data=data)
+        component_filterset = ComponentFilterSet(dataspace=dataspace, data=data)
         self.assertEqual((), component_filterset.qs.query.order_by)
 
         data = {"sort": "name"}
-        component_filterset = ComponentFilterSet(dataspace=nexb_dataspace, data=data)
+        component_filterset = ComponentFilterSet(dataspace=dataspace, data=data)
         self.assertEqual(("name", "version"), component_filterset.qs.query.order_by)
 
         data = {"sort": "version"}
-        component_filterset = ComponentFilterSet(dataspace=nexb_dataspace, data=data)
+        component_filterset = ComponentFilterSet(dataspace=dataspace, data=data)
         self.assertEqual(("version", "name"), component_filterset.qs.query.order_by)
 
         data = {"sort": "primary_language"}
-        component_filterset = ComponentFilterSet(dataspace=nexb_dataspace, data=data)
+        component_filterset = ComponentFilterSet(dataspace=dataspace, data=data)
         self.assertEqual(
             ("primary_language", "name", "version"), component_filterset.qs.query.order_by
         )
 
 
 class PackageFilterSearchTestCase(TestCase):
-    def setUp(self):
-        self.nexb_dataspace = Dataspace.objects.create(name="nexB")
+    def sorted_results(self, qs):
+        return sorted([str(package) for package in qs])
 
-        def create_package(**kwargs):
-            return Package.objects.create(**kwargs, dataspace=self.nexb_dataspace)
+    def setUp(self):
+        self.dataspace = Dataspace.objects.create(name="Reference")
 
         filename = {
             "filename": "setup.exe",
         }
-        create_package(**filename)
+        make_package(self.dataspace, **filename)
 
         simple_purl = {
             "type": "deb",
             "name": "curl",
         }
-        create_package(**simple_purl)
+        make_package(self.dataspace, **simple_purl)
 
         simple_purl2 = {
             **simple_purl,
             "type": "git",
         }
-        create_package(**simple_purl2)
+        make_package(self.dataspace, **simple_purl2)
 
         complete_purl = {
             "type": "deb",
@@ -301,60 +300,66 @@ def create_package(**kwargs):
             "qualifiers": "arch=i386",
             "subpath": "googleapis/api/annotations",
         }
-        create_package(**complete_purl)
+        make_package(self.dataspace, **complete_purl)
 
     def test_package_filterset_search_filter(self):
-        def sorted_results(qs):
-            return sorted([str(package) for package in qs])
-
         data = {"q": ""}
-        filterset = PackageFilterSet(dataspace=self.nexb_dataspace, data=data)
+        filterset = PackageFilterSet(dataspace=self.dataspace, data=data)
         expected = [
             "pkg:deb/debian/[email protected]?arch=i386#googleapis/api/annotations",
             "pkg:git/curl",
             "pkg:deb/curl",
             "setup.exe",
         ]
-        self.assertEqual(sorted(expected), sorted_results(filterset.qs))
+        self.assertEqual(sorted(expected), self.sorted_results(filterset.qs))
 
         data = {"q": "deb/curl"}
-        filterset = PackageFilterSet(dataspace=self.nexb_dataspace, data=data)
+        filterset = PackageFilterSet(dataspace=self.dataspace, data=data)
         expected = [
             "pkg:deb/curl",
             "pkg:deb/debian/[email protected]?arch=i386#googleapis/api/annotations",
         ]
-        self.assertEqual(sorted(expected), sorted_results(filterset.qs))
+        self.assertEqual(sorted(expected), self.sorted_results(filterset.qs))
         data = {"q": "pkg:deb/curl"}
-        filterset = PackageFilterSet(dataspace=self.nexb_dataspace, data=data)
-        self.assertEqual(sorted(expected), sorted_results(filterset.qs))
+        filterset = PackageFilterSet(dataspace=self.dataspace, data=data)
+        self.assertEqual(sorted(expected), self.sorted_results(filterset.qs))
 
         data = {"q": "deb/debian/[email protected]"}
-        filterset = PackageFilterSet(dataspace=self.nexb_dataspace, data=data)
+        filterset = PackageFilterSet(dataspace=self.dataspace, data=data)
         expected = [
             "pkg:deb/debian/[email protected]?arch=i386#googleapis/api/annotations",
         ]
-        self.assertEqual(sorted(expected), sorted_results(filterset.qs))
+        self.assertEqual(sorted(expected), self.sorted_results(filterset.qs))
         data = {"q": "pkg:deb/debian/[email protected]"}
-        filterset = PackageFilterSet(dataspace=self.nexb_dataspace, data=data)
-        self.assertEqual(sorted(expected), sorted_results(filterset.qs))
+        filterset = PackageFilterSet(dataspace=self.dataspace, data=data)
+        self.assertEqual(sorted(expected), self.sorted_results(filterset.qs))
 
         data = {"q": "git/curl"}
-        filterset = PackageFilterSet(dataspace=self.nexb_dataspace, data=data)
+        filterset = PackageFilterSet(dataspace=self.dataspace, data=data)
         expected = [
             "pkg:git/curl",
         ]
-        self.assertEqual(sorted(expected), sorted_results(filterset.qs))
+        self.assertEqual(sorted(expected), self.sorted_results(filterset.qs))
         data = {"q": "pkg:git/curl"}
-        filterset = PackageFilterSet(dataspace=self.nexb_dataspace, data=data)
-        self.assertEqual(sorted(expected), sorted_results(filterset.qs))
+        filterset = PackageFilterSet(dataspace=self.dataspace, data=data)
+        self.assertEqual(sorted(expected), self.sorted_results(filterset.qs))
 
         data = {"q": "setup.exe"}
-        filterset = PackageFilterSet(dataspace=self.nexb_dataspace, data=data)
+        filterset = PackageFilterSet(dataspace=self.dataspace, data=data)
         expected = [
             "setup.exe",
         ]
-        self.assertEqual(sorted(expected), sorted_results(filterset.qs))
+        self.assertEqual(sorted(expected), self.sorted_results(filterset.qs))
 
         data = {"q": "pkg:setup.exe"}
-        filterset = PackageFilterSet(dataspace=self.nexb_dataspace, data=data)
-        self.assertEqual([], sorted_results(filterset.qs))
+        filterset = PackageFilterSet(dataspace=self.dataspace, data=data)
+        self.assertEqual([], self.sorted_results(filterset.qs))
+
+    def test_package_filterset_search_match_order_on_purl_fields(self):
+        make_package(self.dataspace, package_url="pkg:pypi/[email protected]")
+        make_package(self.dataspace, package_url="pkg:pypi/[email protected]", filename="Django-4.0.zip")
+
+        data = {"q": "django"}
+        filterset = PackageFilterSet(dataspace=self.dataspace, data=data)
+        expected = ["pkg:pypi/[email protected]", "pkg:pypi/[email protected]"]
+        self.assertEqual(sorted(expected), self.sorted_results(filterset.qs))

component_catalog/tests/testfiles/search/component_dataset.json

@@ -3,7 +3,7 @@
   "model": "dje.dataspace",
   "fields": {
     "uuid": "f26d21dc-d781-4727-84c3-1f7edc14b919",
-    "name": "nexB",
+    "name": "Reference",
     "homepage_url": "http://www.nexb.com/",
     "contact_info": "http://www.nexb.com/",
     "notes": "Creators/maintainers of the system software",
@@ -25,7 +25,7 @@
   "model": "component_catalog.component",
   "fields": {
     "dataspace": [
-      "nexB"
+      "Reference"
     ],
     "uuid": "53fa7229-617b-4dfa-9eee-8c09e9380046",
     "created_date": "2014-07-01T00:00:00Z",
@@ -65,7 +65,7 @@
   "model": "component_catalog.component",
   "fields": {
     "dataspace": [
-      "nexB"
+      "Reference"
     ],
     "uuid": "03df48a9-60df-43b6-9c04-4a4c9015f309",
     "created_date": "2014-07-01T00:00:00Z",
@@ -105,7 +105,7 @@
   "model": "component_catalog.component",
   "fields": {
     "dataspace": [
-      "nexB"
+      "Reference"
     ],
     "uuid": "fed79fab-5d00-4994-a2d1-ad35164b736a",
     "created_date": "2014-07-01T00:00:00Z",
@@ -145,7 +145,7 @@
   "model": "component_catalog.component",
   "fields": {
     "dataspace": [
-      "nexB"
+      "Reference"
     ],
     "uuid": "4a037d63-f479-41f7-985e-436b8039160a",
     "created_date": "2014-07-01T00:00:00Z",
@@ -185,7 +185,7 @@
   "model": "component_catalog.component",
   "fields": {
     "dataspace": [
-      "nexB"
+      "Reference"
     ],
     "uuid": "663fe571-6042-48cd-95ec-6cf57316a591",
     "created_date": "2015-08-04T15:08:45.738Z",
@@ -225,7 +225,7 @@
   "model": "component_catalog.component",
   "fields": {
     "dataspace": [
-      "nexB"
+      "Reference"
     ],
     "uuid": "d47238fd-09aa-45e9-aafa-9d8c3fe10b93",
     "created_date": "2014-07-01T00:00:00Z",
@@ -265,7 +265,7 @@
   "model": "component_catalog.component",
   "fields": {
     "dataspace": [
-      "nexB"
+      "Reference"
     ],
     "uuid": "31368d10-b832-4c77-87d3-2d300989d30e",
     "created_date": "2014-07-01T00:00:00Z",
@@ -305,7 +305,7 @@
   "model": "component_catalog.component",
   "fields": {
     "dataspace": [
-      "nexB"
+      "Reference"
     ],
     "uuid": "0a8bfdd1-7c48-4ac7-99f1-9e6c46e1bb81",
     "created_date": "2014-07-01T00:00:00Z",
@@ -345,7 +345,7 @@
   "model": "component_catalog.component",
   "fields": {
     "dataspace": [
-      "nexB"
+      "Reference"
     ],
     "uuid": "9b1c732d-081b-44ae-a366-ac0182281422",
     "created_date": "2017-06-13T10:05:03.727Z",

dje/filters.py

@@ -210,7 +210,7 @@ def filter(self, qs, value):
 
 class MatchOrderedSearchFilter(SearchRankFilter):
     """
-    Start with a case-insensitive containment search on the `name` field,
+    Start with a case-insensitive containment search on the `match_order_fields` fields,
     ordering based on the match type using annotations.
 
     If that simple search Return nothing, fallback to the SearchRankFilter