From 0db9401b6e932e400883ace2414c0417ec2402c2 Mon Sep 17 00:00:00 2001 From: tdruez <489057+tdruez@users.noreply.github.com> Date: Fri, 21 Jun 2024 15:10:48 +0400 Subject: [PATCH 1/8] Add ignored_vulnerabilities field on the Project configuration #1271 (#1281) * Add ignored_vulnerabilities field on the Project configuration #1271 Signed-off-by: tdruez * Minor change to the indentation for consistency #1271 Signed-off-by: tdruez --------- Signed-off-by: tdruez --- CHANGELOG.rst | 3 + docs/project-configuration.rst | 40 +- docs/scancode-config.yml | 10 +- scanpipe/forms.py | 18 + scanpipe/models.py | 19 + scanpipe/pipelines/find_vulnerabilities.py | 12 +- scanpipe/pipes/vulnerablecode.py | 21 +- .../templates/scanpipe/project_settings.html | 12 + .../tests/data/settings/scancode-config.yml | 8 +- .../django-5.0_package_data.json | 696 ++++++++++++++++++ scanpipe/tests/pipes/test_vulnerablecode.py | 83 +++ scanpipe/tests/test_forms.py | 2 + scanpipe/tests/test_models.py | 17 + 13 files changed, 925 insertions(+), 16 deletions(-) create mode 100644 scanpipe/tests/data/vulnerablecode/django-5.0_package_data.json create mode 100644 scanpipe/tests/pipes/test_vulnerablecode.py diff --git a/CHANGELOG.rst b/CHANGELOG.rst index c2275d011..7ae0dc7b0 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -11,6 +11,9 @@ v34.6.3 (unreleased) - Fix an issue in the d2d JavaScript mapper. https://github.com/nexB/scancode.io/pull/1274 +- Add support for a ``ignored_vulnerabilities`` field on the Project configuration. + https://github.com/nexB/scancode.io/issues/1271 + v34.6.2 (2024-06-18) -------------------- diff --git a/docs/project-configuration.rst b/docs/project-configuration.rst index cae8d9bab..046394bd7 100644 --- a/docs/project-configuration.rst +++ b/docs/project-configuration.rst @@ -52,16 +52,20 @@ Content of a ``scancode-config.yml`` file: product_name: My Product Name product_version: '1.0' ignored_patterns: - - '*.tmp' - - 'tests/*' + - '*.tmp' + - 'tests/*' ignored_dependency_scopes: - package_type: npm scope: devDependencies - package_type: pypi scope: tests + ignored_vulnerabilities: + - VCID-q4q6-yfng-aaag + - CVE-2024-27351 + - GHSA-vm8q-m57g-pff3 -See the :ref:`project_configuration_settings` section for the details about each -setting. +See the following :ref:`project_configuration_settings` section for the details about +each setting. .. tip:: You can generate the project configuration file from the @@ -125,10 +129,10 @@ packages, define the following in your ``scancode-config.yml`` configuration fil .. code-block:: yaml ignored_dependency_scopes: - - package_type: npm - scope: devDependencies - - package_type: pypi - scope: tests + - package_type: npm + scope: devDependencies + - package_type: pypi + scope: tests If you prefer to use the :ref:`user_interface_project_settings` form, list each ignored scope using the `package_type:scope` syntax, **one per line**, such as: @@ -141,3 +145,23 @@ ignored scope using the `package_type:scope` syntax, **one per line**, such as: .. warning:: Be precise when listing scope names to avoid unintended exclusions. Ensure the scope names are correct and reflect your project requirements. + +ignored_vulnerabilities +^^^^^^^^^^^^^^^^^^^^^^^ + +Provide one or more vulnerability id to be ignored, **one per line**. + +You can provide ``VCID`` from VulnerableCode or any aliases such as ``CVE`` or +``GHSA``. + +For example:: + +.. code-block:: yaml + + ignored_vulnerabilities: + - VCID-q4q6-yfng-aaag + - CVE-2024-27351 + - GHSA-vm8q-m57g-pff3 + - OSV-2020-871 + - BIT-django-2024-24680 + - PYSEC-2024-28 diff --git a/docs/scancode-config.yml b/docs/scancode-config.yml index b431e932c..35cb58e91 100644 --- a/docs/scancode-config.yml +++ b/docs/scancode-config.yml @@ -9,8 +9,8 @@ # - Use ``*`` to match multiple characters. # - Use ``?`` to match a single character. #ignored_patterns: -# - '*.tmp' -# - 'tests/*' +# - '*.tmp' +# - 'tests/*' # Specify certain dependency scopes to be ignored for a given package type. #ignored_dependency_scopes: @@ -18,3 +18,9 @@ # scope: devDependencies # - package_type: pypi # scope: tests + +# Specify certain vulnerabilities to be ignored using VCID, CVE, or any aliases. +#ignored_vulnerabilities: +# - VCID-q4q6-yfng-aaag +# - CVE-2024-27351 +# - GHSA-vm8q-m57g-pff3 diff --git a/scanpipe/forms.py b/scanpipe/forms.py index 781341b4f..88ccbb826 100644 --- a/scanpipe/forms.py +++ b/scanpipe/forms.py @@ -365,10 +365,16 @@ def prepare_value(self, value): """ +ignored_vulnerabilities_help = """ +Specify certain vulnerabilities to be ignored using VCID, CVE, or any aliases. +""" + + class ProjectSettingsForm(forms.ModelForm): settings_fields = [ "ignored_patterns", "ignored_dependency_scopes", + "ignored_vulnerabilities", "attribution_template", "product_name", "product_version", @@ -399,6 +405,18 @@ class ProjectSettingsForm(forms.ModelForm): key_name="package_type", value_name="scope", ) + ignored_vulnerabilities = ListTextarea( + label="Ignored vulnerabilities", + required=False, + help_text=convert_markdown_to_html(ignored_vulnerabilities_help.strip()), + widget=forms.Textarea( + attrs={ + "class": "textarea is-dynamic", + "rows": 2, + "placeholder": "VCID-q4q6-yfng-aaag\nCVE-2024-27351", + }, + ), + ) attribution_template = forms.CharField( label="Attribution template", required=False, diff --git a/scanpipe/models.py b/scanpipe/models.py index be0ea9602..02c9c929f 100644 --- a/scanpipe/models.py +++ b/scanpipe/models.py @@ -863,6 +863,25 @@ def ignored_dependency_scopes_index(self): """ return self.get_ignored_dependency_scopes_index() + def get_ignored_vulnerabilities_set(self): + """ + Return a set of ``ignored_vulnerabilities`` setting values defined in this + Project env. + """ + ignored_vulnerabilities = self.get_env(field_name="ignored_vulnerabilities") + if ignored_vulnerabilities: + return set(entry for entry in ignored_vulnerabilities) + + return [] + + @cached_property + def ignored_vulnerabilities_set(self): + """ + Return the computed value of get_ignored_vulnerabilities_set. + The value is only generated once and cached for further calls. + """ + return self.get_ignored_vulnerabilities_set() + def clear_tmp_directory(self): """ Delete the whole content of the tmp/ directory. diff --git a/scanpipe/pipelines/find_vulnerabilities.py b/scanpipe/pipelines/find_vulnerabilities.py index d34485b89..52d959656 100644 --- a/scanpipe/pipelines/find_vulnerabilities.py +++ b/scanpipe/pipelines/find_vulnerabilities.py @@ -53,9 +53,17 @@ def check_vulnerablecode_service_availability(self): def lookup_packages_vulnerabilities(self): """Check for vulnerabilities for each of the project's discovered package.""" packages = self.project.discoveredpackages.all() - vulnerablecode.fetch_vulnerabilities(packages, logger=self.log) + vulnerablecode.fetch_vulnerabilities( + packages=packages, + ignore_set=self.project.ignored_vulnerabilities_set, + logger=self.log, + ) def lookup_dependencies_vulnerabilities(self): """Check for vulnerabilities for each of the project's discovered dependency.""" dependencies = self.project.discovereddependencies.filter(is_resolved=True) - vulnerablecode.fetch_vulnerabilities(dependencies, logger=self.log) + vulnerablecode.fetch_vulnerabilities( + packages=dependencies, + ignore_set=self.project.ignored_vulnerabilities_set, + logger=self.log, + ) diff --git a/scanpipe/pipes/vulnerablecode.py b/scanpipe/pipes/vulnerablecode.py index 0200d6d37..fe0b341e3 100644 --- a/scanpipe/pipes/vulnerablecode.py +++ b/scanpipe/pipes/vulnerablecode.py @@ -202,7 +202,19 @@ def bulk_search_by_cpes( return request_post(url, data, timeout) -def fetch_vulnerabilities(packages, chunk_size=1000, logger=logger.info): +def filter_vulnerabilities(vulnerabilities, ignore_set): + """Filter out vulnerabilities based on a list of ignored IDs and aliases.""" + return [ + vulnerability + for vulnerability in vulnerabilities + if vulnerability.get("vulnerability_id") not in ignore_set + and not any(alias in ignore_set for alias in vulnerability.get("aliases", [])) + ] + + +def fetch_vulnerabilities( + packages, chunk_size=1000, logger=logger.info, ignore_set=None +): """ Fetch and store vulnerabilities for each provided ``packages``. The PURLs are used for the lookups in batch of ``chunk_size`` per request. @@ -217,7 +229,12 @@ def fetch_vulnerabilities(packages, chunk_size=1000, logger=logger.info): unsaved_objects = [] for package in packages: if package_data := vulnerabilities_by_purl.get(package.package_url): - if affected_by := package_data.get("affected_by_vulnerabilities", []): + affected_by = package_data.get("affected_by_vulnerabilities", []) + + if ignore_set and affected_by: + affected_by = filter_vulnerabilities(affected_by, ignore_set) + + if affected_by: package.affected_by_vulnerabilities = affected_by unsaved_objects.append(package) diff --git a/scanpipe/templates/scanpipe/project_settings.html b/scanpipe/templates/scanpipe/project_settings.html index 4ba42eaee..df6fa8635 100644 --- a/scanpipe/templates/scanpipe/project_settings.html +++ b/scanpipe/templates/scanpipe/project_settings.html @@ -76,6 +76,18 @@ {{ form.ignored_dependency_scopes.help_text|safe|linebreaksbr }} + +
+ +
+ {{ form.ignored_vulnerabilities }} +
+
+ {{ form.ignored_vulnerabilities.help_text|safe|linebreaksbr }} +
+
diff --git a/scanpipe/tests/data/settings/scancode-config.yml b/scanpipe/tests/data/settings/scancode-config.yml index 48115a469..dacc5f706 100644 --- a/scanpipe/tests/data/settings/scancode-config.yml +++ b/scanpipe/tests/data/settings/scancode-config.yml @@ -1,10 +1,14 @@ product_name: My Product Name product_version: '1.0' ignored_patterns: - - '*.tmp' - - 'tests/*' + - '*.tmp' + - 'tests/*' ignored_dependency_scopes: - package_type: npm scope: devDependencies - package_type: pypi scope: tests +ignored_vulnerabilities: + - VCID-q4q6-yfng-aaag + - CVE-2024-27351 + - GHSA-vm8q-m57g-pff3 diff --git a/scanpipe/tests/data/vulnerablecode/django-5.0_package_data.json b/scanpipe/tests/data/vulnerablecode/django-5.0_package_data.json new file mode 100644 index 000000000..f2ccc2190 --- /dev/null +++ b/scanpipe/tests/data/vulnerablecode/django-5.0_package_data.json @@ -0,0 +1,696 @@ +{ + "url": "http://vulnerablecode/api/packages/807568", + "purl": "pkg:pypi/django@5.0", + "type": "pypi", + "namespace": "", + "name": "django", + "version": "5.0", + "qualifiers": {}, + "subpath": "", + "next_non_vulnerable_version": "5.0.3", + "latest_non_vulnerable_version": "5.0.3", + "affected_by_vulnerabilities": [ + { + "url": "http://vulnerablecode/api/vulnerabilities/516832", + "vulnerability_id": "VCID-3gge-bre2-aaac", + "summary": "", + "references": [ + { + "reference_url": "https://access.redhat.com/hydra/rest/securitydata/cve/CVE-2024-24680.json", + "reference_id": "", + "scores": [ + { + "value": "7.5", + "scoring_system": "cvssv3", + "scoring_elements": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H" + } + ], + "url": "https://access.redhat.com/hydra/rest/securitydata/cve/CVE-2024-24680.json" + }, + { + "reference_url": "https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2024-24680", + "reference_id": "", + "scores": [], + "url": "https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2024-24680" + }, + { + "reference_url": "https://docs.djangoproject.com/en/5.0/releases/security", + "reference_id": "", + "scores": [ + { + "value": "5.9", + "scoring_system": "cvssv3.1", + "scoring_elements": "CVSS:3.1/AV:N/AC:H/PR:N/UI:N/S:U/C:N/I:N/A:H" + }, + { + "value": "LOW", + "scoring_system": "generic_textual", + "scoring_elements": "" + } + ], + "url": "https://docs.djangoproject.com/en/5.0/releases/security" + }, + { + "reference_url": "https://docs.djangoproject.com/en/5.0/releases/security/", + "reference_id": "", + "scores": [], + "url": "https://docs.djangoproject.com/en/5.0/releases/security/" + }, + { + "reference_url": "https://ftp.suse.com/pub/projects/security/yaml/suse-cvss-scores.yaml", + "reference_id": "", + "scores": [ + { + "value": "2.1", + "scoring_system": "cvssv2", + "scoring_elements": "AV:N/AC:H/Au:S/C:N/I:P/A:N" + }, + { + "value": "4.4", + "scoring_system": "cvssv3", + "scoring_elements": "CVSS:3.0/AV:L/AC:L/PR:N/UI:R/S:U/C:L/I:N/A:L" + }, + { + "value": "4.7", + "scoring_system": "cvssv3.1", + "scoring_elements": "CVSS:3.1/AV:L/AC:H/PR:L/UI:N/S:U/C:N/I:N/A:H" + } + ], + "url": "https://ftp.suse.com/pub/projects/security/yaml/suse-cvss-scores.yaml" + }, + { + "reference_url": "https://github.com/django/django", + "reference_id": "", + "scores": [ + { + "value": "7.5", + "scoring_system": "cvssv3.1", + "scoring_elements": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H" + }, + { + "value": "HIGH", + "scoring_system": "generic_textual", + "scoring_elements": "" + } + ], + "url": "https://github.com/django/django" + }, + { + "reference_url": "https://github.com/django/django/commit/16a8fe18a3b81250f4fa57e3f93f0599dc4895bc", + "reference_id": "", + "scores": [ + { + "value": "5.9", + "scoring_system": "cvssv3.1", + "scoring_elements": "CVSS:3.1/AV:N/AC:H/PR:N/UI:N/S:U/C:N/I:N/A:H" + }, + { + "value": "MODERATE", + "scoring_system": "generic_textual", + "scoring_elements": "" + } + ], + "url": "https://github.com/django/django/commit/16a8fe18a3b81250f4fa57e3f93f0599dc4895bc" + }, + { + "reference_url": "https://github.com/django/django/commit/55519d6cf8998fe4c8f5c8abffc2b10a7c3d14e9", + "reference_id": "", + "scores": [ + { + "value": "5.9", + "scoring_system": "cvssv3.1", + "scoring_elements": "CVSS:3.1/AV:N/AC:H/PR:N/UI:N/S:U/C:N/I:N/A:H" + }, + { + "value": "MODERATE", + "scoring_system": "generic_textual", + "scoring_elements": "" + } + ], + "url": "https://github.com/django/django/commit/55519d6cf8998fe4c8f5c8abffc2b10a7c3d14e9" + }, + { + "reference_url": "https://github.com/django/django/commit/572ea07e84b38ea8de0551f4b4eda685d91d09d2", + "reference_id": "", + "scores": [ + { + "value": "5.9", + "scoring_system": "cvssv3.1", + "scoring_elements": "CVSS:3.1/AV:N/AC:H/PR:N/UI:N/S:U/C:N/I:N/A:H" + }, + { + "value": "MODERATE", + "scoring_system": "generic_textual", + "scoring_elements": "" + } + ], + "url": "https://github.com/django/django/commit/572ea07e84b38ea8de0551f4b4eda685d91d09d2" + }, + { + "reference_url": "https://github.com/django/django/commit/c1171ffbd570db90ca206c30f8e2b9f691243820", + "reference_id": "", + "scores": [ + { + "value": "5.9", + "scoring_system": "cvssv3.1", + "scoring_elements": "CVSS:3.1/AV:N/AC:H/PR:N/UI:N/S:U/C:N/I:N/A:H" + }, + { + "value": "MODERATE", + "scoring_system": "generic_textual", + "scoring_elements": "" + } + ], + "url": "https://github.com/django/django/commit/c1171ffbd570db90ca206c30f8e2b9f691243820" + }, + { + "reference_url": "https://github.com/pypa/advisory-database/tree/main/vulns/django/PYSEC-2024-28.yaml", + "reference_id": "", + "scores": [ + { + "value": "5.9", + "scoring_system": "cvssv3.1", + "scoring_elements": "CVSS:3.1/AV:N/AC:H/PR:N/UI:N/S:U/C:N/I:N/A:H" + }, + { + "value": "MODERATE", + "scoring_system": "generic_textual", + "scoring_elements": "" + } + ], + "url": "https://github.com/pypa/advisory-database/tree/main/vulns/django/PYSEC-2024-28.yaml" + }, + { + "reference_url": "https://groups.google.com/forum/#%21forum/django-announce", + "reference_id": "", + "scores": [ + { + "value": "5.9", + "scoring_system": "cvssv3.1", + "scoring_elements": "CVSS:3.1/AV:N/AC:H/PR:N/UI:N/S:U/C:N/I:N/A:H" + }, + { + "value": "LOW", + "scoring_system": "generic_textual", + "scoring_elements": "" + } + ], + "url": "https://groups.google.com/forum/#%21forum/django-announce" + }, + { + "reference_url": "https://lists.fedoraproject.org/archives/list/package-announce%40lists.fedoraproject.org/message/D2JIRXEDP4ZET5KFMAPPYSK663Q52NEX", + "reference_id": "", + "scores": [ + { + "value": "5.9", + "scoring_system": "cvssv3.1", + "scoring_elements": "CVSS:3.1/AV:N/AC:H/PR:N/UI:N/S:U/C:N/I:N/A:H" + }, + { + "value": "LOW", + "scoring_system": "generic_textual", + "scoring_elements": "" + } + ], + "url": "https://lists.fedoraproject.org/archives/list/package-announce%40lists.fedoraproject.org/message/D2JIRXEDP4ZET5KFMAPPYSK663Q52NEX" + }, + { + "reference_url": "https://lists.fedoraproject.org/archives/list/package-announce%40lists.fedoraproject.org/message/D2JIRXEDP4ZET5KFMAPPYSK663Q52NEX/", + "reference_id": "", + "scores": [], + "url": "https://lists.fedoraproject.org/archives/list/package-announce%40lists.fedoraproject.org/message/D2JIRXEDP4ZET5KFMAPPYSK663Q52NEX/" + }, + { + "reference_url": "https://lists.fedoraproject.org/archives/list/package-announce%40lists.fedoraproject.org/message/SN2PLJGYSAAG5KUVIUFJYKD3BLQ4OSN6", + "reference_id": "", + "scores": [ + { + "value": "5.9", + "scoring_system": "cvssv3.1", + "scoring_elements": "CVSS:3.1/AV:N/AC:H/PR:N/UI:N/S:U/C:N/I:N/A:H" + }, + { + "value": "LOW", + "scoring_system": "generic_textual", + "scoring_elements": "" + } + ], + "url": "https://lists.fedoraproject.org/archives/list/package-announce%40lists.fedoraproject.org/message/SN2PLJGYSAAG5KUVIUFJYKD3BLQ4OSN6" + }, + { + "reference_url": "https://lists.fedoraproject.org/archives/list/package-announce%40lists.fedoraproject.org/message/SN2PLJGYSAAG5KUVIUFJYKD3BLQ4OSN6/", + "reference_id": "", + "scores": [], + "url": "https://lists.fedoraproject.org/archives/list/package-announce%40lists.fedoraproject.org/message/SN2PLJGYSAAG5KUVIUFJYKD3BLQ4OSN6/" + }, + { + "reference_url": "https://lists.fedoraproject.org/archives/list/package-announce%40lists.fedoraproject.org/message/ZQJOMNRMVPCN5WMIZ7YSX5LQ7IR2NY4D", + "reference_id": "", + "scores": [ + { + "value": "5.9", + "scoring_system": "cvssv3.1", + "scoring_elements": "CVSS:3.1/AV:N/AC:H/PR:N/UI:N/S:U/C:N/I:N/A:H" + }, + { + "value": "LOW", + "scoring_system": "generic_textual", + "scoring_elements": "" + } + ], + "url": "https://lists.fedoraproject.org/archives/list/package-announce%40lists.fedoraproject.org/message/ZQJOMNRMVPCN5WMIZ7YSX5LQ7IR2NY4D" + }, + { + "reference_url": "https://lists.fedoraproject.org/archives/list/package-announce%40lists.fedoraproject.org/message/ZQJOMNRMVPCN5WMIZ7YSX5LQ7IR2NY4D/", + "reference_id": "", + "scores": [], + "url": "https://lists.fedoraproject.org/archives/list/package-announce%40lists.fedoraproject.org/message/ZQJOMNRMVPCN5WMIZ7YSX5LQ7IR2NY4D/" + }, + { + "reference_url": "https://www.djangoproject.com/weblog/2024/feb/06/security-releases", + "reference_id": "", + "scores": [ + { + "value": "5.9", + "scoring_system": "cvssv3.1", + "scoring_elements": "CVSS:3.1/AV:N/AC:H/PR:N/UI:N/S:U/C:N/I:N/A:H" + }, + { + "value": "MODERATE", + "scoring_system": "generic_textual", + "scoring_elements": "" + } + ], + "url": "https://www.djangoproject.com/weblog/2024/feb/06/security-releases" + }, + { + "reference_url": "https://www.djangoproject.com/weblog/2024/feb/06/security-releases/", + "reference_id": "", + "scores": [], + "url": "https://www.djangoproject.com/weblog/2024/feb/06/security-releases/" + }, + { + "reference_url": "https://bugzilla.redhat.com/show_bug.cgi?id=2261856", + "reference_id": "2261856", + "scores": [], + "url": "https://bugzilla.redhat.com/show_bug.cgi?id=2261856" + }, + { + "reference_url": "https://nvd.nist.gov/vuln/search/results?adv_search=true&isCpeNameSearch=true&query=cpe:2.3:a:djangoproject:django:*:*:*:*:*:*:*:*", + "reference_id": "cpe:2.3:a:djangoproject:django:*:*:*:*:*:*:*:*", + "scores": [], + "url": "https://nvd.nist.gov/vuln/search/results?adv_search=true&isCpeNameSearch=true&query=cpe:2.3:a:djangoproject:django:*:*:*:*:*:*:*:*" + }, + { + "reference_url": "https://nvd.nist.gov/vuln/detail/CVE-2024-24680", + "reference_id": "CVE-2024-24680", + "scores": [ + { + "value": "7.5", + "scoring_system": "cvssv3", + "scoring_elements": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H" + } + ], + "url": "https://nvd.nist.gov/vuln/detail/CVE-2024-24680" + }, + { + "reference_url": "https://github.com/advisories/GHSA-xxj9-f6rv-m3x4", + "reference_id": "GHSA-xxj9-f6rv-m3x4", + "scores": [ + { + "value": "MODERATE", + "scoring_system": "cvssv3.1_qr", + "scoring_elements": "" + } + ], + "url": "https://github.com/advisories/GHSA-xxj9-f6rv-m3x4" + }, + { + "reference_url": "https://access.redhat.com/errata/RHSA-2024:1057", + "reference_id": "RHSA-2024:1057", + "scores": [], + "url": "https://access.redhat.com/errata/RHSA-2024:1057" + }, + { + "reference_url": "https://access.redhat.com/errata/RHSA-2024:1878", + "reference_id": "RHSA-2024:1878", + "scores": [], + "url": "https://access.redhat.com/errata/RHSA-2024:1878" + }, + { + "reference_url": "https://access.redhat.com/errata/RHSA-2024:2731", + "reference_id": "RHSA-2024:2731", + "scores": [], + "url": "https://access.redhat.com/errata/RHSA-2024:2731" + }, + { + "reference_url": "https://usn.ubuntu.com/6623-1/", + "reference_id": "USN-6623-1", + "scores": [], + "url": "https://usn.ubuntu.com/6623-1/" + } + ], + "fixed_packages": [ + { + "url": "http://vulnerablecode/api/packages/807572", + "purl": "pkg:pypi/django@5.0.2", + "is_vulnerable": true, + "affected_by_vulnerabilities": [ + { + "vulnerability": "VCID-q4q6-yfng-aaag" + } + ], + "resource_url": "http://vulnerablecode/packages/pkg:pypi/django@5.0.2" + } + ], + "aliases": [ + "BIT-django-2024-24680", + "CVE-2024-24680", + "GHSA-xxj9-f6rv-m3x4", + "PYSEC-2024-28" + ], + "resource_url": "http://vulnerablecode/vulnerabilities/VCID-3gge-bre2-aaac" + }, + { + "url": "http://vulnerablecode/api/vulnerabilities/522941", + "vulnerability_id": "VCID-q4q6-yfng-aaag", + "summary": "In Django 3.2 before 3.2.25, 4.2 before 4.2.11, and 5.0 before 5.0.3, the django.utils.text.Truncator.words() method (with html=True) and the truncatewords_html template filter are subject to a potential regular expression denial-of-service attack via a crafted string. NOTE: this issue exists because of an incomplete fix for CVE-2019-14232 and CVE-2023-43665.", + "references": [ + { + "reference_url": "https://access.redhat.com/hydra/rest/securitydata/cve/CVE-2024-27351.json", + "reference_id": "", + "scores": [ + { + "value": "7.5", + "scoring_system": "cvssv3", + "scoring_elements": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H" + } + ], + "url": "https://access.redhat.com/hydra/rest/securitydata/cve/CVE-2024-27351.json" + }, + { + "reference_url": "https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2024-27351", + "reference_id": "", + "scores": [], + "url": "https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2024-27351" + }, + { + "reference_url": "https://docs.djangoproject.com/en/5.0/releases/security", + "reference_id": "", + "scores": [ + { + "value": "5.9", + "scoring_system": "cvssv3.1", + "scoring_elements": "CVSS:3.1/AV:N/AC:H/PR:N/UI:N/S:U/C:N/I:N/A:H" + }, + { + "value": "LOW", + "scoring_system": "generic_textual", + "scoring_elements": "" + } + ], + "url": "https://docs.djangoproject.com/en/5.0/releases/security" + }, + { + "reference_url": "https://docs.djangoproject.com/en/5.0/releases/security/", + "reference_id": "", + "scores": [], + "url": "https://docs.djangoproject.com/en/5.0/releases/security/" + }, + { + "reference_url": "https://ftp.suse.com/pub/projects/security/yaml/suse-cvss-scores.yaml", + "reference_id": "", + "scores": [ + { + "value": "2.1", + "scoring_system": "cvssv2", + "scoring_elements": "AV:N/AC:H/Au:S/C:N/I:P/A:N" + }, + { + "value": "4.4", + "scoring_system": "cvssv3", + "scoring_elements": "CVSS:3.0/AV:L/AC:L/PR:N/UI:R/S:U/C:L/I:N/A:L" + }, + { + "value": "4.7", + "scoring_system": "cvssv3.1", + "scoring_elements": "CVSS:3.1/AV:L/AC:H/PR:L/UI:N/S:U/C:N/I:N/A:H" + } + ], + "url": "https://ftp.suse.com/pub/projects/security/yaml/suse-cvss-scores.yaml" + }, + { + "reference_url": "https://github.com/django/django", + "reference_id": "", + "scores": [ + { + "value": "7.5", + "scoring_system": "cvssv3.1", + "scoring_elements": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H" + }, + { + "value": "HIGH", + "scoring_system": "generic_textual", + "scoring_elements": "" + } + ], + "url": "https://github.com/django/django" + }, + { + "reference_url": "https://github.com/django/django/commit/072963e4c4d0b3a7a8c5412bc0c7d27d1a9c3521", + "reference_id": "", + "scores": [ + { + "value": "LOW", + "scoring_system": "generic_textual", + "scoring_elements": "" + } + ], + "url": "https://github.com/django/django/commit/072963e4c4d0b3a7a8c5412bc0c7d27d1a9c3521" + }, + { + "reference_url": "https://github.com/django/django/commit/3394fc6132436eca89e997083bae9985fb7e761e", + "reference_id": "", + "scores": [ + { + "value": "LOW", + "scoring_system": "generic_textual", + "scoring_elements": "" + } + ], + "url": "https://github.com/django/django/commit/3394fc6132436eca89e997083bae9985fb7e761e" + }, + { + "reference_url": "https://github.com/django/django/commit/3c9a2771cc80821e041b16eb36c1c37af5349d4a", + "reference_id": "", + "scores": [ + { + "value": "LOW", + "scoring_system": "generic_textual", + "scoring_elements": "" + } + ], + "url": "https://github.com/django/django/commit/3c9a2771cc80821e041b16eb36c1c37af5349d4a" + }, + { + "reference_url": "https://github.com/pypa/advisory-database/tree/main/vulns/django/PYSEC-2024-47.yaml", + "reference_id": "", + "scores": [ + { + "value": "LOW", + "scoring_system": "generic_textual", + "scoring_elements": "" + } + ], + "url": "https://github.com/pypa/advisory-database/tree/main/vulns/django/PYSEC-2024-47.yaml" + }, + { + "reference_url": "https://groups.google.com/forum/#%21forum/django-announce", + "reference_id": "", + "scores": [ + { + "value": "5.9", + "scoring_system": "cvssv3.1", + "scoring_elements": "CVSS:3.1/AV:N/AC:H/PR:N/UI:N/S:U/C:N/I:N/A:H" + }, + { + "value": "LOW", + "scoring_system": "generic_textual", + "scoring_elements": "" + } + ], + "url": "https://groups.google.com/forum/#%21forum/django-announce" + }, + { + "reference_url": "https://lists.fedoraproject.org/archives/list/package-announce%40lists.fedoraproject.org/message/D2JIRXEDP4ZET5KFMAPPYSK663Q52NEX", + "reference_id": "", + "scores": [ + { + "value": "5.9", + "scoring_system": "cvssv3.1", + "scoring_elements": "CVSS:3.1/AV:N/AC:H/PR:N/UI:N/S:U/C:N/I:N/A:H" + }, + { + "value": "LOW", + "scoring_system": "generic_textual", + "scoring_elements": "" + } + ], + "url": "https://lists.fedoraproject.org/archives/list/package-announce%40lists.fedoraproject.org/message/D2JIRXEDP4ZET5KFMAPPYSK663Q52NEX" + }, + { + "reference_url": "https://lists.fedoraproject.org/archives/list/package-announce%40lists.fedoraproject.org/message/D2JIRXEDP4ZET5KFMAPPYSK663Q52NEX/", + "reference_id": "", + "scores": [], + "url": "https://lists.fedoraproject.org/archives/list/package-announce%40lists.fedoraproject.org/message/D2JIRXEDP4ZET5KFMAPPYSK663Q52NEX/" + }, + { + "reference_url": "https://lists.fedoraproject.org/archives/list/package-announce%40lists.fedoraproject.org/message/SN2PLJGYSAAG5KUVIUFJYKD3BLQ4OSN6", + "reference_id": "", + "scores": [ + { + "value": "5.9", + "scoring_system": "cvssv3.1", + "scoring_elements": "CVSS:3.1/AV:N/AC:H/PR:N/UI:N/S:U/C:N/I:N/A:H" + }, + { + "value": "LOW", + "scoring_system": "generic_textual", + "scoring_elements": "" + } + ], + "url": "https://lists.fedoraproject.org/archives/list/package-announce%40lists.fedoraproject.org/message/SN2PLJGYSAAG5KUVIUFJYKD3BLQ4OSN6" + }, + { + "reference_url": "https://lists.fedoraproject.org/archives/list/package-announce%40lists.fedoraproject.org/message/SN2PLJGYSAAG5KUVIUFJYKD3BLQ4OSN6/", + "reference_id": "", + "scores": [], + "url": "https://lists.fedoraproject.org/archives/list/package-announce%40lists.fedoraproject.org/message/SN2PLJGYSAAG5KUVIUFJYKD3BLQ4OSN6/" + }, + { + "reference_url": "https://lists.fedoraproject.org/archives/list/package-announce%40lists.fedoraproject.org/message/ZQJOMNRMVPCN5WMIZ7YSX5LQ7IR2NY4D", + "reference_id": "", + "scores": [ + { + "value": "5.9", + "scoring_system": "cvssv3.1", + "scoring_elements": "CVSS:3.1/AV:N/AC:H/PR:N/UI:N/S:U/C:N/I:N/A:H" + }, + { + "value": "LOW", + "scoring_system": "generic_textual", + "scoring_elements": "" + } + ], + "url": "https://lists.fedoraproject.org/archives/list/package-announce%40lists.fedoraproject.org/message/ZQJOMNRMVPCN5WMIZ7YSX5LQ7IR2NY4D" + }, + { + "reference_url": "https://lists.fedoraproject.org/archives/list/package-announce%40lists.fedoraproject.org/message/ZQJOMNRMVPCN5WMIZ7YSX5LQ7IR2NY4D/", + "reference_id": "", + "scores": [], + "url": "https://lists.fedoraproject.org/archives/list/package-announce%40lists.fedoraproject.org/message/ZQJOMNRMVPCN5WMIZ7YSX5LQ7IR2NY4D/" + }, + { + "reference_url": "https://www.djangoproject.com/weblog/2024/mar/04/security-releases", + "reference_id": "", + "scores": [ + { + "value": "LOW", + "scoring_system": "generic_textual", + "scoring_elements": "" + } + ], + "url": "https://www.djangoproject.com/weblog/2024/mar/04/security-releases" + }, + { + "reference_url": "https://www.djangoproject.com/weblog/2024/mar/04/security-releases/", + "reference_id": "", + "scores": [], + "url": "https://www.djangoproject.com/weblog/2024/mar/04/security-releases/" + }, + { + "reference_url": "http://www.openwall.com/lists/oss-security/2024/03/04/1", + "reference_id": "", + "scores": [ + { + "value": "5.9", + "scoring_system": "cvssv3.1", + "scoring_elements": "CVSS:3.1/AV:N/AC:H/PR:N/UI:N/S:U/C:N/I:N/A:H" + }, + { + "value": "LOW", + "scoring_system": "generic_textual", + "scoring_elements": "" + } + ], + "url": "http://www.openwall.com/lists/oss-security/2024/03/04/1" + }, + { + "reference_url": "https://bugzilla.redhat.com/show_bug.cgi?id=2266045", + "reference_id": "2266045", + "scores": [], + "url": "https://bugzilla.redhat.com/show_bug.cgi?id=2266045" + }, + { + "reference_url": "https://nvd.nist.gov/vuln/detail/CVE-2024-27351", + "reference_id": "CVE-2024-27351", + "scores": [], + "url": "https://nvd.nist.gov/vuln/detail/CVE-2024-27351" + }, + { + "reference_url": "https://github.com/advisories/GHSA-vm8q-m57g-pff3", + "reference_id": "GHSA-vm8q-m57g-pff3", + "scores": [ + { + "value": "LOW", + "scoring_system": "cvssv3.1_qr", + "scoring_elements": "" + } + ], + "url": "https://github.com/advisories/GHSA-vm8q-m57g-pff3" + }, + { + "reference_url": "https://access.redhat.com/errata/RHSA-2024:1878", + "reference_id": "RHSA-2024:1878", + "scores": [], + "url": "https://access.redhat.com/errata/RHSA-2024:1878" + }, + { + "reference_url": "https://access.redhat.com/errata/RHSA-2024:3781", + "reference_id": "RHSA-2024:3781", + "scores": [], + "url": "https://access.redhat.com/errata/RHSA-2024:3781" + }, + { + "reference_url": "https://usn.ubuntu.com/6674-1/", + "reference_id": "USN-6674-1", + "scores": [], + "url": "https://usn.ubuntu.com/6674-1/" + }, + { + "reference_url": "https://usn.ubuntu.com/6674-2/", + "reference_id": "USN-6674-2", + "scores": [], + "url": "https://usn.ubuntu.com/6674-2/" + } + ], + "fixed_packages": [ + { + "url": "http://vulnerablecode/api/packages/810988", + "purl": "pkg:pypi/django@5.0.3", + "is_vulnerable": false, + "affected_by_vulnerabilities": [], + "resource_url": "http://vulnerablecode/packages/pkg:pypi/django@5.0.3" + } + ], + "aliases": [ + "CVE-2024-27351", + "GHSA-vm8q-m57g-pff3", + "PYSEC-2024-47" + ], + "resource_url": "http://vulnerablecode/vulnerabilities/VCID-q4q6-yfng-aaag" + } + ], + "fixing_vulnerabilities": [], + "resource_url": "http://vulnerablecode/packages/pkg:pypi/django@5.0" +} \ No newline at end of file diff --git a/scanpipe/tests/pipes/test_vulnerablecode.py b/scanpipe/tests/pipes/test_vulnerablecode.py new file mode 100644 index 000000000..859308d52 --- /dev/null +++ b/scanpipe/tests/pipes/test_vulnerablecode.py @@ -0,0 +1,83 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# http://nexb.com and https://github.com/nexB/scancode.io +# The ScanCode.io software is licensed under the Apache License version 2.0. +# Data generated with ScanCode.io is provided as-is without warranties. +# ScanCode is a trademark of nexB Inc. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# ScanCode.io should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# +# ScanCode.io is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/nexB/scancode.io for support and download. + +import io +import json +from pathlib import Path +from unittest import mock + +from django.test import TestCase + +from scanpipe.models import Project +from scanpipe.pipes.vulnerablecode import fetch_vulnerabilities +from scanpipe.pipes.vulnerablecode import filter_vulnerabilities +from scanpipe.tests import make_package + + +class ScanPipeVulnerableCodeTest(TestCase): + data = Path(__file__).parent.parent / "data" + + def setUp(self): + self.project1 = Project.objects.create(name="Analysis") + + @mock.patch("scanpipe.pipes.vulnerablecode.bulk_search_by_purl") + def test_scanpipe_pipes_vulnerablecode_fetch_vulnerabilities( + self, mock_search_by_purl + ): + django_5_0 = make_package(self.project1, "pkg:pypi/django@5.0") + data_location = self.data / "vulnerablecode/django-5.0_package_data.json" + package_data = json.loads(data_location.read_text()) + mock_search_by_purl.return_value = [package_data] + buffer = io.StringIO() + + fetch_vulnerabilities(packages=[django_5_0], logger=buffer.write) + django_5_0.refresh_from_db() + self.assertEqual(2, len(package_data.get("affected_by_vulnerabilities"))) + self.assertEqual( + package_data.get("affected_by_vulnerabilities"), + django_5_0.affected_by_vulnerabilities, + ) + self.assertEqual( + "1 discovered packages updated with vulnerability data.", buffer.getvalue() + ) + + fetch_vulnerabilities(packages=[django_5_0], ignore_set={"VCID-3gge-bre2-aaac"}) + django_5_0.refresh_from_db() + self.assertEqual(1, len(django_5_0.affected_by_vulnerabilities)) + + def test_scanpipe_pipes_vulnerablecode_filter_vulnerabilities(self): + data_location = self.data / "vulnerablecode/django-5.0_package_data.json" + package_data = json.loads(data_location.read_text()) + vulnerability_data = package_data["affected_by_vulnerabilities"] + self.assertEqual(2, len(vulnerability_data)) + + vulnerability1 = vulnerability_data[0] + self.assertEqual("VCID-3gge-bre2-aaac", vulnerability1.get("vulnerability_id")) + ignore_set = {vulnerability1.get("vulnerability_id")} + self.assertEqual(1, len(filter_vulnerabilities(vulnerability_data, ignore_set))) + + ignore_set = {vulnerability1.get("aliases")[0]} + self.assertEqual(1, len(filter_vulnerabilities(vulnerability_data, ignore_set))) + + vulnerability2 = vulnerability_data[1] + ignore_set.add(vulnerability2.get("aliases")[1]) + self.assertEqual([], filter_vulnerabilities(vulnerability_data, ignore_set)) diff --git a/scanpipe/tests/test_forms.py b/scanpipe/tests/test_forms.py index eb6d59dab..e10488477 100644 --- a/scanpipe/tests/test_forms.py +++ b/scanpipe/tests/test_forms.py @@ -133,6 +133,7 @@ def test_scanpipe_forms_project_settings_form_update_project_settings(self): expected = { "ignored_patterns": ["*.ext", "dir/*"], + "ignored_vulnerabilities": None, "ignored_dependency_scopes": None, "product_name": "", "product_version": "", @@ -181,6 +182,7 @@ def test_scanpipe_forms_project_settings_form_ignored_dependency_scopes(self): project = form.save() expected = { "ignored_patterns": None, + "ignored_vulnerabilities": None, "ignored_dependency_scopes": [ {"package_type": "npm", "scope": "devDependencies"}, {"package_type": "pypi", "scope": "tests"}, diff --git a/scanpipe/tests/test_models.py b/scanpipe/tests/test_models.py index dfb25b889..12fa7dc8f 100644 --- a/scanpipe/tests/test_models.py +++ b/scanpipe/tests/test_models.py @@ -677,6 +677,11 @@ def test_scanpipe_project_get_env(self): {"package_type": "npm", "scope": "devDependencies"}, {"package_type": "pypi", "scope": "tests"}, ], + "ignored_vulnerabilities": [ + "VCID-q4q6-yfng-aaag", + "CVE-2024-27351", + "GHSA-vm8q-m57g-pff3", + ], } self.assertEqual(expected, self.project1.get_env()) @@ -725,6 +730,18 @@ def test_scanpipe_project_get_ignored_dependency_scopes_index(self): expected = {"npm": ["devDependencies"], "pypi": ["tests", "build"]} self.assertEqual(expected, self.project1.get_ignored_dependency_scopes_index()) + def test_scanpipe_project_get_ignored_vulnerabilities_set(self): + self.project1.settings = { + "ignored_vulnerabilities": [ + "VCID-q4q6-yfng-aaag", + "CVE-2024-27351", + "GHSA-vm8q-m57g-pff3", + ], + } + expected = {"VCID-q4q6-yfng-aaag", "CVE-2024-27351", "GHSA-vm8q-m57g-pff3"} + self.assertEqual(expected, self.project1.ignored_vulnerabilities_set) + self.assertEqual(expected, self.project1.get_ignored_vulnerabilities_set()) + def test_scanpipe_project_model_labels(self): self.project1.labels.add("label1", "label2") self.assertEqual(2, UUIDTaggedItem.objects.count()) From 7418cfe485e9b262495dc0d6e12b7b4d25ca8036 Mon Sep 17 00:00:00 2001 From: tdruez Date: Fri, 21 Jun 2024 15:16:47 +0400 Subject: [PATCH 2/8] Fix formatting in the project-configuration.rst #1271 Signed-off-by: tdruez --- docs/project-configuration.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/project-configuration.rst b/docs/project-configuration.rst index 046394bd7..a133e67fc 100644 --- a/docs/project-configuration.rst +++ b/docs/project-configuration.rst @@ -154,8 +154,6 @@ Provide one or more vulnerability id to be ignored, **one per line**. You can provide ``VCID`` from VulnerableCode or any aliases such as ``CVE`` or ``GHSA``. -For example:: - .. code-block:: yaml ignored_vulnerabilities: From d528a2a70862bec2e92447199411e5e506760f64 Mon Sep 17 00:00:00 2001 From: tdruez Date: Fri, 21 Jun 2024 16:14:55 +0400 Subject: [PATCH 3/8] Bump version to v34.6.3 for release Signed-off-by: tdruez --- CHANGELOG.rst | 2 +- scancodeio/__init__.py | 2 +- setup.cfg | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 7ae0dc7b0..909cb4c4e 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,7 +1,7 @@ Changelog ========= -v34.6.3 (unreleased) +v34.6.3 (2024-06-21) -------------------- - Use the ``--option=value`` syntax for args entries in place of ``--option value`` diff --git a/scancodeio/__init__.py b/scancodeio/__init__.py index 0abafc6e2..310ad6fdb 100644 --- a/scancodeio/__init__.py +++ b/scancodeio/__init__.py @@ -28,7 +28,7 @@ import git -VERSION = "34.6.2" +VERSION = "34.6.3" PROJECT_DIR = Path(__file__).resolve().parent ROOT_DIR = PROJECT_DIR.parent diff --git a/setup.cfg b/setup.cfg index aa1cdbe57..a2475610b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = scancodeio -version = 34.6.2 +version = 34.6.3 license = Apache-2.0 description = Automate software composition analysis pipelines long_description = file:README.rst @@ -170,7 +170,7 @@ ignore = D1,D203,D205,D212,D400,D415 [bumpver] version_pattern = "MAJOR.MINOR.PATCH" -current_version = "34.6.2" +current_version = "34.6.3" [bumpver:file_patterns] setup.cfg = From 5461cbdb3ad2379ffffa6e27bd1afa9b347fd9d3 Mon Sep 17 00:00:00 2001 From: tdruez <489057+tdruez@users.noreply.github.com> Date: Mon, 24 Jun 2024 13:15:49 +0400 Subject: [PATCH 4/8] Add all "classify" plugin fields on the CodebaseResource model #1275 (#1286) Signed-off-by: tdruez --- CHANGELOG.rst | 6 + scanpipe/api/serializers.py | 4 + scanpipe/filters.py | 6 +- ...0061_codebaseresource_is_legal_and_more.py | 52 + scanpipe/models.py | 45 +- .../includes/resource_file_viewer.html | 20 +- scanpipe/tests/__init__.py | 4 + .../data/alpine_3_15_4_scan_codebase.json | 716 ++ scanpipe/tests/data/asgiref-3.3.0.spdx.json | 42 +- .../tests/data/asgiref-3.3.0_fixtures.json | 562 +- ...asgiref-3.3.0_load_inventory_expected.json | 72 + .../data/asgiref-3.3.0_scanpipe_output.json | 156 +- .../data/asgiref-3.3.0_toolkit_scan.json | 66 +- .../asgiref-3.3.0_walk_test_fixtures.json | 562 +- .../data/basic-rootfs_root_filesystems.json | 64 + scanpipe/tests/data/centos_scan_codebase.json | 116 + .../data/cyclonedx/asgiref-3.3.0.cdx.json | 18 +- .../tests/data/d2d/about_files/expected.json | 176 + ...-0.6.0-py3-none-any.whl_scan_codebase.json | 44 + scanpipe/tests/data/debian_scan_codebase.json | 64 + scanpipe/tests/data/flume-ng-node-d2d.json | 228 + .../gcr_io_distroless_base_scan_codebase.json | 7024 +++++++++++++++++ .../minitag.tar-expected-scan.json | 32 + .../data/is-npm-1.0.0_scan_codebase.json | 24 + .../is-npm-1.0.0_scan_package_summary.json | 4 + ...ple-is-npm-1.0.0_scan_package_summary.json | 4 + .../data/scancode/is-npm-1.0.0_summary.json | 4 + .../scancode/package_assembly_codebase.json | 20 +- scanpipe/tests/pipes/test_output.py | 2 +- scanpipe/tests/test_api.py | 2 +- scanpipe/views.py | 6 +- 31 files changed, 9553 insertions(+), 592 deletions(-) create mode 100644 scanpipe/migrations/0061_codebaseresource_is_legal_and_more.py diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 909cb4c4e..fe9ca12fc 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,12 @@ Changelog ========= +v34.6.4 (unreleased) +-------------------- + +- Add all "classify" plugin fields from scancode-toolkit on the CodebaseResource model. + https://github.com/nexB/scancode.io/issues/1275 + v34.6.3 (2024-06-21) -------------------- diff --git a/scanpipe/api/serializers.py b/scanpipe/api/serializers.py index b6fdf0de3..98a039db2 100644 --- a/scanpipe/api/serializers.py +++ b/scanpipe/api/serializers.py @@ -315,6 +315,10 @@ class Meta: "is_text", "is_archive", "is_media", + "is_legal", + "is_manifest", + "is_readme", + "is_top_level", "is_key_file", "detected_license_expression", "detected_license_expression_spdx", diff --git a/scanpipe/filters.py b/scanpipe/filters.py index 4c32b13f7..3a06db58c 100644 --- a/scanpipe/filters.py +++ b/scanpipe/filters.py @@ -566,8 +566,12 @@ class Meta: "is_binary", "is_text", "is_archive", - "is_key_file", "is_media", + "is_legal", + "is_manifest", + "is_readme", + "is_top_level", + "is_key_file", ] def __init__(self, *args, **kwargs): diff --git a/scanpipe/migrations/0061_codebaseresource_is_legal_and_more.py b/scanpipe/migrations/0061_codebaseresource_is_legal_and_more.py new file mode 100644 index 000000000..7ed1edcdd --- /dev/null +++ b/scanpipe/migrations/0061_codebaseresource_is_legal_and_more.py @@ -0,0 +1,52 @@ +# Generated by Django 5.0.6 on 2024-06-24 05:57 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("scanpipe", "0060_discovereddependency_renames"), + ] + + operations = [ + migrations.AddField( + model_name="codebaseresource", + name="is_legal", + field=models.BooleanField( + default=False, + help_text="True if this file is likely a legal, license-related file such as a COPYING or LICENSE file.", + ), + ), + migrations.AddField( + model_name="codebaseresource", + name="is_manifest", + field=models.BooleanField( + default=False, + help_text="True if this file is likely a package manifest file such as a Maven pom.xml or an npm package.json", + ), + ), + migrations.AddField( + model_name="codebaseresource", + name="is_readme", + field=models.BooleanField( + default=False, help_text="True if this file is likely a README file." + ), + ), + migrations.AddField( + model_name="codebaseresource", + name="is_top_level", + field=models.BooleanField( + default=False, + help_text="True if this file is top-level file located either at the root of a package or in a well-known common location.", + ), + ), + migrations.AlterField( + model_name="codebaseresource", + name="is_key_file", + field=models.BooleanField( + default=False, + help_text="True if this file is top-level file and either a legal, readme or manifest file.", + ), + ), + ] diff --git a/scanpipe/models.py b/scanpipe/models.py index 02c9c929f..06b55c4a6 100644 --- a/scanpipe/models.py +++ b/scanpipe/models.py @@ -2343,9 +2343,53 @@ def compute_compliance_alert(self): return self.Compliance.OK +class FileClassifierFieldsModelMixin(models.Model): + """ + Fields returned by the ScanCode-toolkit ``--classify`` plugin. + See ``summarycode.classify_plugin.FileClassifier``. + """ + + is_legal = models.BooleanField( + default=False, + help_text=_( + "True if this file is likely a legal, license-related file such as a " + "COPYING or LICENSE file." + ), + ) + is_manifest = models.BooleanField( + default=False, + help_text=_( + "True if this file is likely a package manifest file such as a Maven " + "pom.xml or an npm package.json" + ), + ) + is_readme = models.BooleanField( + default=False, + help_text=_("True if this file is likely a README file."), + ) + is_top_level = models.BooleanField( + default=False, + help_text=_( + "True if this file is top-level file located either at the root of a " + "package or in a well-known common location." + ), + ) + is_key_file = models.BooleanField( + default=False, + help_text=_( + "True if this file is top-level file and either a legal, readme or " + "manifest file." + ), + ) + + class Meta: + abstract = True + + class CodebaseResource( ProjectRelatedModel, ScanFieldsModelMixin, + FileClassifierFieldsModelMixin, ExtraDataFieldMixin, SaveProjectMessageMixin, UpdateFromDataMixin, @@ -2440,7 +2484,6 @@ class Type(models.TextChoices): is_binary = models.BooleanField(default=False) is_text = models.BooleanField(default=False) is_archive = models.BooleanField(default=False) - is_key_file = models.BooleanField(default=False) is_media = models.BooleanField(default=False) package_data = models.JSONField( default=list, diff --git a/scanpipe/templates/scanpipe/includes/resource_file_viewer.html b/scanpipe/templates/scanpipe/includes/resource_file_viewer.html index c9eb489b1..f1133efb4 100644 --- a/scanpipe/templates/scanpipe/includes/resource_file_viewer.html +++ b/scanpipe/templates/scanpipe/includes/resource_file_viewer.html @@ -12,12 +12,26 @@ {% if object.is_archive %} {% include "scanpipe/dropdowns/dropdown_hoverable.html" with trigger='Archive' content="Archive or compressed file. Not scanned directly." only %} {% endif %} - {% if object.is_key_file %} - {% include "scanpipe/dropdowns/dropdown_hoverable.html" with trigger='Key file' content="Top-level file with key content and metadata and either a legal, readme or package manifest." only %} - {% endif %} {% if object.is_media %} {% include "scanpipe/dropdowns/dropdown_hoverable.html" with trigger='Media' content="Image, sound, video or similar media file." only %} {% endif %} + {% if object.is_legal %} + {% include "scanpipe/dropdowns/dropdown_hoverable.html" with trigger='Legal' content="Legal, license-related file such as a COPYING or LICENSE file." only %} + {% endif %} + + {% if object.is_manifest %} + {% include "scanpipe/dropdowns/dropdown_hoverable.html" with trigger='Manifest' content="Package manifest file such as a Maven pom.xml or a npm package.json" only %} + {% endif %} + {% if object.is_readme %} + {% include "scanpipe/dropdowns/dropdown_hoverable.html" with trigger='Readme' content="README file" only %} + {% endif %} + {% if object.is_top_level %} + {% include "scanpipe/dropdowns/dropdown_hoverable.html" with trigger='Top level' content="Top-level file located either at the root of a package or in a well-known common location." only %} + {% endif %} + + {% if object.is_key_file %} + {% include "scanpipe/dropdowns/dropdown_hoverable.html" with trigger='Key file' content="Top-level file with key content and metadata and either a legal, readme or package manifest." only %} + {% endif %}