Merge branch 'main' into resolve-dependencies-from-lockfile

Signed-off-by: Ayan Sinha Mahapatra <[email protected]>

.gitignore

@@ -47,7 +47,6 @@ local
 /.python-version
 /.pytest_cache/
 /scancodeio.egg-info/
-policies.yml
 *.rdb
 *.aof
 .vscode

CHANGELOG.rst

@@ -1,7 +1,21 @@
 Changelog
 =========
 
-v34.6.3 (unreleased)
+v34.6.4 (unreleased)
+--------------------
+
+- Add all "classify" plugin fields from scancode-toolkit on the CodebaseResource model.
+  https://github.com/nexB/scancode.io/issues/1275
+
+- Refine the extraction errors reporting to include the resource path for rendering
+  link to the related resources in the UI.
+  https://github.com/nexB/scancode.io/issues/1273
+
+- Add a ``flush-projects`` management command, to Delete all project data and their
+  related work directories created more than a specified number of days ago.
+  https://github.com/nexB/scancode.io/issues/1289
+
+v34.6.3 (2024-06-21)
 --------------------
 
 - Use the ``--option=value`` syntax for args entries in place of ``--option value``
@@ -11,6 +25,9 @@ v34.6.3 (unreleased)
 - Fix an issue in the d2d JavaScript mapper.
   https://github.com/nexB/scancode.io/pull/1274
 
+- Add support for a ``ignored_vulnerabilities`` field on the Project configuration.
+  https://github.com/nexB/scancode.io/issues/1271
+
 v34.6.2 (2024-06-18)
 --------------------
 

docs/command-line-interface.rst

@@ -298,6 +298,27 @@ Optional arguments:
 - ``--no-input`` Does not prompt the user for input of any kind.
 
 
+.. _cli_flush_projects:
+
+`$ scanpipe flush-projects`
+---------------------------
+
+Delete all project data and their related work directories created more than a
+specified number of days ago.
+
+Optional arguments:
+
+- ``---retain-days RETAIN_DAYS`` Specify the number of days to retain data.
+  All data older than this number of days will be deleted.
+  **Defaults to 0 (delete all data)**.
+
+  For example, to delete all projects created more than one week ago::
+
+    scanpipe flush-projects --retain-days 7
+
+- ``--no-input`` Does not prompt the user for input of any kind.
+
+
 .. _cli_create_user:
 
 `$ scanpipe create-user <username>`

docs/faq.rst

@@ -249,3 +249,32 @@ Note that only the HTTPS type of URL is supported::
 A GitHub repository URL example::
 
     https://github.com/username/repository.git
+
+How can I cleanup my ScanCode.io installation, removing all projects and related data?
+--------------------------------------------------------------------------------------
+
+You can use the :ref:`cli_flush_projects` command to perform bulk deletion of projects
+and their associated data stored on disk::
+
+    $ scanpipe flush-projects
+
+**Confirmation will be required before deletion.**
+
+To automate this process, such as running it from a cron job, you can use the
+``--no-input`` option to skip confirmation prompts.
+
+Additionally, you can retain specific projects and their data based on their
+creation date using the ``--retain-days`` option.
+
+Here's an example of a crontab entry that runs daily and flushes all projects and
+data older than 7 days::
+
+    @daily scanpipe flush-projects --retain-days 7 --no-input
+
+.. note:: If you are use Docker for running ScanCode.io, you can run the scanpipe
+  ``flush-projects`` command using::
+
+    docker compose run --rm web scanpipe flush-projects
+
+  See :ref:`command_line_interface` chapter for more information about the scanpipe
+  command.

docs/project-configuration.rst

@@ -52,16 +52,20 @@ Content of a ``scancode-config.yml`` file:
     product_name: My Product Name
     product_version: '1.0'
     ignored_patterns:
-      - '*.tmp'
-      - 'tests/*'
+     - '*.tmp'
+     - 'tests/*'
     ignored_dependency_scopes:
      - package_type: npm
        scope: devDependencies
      - package_type: pypi
        scope: tests
+    ignored_vulnerabilities:
+     - VCID-q4q6-yfng-aaag
+     - CVE-2024-27351
+     - GHSA-vm8q-m57g-pff3
 
-See the :ref:`project_configuration_settings` section for the details about each
-setting.
+See the following :ref:`project_configuration_settings` section for the details about
+each setting.
 
 .. tip::
     You can generate the project configuration file from the
@@ -125,10 +129,10 @@ packages, define the following in your ``scancode-config.yml`` configuration fil
 .. code-block:: yaml
 
     ignored_dependency_scopes:
-      - package_type: npm
-        scope: devDependencies
-      - package_type: pypi
-        scope: tests
+     - package_type: npm
+       scope: devDependencies
+     - package_type: pypi
+       scope: tests
 
 If you prefer to use the :ref:`user_interface_project_settings` form, list each
 ignored scope using the `package_type:scope` syntax, **one per line**, such as:
@@ -141,3 +145,21 @@ ignored scope using the `package_type:scope` syntax, **one per line**, such as:
 .. warning::
     Be precise when listing scope names to avoid unintended exclusions.
     Ensure the scope names are correct and reflect your project requirements.
+
+ignored_vulnerabilities
+^^^^^^^^^^^^^^^^^^^^^^^
+
+Provide one or more vulnerability id to be ignored, **one per line**.
+
+You can provide ``VCID`` from VulnerableCode or any aliases such as ``CVE`` or
+``GHSA``.
+
+.. code-block:: yaml
+
+    ignored_vulnerabilities:
+     - VCID-q4q6-yfng-aaag
+     - CVE-2024-27351
+     - GHSA-vm8q-m57g-pff3
+     - OSV-2020-871
+     - BIT-django-2024-24680
+     - PYSEC-2024-28

docs/scancode-config.yml

@@ -9,12 +9,18 @@
 # - Use ``*`` to match multiple characters.
 # - Use ``?`` to match a single character.
 #ignored_patterns:
-#  - '*.tmp'
-#  - 'tests/*'
+# - '*.tmp'
+# - 'tests/*'
 
 # Specify certain dependency scopes to be ignored for a given package type.
 #ignored_dependency_scopes:
 # - package_type: npm
 #   scope: devDependencies
 # - package_type: pypi
 #   scope: tests
+
+# Specify certain vulnerabilities to be ignored using VCID, CVE, or any aliases.
+#ignored_vulnerabilities:
+# - VCID-q4q6-yfng-aaag
+# - CVE-2024-27351
+# - GHSA-vm8q-m57g-pff3

etc/thirdparty/virtualenv.pyz.ABOUT

@@ -1,7 +1,7 @@
 about_resource: virtualenv.pyz
 name: get-virtualenv
-version: 20.25.1
-download_url: https://github.com/pypa/get-virtualenv/raw/20.25.1/public/virtualenv.pyz
+version: 20.26.3
+download_url: https://github.com/pypa/get-virtualenv/raw/20.26.3/public/virtualenv.pyz
 description: virtualenv is a tool to create isolated Python environments.
 homepage_url: https://github.com/pypa/virtualenv
 license_expression: lgpl-2.1-plus AND (bsd-new OR apache-2.0) AND mit AND python AND bsd-new
@@ -10,4 +10,4 @@ copyright: Copyright (c) The Python Software Foundation and others
 redistribute: yes
 attribute: yes
 track_changes: yes
-package_url: pkg:github/pypa/get-virtualenv@20.25.1#public/virtualenv.pyz
+package_url: pkg:github/pypa/get-virtualenv@20.26.3#public/virtualenv.pyz

scancodeio/__init__.py

@@ -28,7 +28,7 @@
 
 import git
 
-VERSION = "34.6.2"
+VERSION = "34.6.3"
 
 PROJECT_DIR = Path(__file__).resolve().parent
 ROOT_DIR = PROJECT_DIR.parent

scanpipe/api/serializers.py

@@ -315,6 +315,10 @@ class Meta:
             "is_text",
             "is_archive",
             "is_media",
+            "is_legal",
+            "is_manifest",
+            "is_readme",
+            "is_top_level",
             "is_key_file",
             "detected_license_expression",
             "detected_license_expression_spdx",

scanpipe/filters.py

@@ -566,8 +566,12 @@ class Meta:
             "is_binary",
             "is_text",
             "is_archive",
-            "is_key_file",
             "is_media",
+            "is_legal",
+            "is_manifest",
+            "is_readme",
+            "is_top_level",
+            "is_key_file",
         ]
 
     def __init__(self, *args, **kwargs):

scanpipe/forms.py

@@ -365,10 +365,16 @@ def prepare_value(self, value):
 """
 
 
+ignored_vulnerabilities_help = """
+Specify certain vulnerabilities to be ignored using VCID, CVE, or any aliases.
+"""
+
+
 class ProjectSettingsForm(forms.ModelForm):
     settings_fields = [
         "ignored_patterns",
         "ignored_dependency_scopes",
+        "ignored_vulnerabilities",
         "attribution_template",
         "product_name",
         "product_version",
@@ -399,6 +405,18 @@ class ProjectSettingsForm(forms.ModelForm):
         key_name="package_type",
         value_name="scope",
     )
+    ignored_vulnerabilities = ListTextarea(
+        label="Ignored vulnerabilities",
+        required=False,
+        help_text=convert_markdown_to_html(ignored_vulnerabilities_help.strip()),
+        widget=forms.Textarea(
+            attrs={
+                "class": "textarea is-dynamic",
+                "rows": 2,
+                "placeholder": "VCID-q4q6-yfng-aaag\nCVE-2024-27351",
+            },
+        ),
+    )
     attribution_template = forms.CharField(
         label="Attribution template",
         required=False,

scanpipe/management/commands/flush-projects.py

@@ -0,0 +1,97 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# http://nexb.com and https://github.com/nexB/scancode.io
+# The ScanCode.io software is licensed under the Apache License version 2.0.
+# Data generated with ScanCode.io is provided as-is without warranties.
+# ScanCode is a trademark of nexB Inc.
+#
+# You may not use this software except in compliance with the License.
+# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+#
+# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES
+# OR CONDITIONS OF ANY KIND, either express or implied. No content created from
+# ScanCode.io should be considered or used as legal advice. Consult an Attorney
+# for any legal advice.
+#
+# ScanCode.io is a free software code scanning tool from nexB Inc. and others.
+# Visit https://github.com/nexB/scancode.io for support and download.
+
+import datetime
+import sys
+
+from django.core.management.base import BaseCommand
+from django.template.defaultfilters import pluralize
+from django.utils import timezone
+
+from scanpipe.models import Project
+
+
+class Command(BaseCommand):
+    help = (
+        "Delete all project data and their related work directories created more than "
+        "a specified number of days ago."
+    )
+
+    def add_arguments(self, parser):
+        super().add_arguments(parser)
+        parser.add_argument(
+            "--retain-days",
+            type=int,
+            help=(
+                "Optional. Specify the number of days to retain data. "
+                "All data older than this number of days will be deleted. "
+                "Defaults to 0 (delete all data)."
+            ),
+            default=0,
+        )
+        parser.add_argument(
+            "--no-input",
+            action="store_false",
+            dest="interactive",
+            help="Do not prompt the user for input of any kind.",
+        )
+
+    def handle(self, *inputs, **options):
+        verbosity = options["verbosity"]
+        retain_days = options["retain_days"]
+        projects = Project.objects.all()
+
+        if retain_days:
+            cutoff_date = timezone.now() - datetime.timedelta(days=retain_days)
+            projects = projects.filter(created_date__lt=cutoff_date)
+
+        projects_count = projects.count()
+        if projects_count == 0:
+            if verbosity > 0:
+                self.stdout.write("No projects to remove.")
+            sys.exit(0)
+
+        if options["interactive"]:
+            confirm = input(
+                f"You have requested the deletion of {projects_count} "
+                f"project{pluralize(projects_count)}.\n"
+                "This will IRREVERSIBLY DESTROY all data related to those projects.\n"
+                "Are you sure you want to do this?\n"
+                "Type 'yes' to continue, or 'no' to cancel: "
+            )
+            if confirm != "yes":
+                if verbosity > 0:
+                    self.stdout.write("Flush cancelled.")
+                sys.exit(0)
+
+        deletion_count = 0
+        for project in projects:
+            project.delete()
+            deletion_count += 1
+
+        if verbosity > 0:
+            msg = (
+                f"{deletion_count} project{pluralize(deletion_count)} and "
+                f"{pluralize(deletion_count, 'its,their')} related data have been "
+                f"removed."
+            )
+            self.stdout.write(msg, self.style.SUCCESS)