Merge pull request #1 from nexB/main

Merging updated codebase

.github/workflows/main.yml

@@ -38,7 +38,7 @@ jobs:
           pip install -r requirements.txt
     
       - name: Run tests
-        run: python -m pytest
+        run: python -m pytest -v -m "not webtest"
         env:
           # The hostname, username used to communicate with the PostgreSQL service container
           POSTGRES_HOST: localhost

.github/workflows/upstream_test.yml

@@ -0,0 +1,51 @@
+on:
+  workflow_dispatch: # allow manual execution
+  push:
+  schedule:
+    # run on every 9 o'clock
+    - cron: '0 9 * * *'
+
+jobs:
+  unit_tests:
+    runs-on: ubuntu-latest
+
+    services:
+      # Label used to access the service container
+      postgres:
+        image: postgres
+        env:
+          POSTGRES_PASSWORD: vulnerablecode
+          POSTGRES_DB: vulnerablecode
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          # Maps tcp port 5432 on service container to the host
+          - 5432:5432
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v2
+
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+
+      - name: Install dependencies
+        run: |
+          sudo apt install  python3-dev postgresql libpq-dev build-essential libxml2-dev libxslt1-dev 
+          python -m pip install --upgrade pip 
+          pip install -r requirements.txt
+    
+      - name: Run tests
+        run: pytest -v -m webtest
+        env:
+          # The hostname, username used to communicate with the PostgreSQL service container
+          POSTGRES_HOST: localhost
+          VC_DB_USER: postgres
+          POSTGRES_PORT: 5432
+          DJANGO_DEV: 1
+          GH_TOKEN: 1 

.travis.yml

@@ -18,7 +18,7 @@ before_script:
 
 script:
   - ./manage.py collectstatic
-  - python -m pytest
+  - python -m pytest -v -m "not webtest"
 
 notifications:
   email: false

AUTHORS.rst

@@ -11,4 +11,5 @@ The following organizations or individuals have contributed to this repo:
 - Islam Elhakmi @EslamHiko
 - Edoardo Lanzini @elanzini
 - Navonil Das @NavonilDas
-- Tushar Upadhyay @tushar912
+- Tushar Upadhyay @tushar912
+- Hritik Vijay @hritik14

Dockerfile

@@ -6,7 +6,8 @@ ENV PYTHONUNBUFFERED 1
 RUN mkdir /vulnerablecode
 WORKDIR /vulnerablecode
 ADD . /vulnerablecode/
-RUN pip install -r requirements.txt
+RUN pip install -r requirements.txt && \
+    DJANGO_DEV=1 python manage.py collectstatic
 
 LABEL "base_image": "pkg:docker/python@sha256%3Ae9b7e3b4e9569808066c5901b8a9ad315a9f14ae8d3949ece22ae339fff2cad0"
 LABEL "dockerfile_url":  "https://github.com/nexB/vulnerablecode/blob/develop/Dockerfile"

README.rst

@@ -141,11 +141,12 @@ On Debian-based distros, these can be installed with::
 
 **Application dependencies**
 
-Create a virtualenv, install dependencies, and run the database migrations::
+Create a virtualenv, install dependencies, generate static files and run the database migrations::
 
     python3 -m venv venv
     source venv/bin/activate
     pip install -r requirements.txt
+    DJANGO_DEV=1 python manage.py collectstatic
     DJANGO_DEV=1 python manage.py migrate
 
 The environment variable ``DJANGO_DEV`` is used to load settings suitable for
@@ -160,6 +161,11 @@ for this purpose::
 
     SECRET_KEY=$(python -c "from django.core.management import utils; print(utils.get_random_secret_key())")
 
+You will also need to setup the VC_ALLOWED_HOSTS environment variable to match the hostname where the app is deployed::
+
+    VC_ALLOWED_HOSTS=vulnerablecode.your.domain.example.com
+
+You can specify several host by separating them with a colon `:`
 
 Using Nix
 ~~~~~~~~~

etc/nix/flake.nix

@@ -53,9 +53,9 @@
           # mach-nix release) is usually insufficient. Use
           # ./get-latest-pypi-deps-db.sh to obtain the data rev & hash.
           pypiDataRev =
-            "499750266bb4b2840cbe856c2cc0e3297685e362"; # 2021-03-06T08:13:08Z
+            "e9b0fc6b92cd6efbca7ba3b3d4a551bcc13a73c5"; # 2021-03-27T08:13:04Z
           pypiDataSha256 =
-            "188g24k8pk4lgqybywimkvwjwh8014v6l2mrkvzv309882i9p5gc";
+            "1ssa48l2iz8kncby1gfrbds79mg114dkhpxrridwcq6q2c37p62s";
         });
 
     in {

requirements.txt

@@ -9,17 +9,17 @@ contextlib2==0.5.5
 decorator==4.4.2
 dephell-specifier==0.2.1
 dj-database-url==0.4.2
-Django==3.0.7
+Django==3.0.13
 django-filter==2.2.0
-djangorestframework==3.11.0
+djangorestframework==3.11.2
 django-widget-tweaks==1.4.8
 drf-spectacular==0.13.0
 gunicorn==19.7.1
 importlib-metadata==1.3.0
 ipython==7.13.0
 ipython-genutils==0.2.0
 jedi==0.17.0
-lxml==4.6.2
+lxml==4.6.3
 more-itertools==8.0.2
 packageurl-python==0.9.3
 packaging==19.2
@@ -34,7 +34,7 @@ ptyprocess==0.6.0
 py==1.8.0
 pycparser==2.20
 pygit2==1.5.0
-Pygments==2.6.1
+Pygments==2.7.4
 pyparsing==2.4.5
 pytest==5.3.2
 django-widget-tweaks==1.4.8 
@@ -54,5 +54,5 @@ whitenoise==5.0.1
 zipp==0.6.0
 requests==2.23.0
 toml==0.10.2
-PyYAML==5.3.1
+PyYAML==5.4
 freezegun==1.1.0

vulnerabilities/api.py

@@ -152,6 +152,7 @@ def bulk_search(self, request):
             )
         for purl in request.data["purls"]:
             try:
+                purl_string = purl
                 purl = PackageURL.from_string(purl).to_dict()
             except ValueError as ve:
                 return Response(status=400, data={"Error": f"Invalid Package URL: {purl}"})
@@ -165,6 +166,7 @@ def bulk_search(self, request):
                 purl_response = purl
                 purl_response["unresolved_vulnerabilities"] = []
                 purl_response["resolved_vulnerabilities"] = []
+                purl_response["purl"] = purl_string
             response.append(purl_response)
 
         return Response(response)

vulnerabilities/data_source.py

@@ -44,6 +44,7 @@
 
 from vulnerabilities.oval_parser import OvalParser
 from vulnerabilities.severity_systems import ScoringSystem
+from vulnerabilities.helpers import is_cve
 
 logger = logging.getLogger(__name__)
 
@@ -88,6 +89,10 @@ class Advisory:
     resolved_package_urls: Iterable[PackageURL] = dataclasses.field(default_factory=list)
     references: List[Reference] = dataclasses.field(default_factory=list)
 
+    def __post_init__(self):
+        if self.vulnerability_id and not is_cve(self.vulnerability_id):
+            raise ValueError("CVE expected, found: {}".format(self.vulnerability_id))
+
     def normalized(self):
         impacted_package_urls = {package_url for package_url in self.impacted_package_urls}
         resolved_package_urls = {package_url for package_url in self.resolved_package_urls}

vulnerabilities/helpers.py

@@ -25,6 +25,7 @@
 
 import requests
 import toml
+import urllib3
 import yaml
 
 # TODO add logging here
@@ -79,3 +80,21 @@ def create_etag(data_src, url, etag_key):
 
 
 is_cve = re.compile(r"CVE-\d+-\d+", re.IGNORECASE).match
+
+
+def requests_with_5xx_retry(max_retries=5, backoff_factor=0.5):
+    """
+    Returns a requests sessions which retries on 5xx errors with
+    a backoff_factor
+    """
+    retries = urllib3.util.Retry(
+        total=max_retries,
+        backoff_factor=backoff_factor,
+        raise_on_status=True,
+        status_forcelist=range(500, 600, 1),
+    )
+    adapter = requests.adapters.HTTPAdapter(max_retries=retries)
+    session = requests.Session()
+    session.mount("https://", adapter)
+    session.mount("http://", adapter)
+    return session

vulnerabilities/importers/alpine_linux.py

@@ -38,6 +38,7 @@
 from vulnerabilities.data_source import Advisory
 from vulnerabilities.data_source import DataSource
 from vulnerabilities.data_source import Reference
+from vulnerabilities.helpers import is_cve
 
 BASE_URL = "https://secdb.alpinelinux.org/"
 
@@ -193,7 +194,7 @@ def _load_advisories(
                         impacted_package_urls=[],
                         resolved_package_urls=resolved_purls,
                         references=references,
-                        vulnerability_id=vuln_ids[0] if vuln_ids[0] != "CVE-????-?????" else "",
+                        vulnerability_id=vuln_ids[0] if is_cve(vuln_ids[0]) else "",
                     )
                 )
 

vulnerabilities/importers/debian.py

@@ -45,6 +45,7 @@ def validate_schema(advisory_dict):
 
     deb_versions = [
         "bullseye",
+        "bullseye-security",
         "buster",
         "buster-security",
         "sid",

vulnerabilities/importers/istio.py

@@ -31,6 +31,8 @@
 from vulnerabilities.data_source import Advisory, GitDataSource, Reference
 from vulnerabilities.package_managers import GitHubTagsAPI
 
+is_release = re.compile(r"^[\d.]+$", re.IGNORECASE).match
+
 
 class IstioDataSource(GitDataSource):
     def __enter__(self):
@@ -47,7 +49,7 @@ def set_api(self):
         asyncio.run(self.version_api.load_api(["istio/istio"]))
 
     def updated_advisories(self) -> Set[Advisory]:
-        files = self._updated_files
+        files = self._added_files.union(self._updated_files)
         advisories = []
         for f in files:
             processed_data = self.process_file(f)
@@ -195,5 +197,3 @@ def get_data_from_md(self, path):
         with open(path) as f:
             yaml_lines = self.get_yaml_lines(f)
             return self.get_data_from_yaml_lines(yaml_lines)
-
-    is_release = re.compile(r"^[\d.]+$", re.IGNORECASE).match

vulnerabilities/importers/project_kb_msr2019.py

@@ -47,7 +47,7 @@ class ProjectKBMSRDataSource(DataSource):
 
     CONFIG_CLASS = ProjectKBDataSourceConfiguration
 
-    url = "https://raw.githubusercontent.com/SAP/project-kb/master/MSR2019/dataset/vulas_db_msr2019_release.csv"  # nopep8
+    url = "https://raw.githubusercontent.com/SAP/project-kb/master/MSR2019/dataset/vulas_db_msr2019_release.csv"
 
     def updated_advisories(self):
         if create_etag(data_src=self, url=self.url, etag_key="ETag"):
@@ -82,7 +82,7 @@ def to_advisories(csv_reader):
                     summary="",
                     impacted_package_urls=[],
                     references=[reference],
-                    cve_id=vuln_id,
+                    vulnerability_id=vuln_id,
                 )
             )