Merge pull request #91 from nexB/source_only

Add --prefer-source option #90
aboutcode-org · Nov 3, 2022 · 0da2e84 · 0da2e84
2 parents 9008e9f + e0ce657
commit 0da2e84
Show file tree

Hide file tree

Showing 27 changed files with 659 additions and 88 deletions.
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
@@ -2,6 +2,13 @@ Changelog
 =========
 
 
+v0.9.1
+------
+
+- Add --prefer-source option, to prefer source packages over binary ones
+  if no source distribution is available then binary distributions are used.
+
+
 v0.9.0
 ------
 

diff --git a/docs/source/dependencies-design.rst b/docs/source/dependencies-design.rst
@@ -223,6 +223,9 @@ repositories.
    PyPI "simple" API. Both the "simple" API and the PyPI JSON
    "warehouse-style" API are supported.
 
+- ``--prefer-source``: when set, prefer source distribution instead
+  of binary distribution. In case there is no source distribution
+  available, the tool should provide binary distribution.
 
 Strategy and error processing:
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

diff --git a/src/python_inspector/api.py b/src/python_inspector/api.py
@@ -77,6 +77,7 @@ def resolve_dependencies(
     use_pypi_json_api=False,
     verbose=False,
     analyze_setup_py_insecurely=False,
+    prefer_source=False,
     printer=print,
 ):
     """
@@ -249,7 +250,9 @@ def resolve_dependencies(
             [
                 pkg.to_dict()
                 for pkg in list(
-                    get_pypi_data_from_purl(package, repos=repos, environment=environment)
+                    get_pypi_data_from_purl(
+                        package, repos=repos, environment=environment, prefer_source=prefer_source
+                    )
                 )
             ],
         )

diff --git a/src/python_inspector/package_data.py b/src/python_inspector/package_data.py
@@ -25,14 +25,16 @@
 
 
 def get_pypi_data_from_purl(
-    purl: str, environment: Environment, repos: List[PypiSimpleRepository]
+    purl: str, environment: Environment, repos: List[PypiSimpleRepository], prefer_source: bool
 ) -> PackageData:
     """
     Generate `Package` object from the `purl` string of pypi type
 
     ``purl`` is a package-url of pypi type
     ``environment`` is a `Environment` object defaulting Python version 3.8 and linux OS
     ``repos`` is a list of `PypiSimpleRepository` objects
+    ``prefer_source`` is a boolean value to prefer source distribution over wheel,
+    if no source distribution is available then wheel is used
     """
     purl = PackageURL.from_string(purl)
     name = purl.name
@@ -53,23 +55,29 @@ def get_pypi_data_from_purl(
     bug_tracking_url = get_pypi_bugtracker_url(project_urls)
     python_version = get_python_version_from_env_tag(python_version=environment.python_version)
     valid_distribution_urls = []
-    valid_distribution_urls.extend(
-        list(
-            get_wheel_download_urls(
-                purl=purl,
-                repos=repos,
-                environment=environment,
-                python_version=python_version,
-            )
-        )
-    )
+
     valid_distribution_urls.append(
         get_sdist_download_url(
             purl=purl,
             repos=repos,
             python_version=python_version,
         )
     )
+
+    # if prefer_source is True then only source distribution is used
+    # in case of no source distribution available then wheel is used
+    if not valid_distribution_urls or not prefer_source:
+        valid_distribution_urls.extend(
+            list(
+                get_wheel_download_urls(
+                    purl=purl,
+                    repos=repos,
+                    environment=environment,
+                    python_version=python_version,
+                )
+            )
+        )
+
     urls = response.get("urls") or []
     for url in urls:
         dist_url = url.get("url")

diff --git a/src/python_inspector/resolve_cli.py b/src/python_inspector/resolve_cli.py
@@ -20,7 +20,7 @@
 
 TRACE = False
 
-__version__ = "0.9.0"
+__version__ = "0.9.1"
 
 DEFAULT_PYTHON_VERSION = "38"
 PYPI_SIMPLE_URL = "https://pypi.org/simple"
@@ -151,6 +151,12 @@ def print_version(ctx, param, value):
     help="Enable collection of requirements in setup.py that compute these"
     " dynamically. This is an insecure operation as it can run arbitrary code.",
 )
+@click.option(
+    "--prefer-source",
+    is_flag=True,
+    help="Prefer source distributions over binary distributions"
+    " if no source distribution is available then binary distributions are used",
+)
 @click.option(
     "--verbose",
     is_flag=True,
@@ -182,6 +188,7 @@ def resolve_dependencies(
     use_cached_index=False,
     use_pypi_json_api=False,
     analyze_setup_py_insecurely=False,
+    prefer_source=False,
     verbose=TRACE,
 ):
     """
@@ -194,7 +201,10 @@ def resolve_dependencies(
     linux OS.
 
     Download from the provided PyPI simple --index-url INDEX(s) URLs defaulting
-    to PyPI.org
+    to PyPI.org.
+
+    Provide source distributions over binary distributions with the --prefer-source
+    option. If no source distribution is available then binary distributions are used.
 
     Error and progress are printed to stderr.
 
@@ -250,6 +260,7 @@ def resolve_dependencies(
             verbose=verbose,
             analyze_setup_py_insecurely=analyze_setup_py_insecurely,
             printer=click.secho,
+            prefer_source=prefer_source,
         )
         output = dict(
             headers=headers,

diff --git a/src/python_inspector/utils_pypi.py b/src/python_inspector/utils_pypi.py
@@ -336,7 +336,7 @@ def download_sdist(
     fetched_sdist_filename = None
 
     for repo in repos:
-        sdist = get_valid_sdist(repo, name, version, python_version=DEFAULT_PYTHON_VERSION)
+        sdist = get_valid_sdist(repo, name, version, python_version=python_version)
         if not sdist:
             if TRACE_DEEP:
                 print(f"    download_sdist: No valid sdist for {name}=={version}")

diff --git a/tests/data/azure-devops.req-310-expected.json b/tests/data/azure-devops.req-310-expected.json
@@ -2,7 +2,7 @@
   "headers": {
     "tool_name": "python-inspector",
     "tool_homepageurl": "https://github.com/nexB/python-inspector",
-    "tool_version": "0.9.0",
+    "tool_version": "0.9.1",
     "options": [
       "--requirement /home/tg1999/Desktop/python-inspector-1/tests/data/azure-devops.req.txt",
       "--index-url https://pypi.org/simple",
@@ -945,12 +945,12 @@
       "type": "pypi",
       "namespace": null,
       "name": "cryptography",
-      "version": "38.0.2",
+      "version": "38.0.3",
       "qualifiers": {},
       "subpath": null,
       "primary_language": "Python",
       "description": "cryptography is a package which provides cryptographic recipes and primitives to Python developers.\npyca/cryptography\n=================\n\n.. image:: https://img.shields.io/pypi/v/cryptography.svg\n    :target: https://pypi.org/project/cryptography/\n    :alt: Latest Version\n\n.. image:: https://readthedocs.org/projects/cryptography/badge/?version=latest\n    :target: https://cryptography.io\n    :alt: Latest Docs\n\n.. image:: https://github.com/pyca/cryptography/workflows/CI/badge.svg?branch=main\n    :target: https://github.com/pyca/cryptography/actions?query=workflow%3ACI+branch%3Amain\n\n\n``cryptography`` is a package which provides cryptographic recipes and\nprimitives to Python developers.  Our goal is for it to be your \"cryptographic\nstandard library\". It supports Python 3.6+ and PyPy3 7.2+.\n\n``cryptography`` includes both high level recipes and low level interfaces to\ncommon cryptographic algorithms such as symmetric ciphers, message digests, and\nkey derivation functions. For example, to encrypt something with\n``cryptography``'s high level symmetric encryption recipe:\n\n.. code-block:: pycon\n\n    >>> from cryptography.fernet import Fernet\n    >>> # Put this somewhere safe!\n    >>> key = Fernet.generate_key()\n    >>> f = Fernet(key)\n    >>> token = f.encrypt(b\"A really secret message. Not for prying eyes.\")\n    >>> token\n    '...'\n    >>> f.decrypt(token)\n    'A really secret message. Not for prying eyes.'\n\nYou can find more information in the `documentation`_.\n\nYou can install ``cryptography`` with:\n\n.. code-block:: console\n\n    $ pip install cryptography\n\nFor full details see `the installation documentation`_.\n\nDiscussion\n~~~~~~~~~~\n\nIf you run into bugs, you can file them in our `issue tracker`_.\n\nWe maintain a `cryptography-dev`_ mailing list for development discussion.\n\nYou can also join ``#pyca`` on ``irc.libera.chat`` to ask questions or get\ninvolved.\n\nSecurity\n~~~~~~~~\n\nNeed to report a security issue? Please consult our `security reporting`_\ndocumentation.\n\n\n.. _`documentation`: https://cryptography.io/\n.. _`the installation documentation`: https://cryptography.io/en/latest/installation/\n.. _`issue tracker`: https://github.com/pyca/cryptography/issues\n.. _`cryptography-dev`: https://mail.python.org/mailman/listinfo/cryptography-dev\n.. _`security reporting`: https://cryptography.io/en/latest/security/",
-      "release_date": "2022-10-11T18:57:26",
+      "release_date": "2022-11-01T21:48:36",
       "parties": [
         {
           "type": "person",
@@ -982,11 +982,11 @@
         "Topic :: Security :: Cryptography"
       ],
       "homepage_url": "https://github.com/pyca/cryptography",
-      "download_url": "https://files.pythonhosted.org/packages/92/3d/6f9b9f562c2cc7ff4985bc18822308edbf546de1475563ad51410874c7e3/cryptography-38.0.2-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl",
-      "size": 4151333,
+      "download_url": "https://files.pythonhosted.org/packages/bd/b4/2f8532124bda7470af31b6d9322b5bbb74e3bde94030f9b3a88450f12c8e/cryptography-38.0.3-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl",
+      "size": 4151214,
       "sha1": null,
-      "md5": "78cc9f5b632f9a44b51566366ed5e37c",
-      "sha256": "9b99713109d76ad35736dcc4e47d54fbaa36cce761adc0333db75e86621fa68c",
+      "md5": "5f979c7f0729477e2fb830f8fe525799",
+      "sha256": "b1b52c9e5f8aa2b802d48bd693190341fae201ea51c7a167d69fc48b60e8a959",
       "sha512": null,
       "bug_tracking_url": null,
       "code_view_url": "https://github.com/pyca/cryptography/",
@@ -1007,20 +1007,20 @@
       "dependencies": [],
       "repository_homepage_url": null,
       "repository_download_url": null,
-      "api_data_url": "https://pypi.org/pypi/cryptography/38.0.2/json",
+      "api_data_url": "https://pypi.org/pypi/cryptography/38.0.3/json",
       "datasource_id": null,
-      "purl": "pkg:pypi/[email protected].2"
+      "purl": "pkg:pypi/[email protected].3"
     },
     {
       "type": "pypi",
       "namespace": null,
       "name": "cryptography",
-      "version": "38.0.2",
+      "version": "38.0.3",
       "qualifiers": {},
       "subpath": null,
       "primary_language": "Python",
       "description": "cryptography is a package which provides cryptographic recipes and primitives to Python developers.\npyca/cryptography\n=================\n\n.. image:: https://img.shields.io/pypi/v/cryptography.svg\n    :target: https://pypi.org/project/cryptography/\n    :alt: Latest Version\n\n.. image:: https://readthedocs.org/projects/cryptography/badge/?version=latest\n    :target: https://cryptography.io\n    :alt: Latest Docs\n\n.. image:: https://github.com/pyca/cryptography/workflows/CI/badge.svg?branch=main\n    :target: https://github.com/pyca/cryptography/actions?query=workflow%3ACI+branch%3Amain\n\n\n``cryptography`` is a package which provides cryptographic recipes and\nprimitives to Python developers.  Our goal is for it to be your \"cryptographic\nstandard library\". It supports Python 3.6+ and PyPy3 7.2+.\n\n``cryptography`` includes both high level recipes and low level interfaces to\ncommon cryptographic algorithms such as symmetric ciphers, message digests, and\nkey derivation functions. For example, to encrypt something with\n``cryptography``'s high level symmetric encryption recipe:\n\n.. code-block:: pycon\n\n    >>> from cryptography.fernet import Fernet\n    >>> # Put this somewhere safe!\n    >>> key = Fernet.generate_key()\n    >>> f = Fernet(key)\n    >>> token = f.encrypt(b\"A really secret message. Not for prying eyes.\")\n    >>> token\n    '...'\n    >>> f.decrypt(token)\n    'A really secret message. Not for prying eyes.'\n\nYou can find more information in the `documentation`_.\n\nYou can install ``cryptography`` with:\n\n.. code-block:: console\n\n    $ pip install cryptography\n\nFor full details see `the installation documentation`_.\n\nDiscussion\n~~~~~~~~~~\n\nIf you run into bugs, you can file them in our `issue tracker`_.\n\nWe maintain a `cryptography-dev`_ mailing list for development discussion.\n\nYou can also join ``#pyca`` on ``irc.libera.chat`` to ask questions or get\ninvolved.\n\nSecurity\n~~~~~~~~\n\nNeed to report a security issue? Please consult our `security reporting`_\ndocumentation.\n\n\n.. _`documentation`: https://cryptography.io/\n.. _`the installation documentation`: https://cryptography.io/en/latest/installation/\n.. _`issue tracker`: https://github.com/pyca/cryptography/issues\n.. _`cryptography-dev`: https://mail.python.org/mailman/listinfo/cryptography-dev\n.. _`security reporting`: https://cryptography.io/en/latest/security/",
-      "release_date": "2022-10-11T18:59:25",
+      "release_date": "2022-11-01T21:53:57",
       "parties": [
         {
           "type": "person",
@@ -1052,11 +1052,11 @@
         "Topic :: Security :: Cryptography"
       ],
       "homepage_url": "https://github.com/pyca/cryptography",
-      "download_url": "https://files.pythonhosted.org/packages/63/82/a6e21842f2e31b3874f01c112093b8bf8af119f5ed999bbd667a81de720b/cryptography-38.0.2.tar.gz",
-      "size": 599757,
+      "download_url": "https://files.pythonhosted.org/packages/13/dd/a9608b7aebe5d2dc0c98a4b2090a6b815628efa46cc1c046b89d8cd25f4c/cryptography-38.0.3.tar.gz",
+      "size": 599876,
       "sha1": null,
-      "md5": "5560580a72fe2d7a1731a84ee191dd1f",
-      "sha256": "7a022ec87c7a8bdad99f516a4ee6ffcb3a2bc31487577f9eccbc9b2edb1a8fd4",
+      "md5": "2148f1283f22df0677e204e46bccaf06",
+      "sha256": "bfbe6ee19615b07a98b1d2287d6a6073f734735b49ee45b11324d85efc4d5cbd",
       "sha512": null,
       "bug_tracking_url": null,
       "code_view_url": "https://github.com/pyca/cryptography/",
@@ -1077,9 +1077,9 @@
       "dependencies": [],
       "repository_homepage_url": null,
       "repository_download_url": null,
-      "api_data_url": "https://pypi.org/pypi/cryptography/38.0.2/json",
+      "api_data_url": "https://pypi.org/pypi/cryptography/38.0.3/json",
       "datasource_id": null,
-      "purl": "pkg:pypi/[email protected].2"
+      "purl": "pkg:pypi/[email protected].3"
     },
     {
       "type": "pypi",
@@ -2419,7 +2419,7 @@
       "package": "pkg:pypi/[email protected]",
       "dependencies": [
         "pkg:pypi/[email protected]",
-        "pkg:pypi/[email protected].2",
+        "pkg:pypi/[email protected].3",
         "pkg:pypi/[email protected]"
       ]
     },
@@ -2442,7 +2442,7 @@
       "dependencies": []
     },
     {
-      "package": "pkg:pypi/[email protected].2",
+      "package": "pkg:pypi/[email protected].3",
       "dependencies": [
         "pkg:pypi/[email protected]"
       ]