Add parser for swift-show-dependencies.deplock (#3829)

* Add parser for swift-show-dependencies.deplock * Add package assembly for swift-show-dependencies.deplock Reference: Reference: aboutcode-org/scancode.io#1278 Signed-off-by: Keshav Priyadarshi <[email protected]>
aboutcode-org · Jun 28, 2024 · 60b0e0f · 60b0e0f
1 parent 89f8851
commit 60b0e0f
Show file tree

Hide file tree

Showing 10 changed files with 1,735 additions and 31 deletions.
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
@@ -42,6 +42,10 @@ v33.0.0 (next next, roadmap)
   from cocoapods lockfile `Podfile.lock`.
   See https://github.com/nexB/scancode-toolkit/pull/3827
 
+- Add support for parsing packages and dependency relationships
+  from swift `swift-show-dependencies.deplock` generated by DepLock.
+  See https://github.com/nexB/scancode-toolkit/pull/3829
+
 v32.2.0 - 2024-06-19
 ----------------------
 

diff --git a/docs/source/reference/available_package_parsers.rst b/docs/source/reference/available_package_parsers.rst
@@ -766,8 +766,9 @@ parsers in scancode-toolkit during documentation builds.
      - ``squashfs_disk_image``
      - None
      - https://en.wikipedia.org/wiki/SquashFS
-   * - JSON dump of Package.swift created with ``swift package dump-package &gt; Package.swift.json``
+   * - JSON dump of Package.swift created by DepLock or with ``swift package dump-package &gt; Package.swift.json``
      - ``*/Package.swift.json``
+       ``*/Package.swift.deplock``
      - ``swift``
      - ``swift_package_manifest_json``
      - Swift
@@ -779,6 +780,12 @@ parsers in scancode-toolkit during documentation builds.
      - ``swift_package_resolved``
      - swift
      - https://docs.swift.org/package-manager/PackageDescription/PackageDescription.html#package-dependency
+   * - Swift dependency graph created by DepLock
+     - ``*/swift-show-dependencies.deplock``
+     - ``swift``
+     - ``swift_package_show_dependencies``
+     - Swift
+     - https://forums.swift.org/t/swiftpm-show-dependencies-without-fetching-dependencies/51154
    * - Java Web Application Archive
      - ``*.war``
      - ``war``

diff --git a/src/packagedcode/__init__.py b/src/packagedcode/__init__.py
@@ -203,6 +203,7 @@
 
     swift.SwiftManifestJsonHandler,
     swift.SwiftPackageResolvedHandler,
+    swift.SwiftShowDependenciesDepLockHandler,
 
     windows.MicrosoftUpdateManifestHandler,
 

diff --git a/src/packagedcode/swift.py b/src/packagedcode/swift.py
@@ -47,12 +47,76 @@ def logger_debug(*args):
         return logger.debug(" ".join(isinstance(a, str) and a or repr(a) for a in args))
 
 
+class SwiftShowDependenciesDepLockHandler(models.DatafileHandler):
+    datasource_id = "swift_package_show_dependencies"
+    path_patterns = ("*/swift-show-dependencies.deplock",)
+    default_package_type = "swift"
+    default_primary_language = "Swift"
+    description = "Swift dependency graph created by DepLock"
+    documentation_url = "https://forums.swift.org/t/swiftpm-show-dependencies-without-fetching-dependencies/51154"
+
+    @classmethod
+    def _parse(cls, swift_dependency_relation, package_only=False):
+
+        if TRACE:
+            logger_debug(
+                f"SwiftShowDependenciesDepLockHandler: deplock: package: {swift_dependency_relation}"
+            )
+
+        dependencies = get_flatten_dependencies(
+            dependency_tree=swift_dependency_relation.get("dependencies")
+        )
+
+        package_data = dict(
+            datasource_id=cls.datasource_id,
+            type=cls.default_package_type,
+            primary_language=cls.default_primary_language,
+            # ``namespace`` is derived from repo URL and same is not available in dependency graph
+            # See related issue: https://github.com/nexB/scancode-toolkit/issues/3793
+            name=swift_dependency_relation.get("name"),
+            dependencies=dependencies,
+        )
+
+        return models.PackageData.from_data(package_data, package_only)
+
+    @classmethod
+    def parse(cls, location, package_only=False):
+        with io.open(location, encoding="utf-8") as loc:
+            swift_dependency_relation = json.load(loc)
+
+        yield cls._parse(swift_dependency_relation, package_only)
+
+    @classmethod
+    def assemble(
+        cls, package_data, resource, codebase, package_adder=models.add_to_package
+    ):
+        siblings = resource.siblings(codebase)
+        swift_manifest_resource = [
+            r
+            for r in siblings
+            if r.name in ("Package.swift.json", "Package.swift.deplock")
+        ]
+
+        # Skip the assembly if the Swift manifest is present.
+        # SwiftManifestJsonHandler's assembly will take care of the
+        # dependencies from swift-show-dependencies.deplock file.
+        if swift_manifest_resource:
+            return []
+
+        yield from super(SwiftShowDependenciesDepLockHandler, cls).assemble(
+            package_data=package_data,
+            resource=resource,
+            codebase=codebase,
+            package_adder=package_adder,
+        )
+
+
 class SwiftManifestJsonHandler(models.DatafileHandler):
     datasource_id = "swift_package_manifest_json"
-    path_patterns = ("*/Package.swift.json",)
+    path_patterns = ("*/Package.swift.json", "*/Package.swift.deplock")
     default_package_type = "swift"
     default_primary_language = "Swift"
-    description = "JSON dump of Package.swift created with ``swift package dump-package > Package.swift.json``"
+    description = "JSON dump of Package.swift created by DepLock or with ``swift package dump-package > Package.swift.json``"
     documentation_url = "https://docs.swift.org/package-manager/PackageDescription/PackageDescription.html"
 
     @classmethod
@@ -98,41 +162,64 @@ def assemble(
         top-level package with resolved dependencies.
         """
         siblings = resource.siblings(codebase)
+        processed_dependencies = []
         swift_resolved_package_resource = [
             r for r in siblings if r.name == "Package.resolved"
         ]
-        dependencies_from_manifest = package_data.dependencies
 
-        processed_dependencies = []
-        if swift_resolved_package_resource:
-            swift_resolved_package_resource = swift_resolved_package_resource[0]
-            swift_resolved_package_data = swift_resolved_package_resource.package_data
+        swift_show_dependencies_resources = [
+            r for r in siblings if r.name == "swift-show-dependencies.deplock"
+        ]
 
-            for package in swift_resolved_package_data:
-                version = package.get("version")
-                name = package.get("name")
+        if swift_show_dependencies_resources:
+            swift_show_dependencies_resource = swift_show_dependencies_resources[0]
+            swift_show_dependencies_package_data = (
+                swift_show_dependencies_resource.package_data
+            )
+
+            # Dependencies from `swift-show-dependencies.deplock` supersede dependencies from other datafiles.
+            processed_dependencies = swift_show_dependencies_package_data[0][
+                "dependencies"
+            ]
+            processed_dependencies = [
+                models.DependentPackage.from_dict(i) for i in processed_dependencies
+            ]
+
+        # Use dependencies from `Package.resolved` when `swift-show-dependencies.deplock` is not present.
+        else:
+            dependencies_from_manifest = package_data.dependencies
 
-                purl = PackageURL(
-                    type=cls.default_package_type, name=name, version=version
+            if swift_resolved_package_resource:
+                swift_resolved_package_resource = swift_resolved_package_resource[0]
+                swift_resolved_package_data = (
+                    swift_resolved_package_resource.package_data
                 )
-                processed_dependencies.append(
-                    models.DependentPackage(
-                        purl=purl.to_string(),
-                        scope="dependencies",
-                        is_runtime=True,
-                        is_optional=False,
-                        is_resolved=True,
-                        extracted_requirement=version,
+
+                for package in swift_resolved_package_data:
+                    version = package.get("version")
+                    name = package.get("name")
+
+                    purl = PackageURL(
+                        type=cls.default_package_type, name=name, version=version
+                    )
+                    processed_dependencies.append(
+                        models.DependentPackage(
+                            purl=purl.to_string(),
+                            scope="dependencies",
+                            is_runtime=True,
+                            is_optional=False,
+                            is_resolved=True,
+                            extracted_requirement=version,
+                        )
                     )
-                )
 
-                for dependency in dependencies_from_manifest[:]:
-                    dependency_purl = PackageURL.from_string(dependency.purl)
+                    for dependency in dependencies_from_manifest[:]:
+                        dependency_purl = PackageURL.from_string(dependency.purl)
 
-                    if dependency_purl.name == name:
-                        dependencies_from_manifest.remove(dependency)
+                        if dependency_purl.name == name:
+                            dependencies_from_manifest.remove(dependency)
 
-        processed_dependencies.extend(dependencies_from_manifest)
+            processed_dependencies.extend(dependencies_from_manifest)
 
         datafile_path = resource.path
         if package_data.purl:
@@ -141,7 +228,12 @@ def assemble(
                 datafile_path=datafile_path,
             )
 
-            if swift_resolved_package_resource:
+            if swift_show_dependencies_resources:
+                package.datafile_paths.append(swift_show_dependencies_resource.path)
+                package.datasource_ids.append(
+                    SwiftShowDependenciesDepLockHandler.datasource_id
+                )
+            elif swift_resolved_package_resource:
                 package.datafile_paths.append(swift_resolved_package_resource.path)
                 package.datasource_ids.append(SwiftPackageResolvedHandler.datasource_id)
 
@@ -196,7 +288,9 @@ def assemble(
     ):
         siblings = resource.siblings(codebase)
         swift_manifest_resource = [
-            r for r in siblings if r.name == "Package.swift.json"
+            r
+            for r in siblings
+            if r.name in ("Package.swift.json", "Package.swift.deplock")
         ]
 
         # Skip the assembly if the ``Package.swift.json`` manifest is present.
@@ -328,3 +422,109 @@ def get_namespace_and_name(url):
     canonical_name = hostname + path
 
     return canonical_name.rsplit("/", 1)
+
+
+def get_flatten_dependencies(dependency_tree):
+    """
+    Get the list of dependencies from the dependency graph where each
+    element is a DependentPackage containing its 1st order dependencies.
+    """
+    dependencies = []
+    transitive_dependencies = []
+
+    # process direct dependency
+    for dependency in dependency_tree:
+        transitives = dependency.get("dependencies", [])
+        transitive_dependencies.append(transitives)
+        parent_child_dep = get_dependent_package_from_subtree(
+            dependency=dependency,
+            is_top_level_dependency=True,
+        )
+        dependencies.append(parent_child_dep)
+
+    # process all transitive dependencies
+    while transitive_dependencies:
+        transitive_dependency_tree = transitive_dependencies.pop(0)
+        if not transitive_dependency_tree:
+            continue
+
+        for transitive in transitive_dependency_tree:
+            dependencies_of_transitive_dependency = transitive.get("dependencies", [])
+            # add nested dependencies in transitive_dependencies queue for processing
+            transitive_dependencies.append(dependencies_of_transitive_dependency)
+
+            parent_child_dep = get_dependent_package_from_subtree(
+                dependency=transitive,
+                is_top_level_dependency=False,
+            )
+
+            dependencies.append(parent_child_dep)
+
+    return dependencies
+
+
+def get_dependent_package_from_subtree(dependency, is_top_level_dependency):
+    """
+    Get the DependentPackage for a ``dependency`` subtree along with its 1st
+    order dependencies. Set `is_direct` to True if the subtree is a direct
+    dependency for the top-level package.
+    """
+    dependencies_of_parent = []
+    repository_url = dependency.get("url")
+    version = dependency.get("version")
+    transitives = dependency.get("dependencies", [])
+    namespace, name = get_namespace_and_name(repository_url)
+    purl = PackageURL(
+        type="swift",
+        namespace=namespace,
+        name=name,
+        version=version,
+    )
+
+    for transitive in transitives:
+        transitive_repository_url = transitive.get("url")
+        transitive_version = transitive.get("version")
+        transitive_namespace, transitive_name = get_namespace_and_name(
+            transitive_repository_url
+        )
+        transitive_purl = PackageURL(
+            type="swift",
+            namespace=transitive_namespace,
+            name=transitive_name,
+            version=transitive_version,
+        )
+
+        child_dependency = models.DependentPackage(
+            purl=transitive_purl.to_string(),
+            scope="dependencies",
+            extracted_requirement=transitive_version,
+            is_runtime=False,
+            is_optional=False,
+            is_resolved=True,
+            is_direct=True,
+        ).to_dict()
+
+        dependencies_of_parent.append(child_dependency)
+
+    parent_package_data_mapping = dict(
+        datasource_id=SwiftShowDependenciesDepLockHandler.datasource_id,
+        type=SwiftShowDependenciesDepLockHandler.default_package_type,
+        primary_language=SwiftShowDependenciesDepLockHandler.default_primary_language,
+        namespace=namespace,
+        name=name,
+        version=version,
+        dependencies=dependencies_of_parent,
+        is_virtual=True,
+    )
+    parent_dependency = models.PackageData.from_data(parent_package_data_mapping)
+
+    return models.DependentPackage(
+        purl=purl.to_string(),
+        scope="dependencies",
+        extracted_requirement=version,
+        is_runtime=False,
+        is_optional=False,
+        is_resolved=True,
+        is_direct=is_top_level_dependency,
+        resolved_package=parent_dependency,
+    )
diff --git a/tests/packagedcode/data/plugin/help.txt b/tests/packagedcode/data/plugin/help.txt
@@ -836,8 +836,8 @@ Package type:  swift
   datasource_id:     swift_package_manifest_json
   documentation URL: https://docs.swift.org/package-manager/PackageDescription/PackageDescription.html
   primary language:  Swift
-  description:       JSON dump of Package.swift created with ``swift package dump-package > Package.swift.json``
-  path_patterns:    '*/Package.swift.json'
+  description:       JSON dump of Package.swift created by DepLock or with ``swift package dump-package > Package.swift.json``
+  path_patterns:    '*/Package.swift.json', '*/Package.swift.deplock'
 --------------------------------------------
 Package type:  swift
   datasource_id:     swift_package_resolved
@@ -846,6 +846,13 @@ Package type:  swift
   description:       Resolved full dependency lockfile for Package.swift created with ``swift package resolve``
   path_patterns:    '*/Package.resolved', '*/.package.resolved'
 --------------------------------------------
+Package type:  swift
+  datasource_id:     swift_package_show_dependencies
+  documentation URL: https://forums.swift.org/t/swiftpm-show-dependencies-without-fetching-dependencies/51154
+  primary language:  Swift
+  description:       Swift dependency graph created by DepLock
+  path_patterns:    '*/swift-show-dependencies.deplock'
+--------------------------------------------
 Package type:  war
   datasource_id:     java_war_archive
   documentation URL: https://en.wikipedia.org/wiki/WAR_(file_format)