Refactoring oss item (#213)

* Update graph converter to ossitem

---------

Signed-off-by: jiyeong.seok <[email protected]>

requirements.txt

@@ -4,7 +4,7 @@ lxml
 virtualenv
 pyyaml
 lastversion
-fosslight_util~=1.4.47
+fosslight_util>=2.0.0
 PyGithub
 requirements-parser
 defusedxml

src/fosslight_dependency/_analyze_dependency.py

@@ -28,7 +28,7 @@ def analyze_dependency(package_manager_name, input_dir, output_dir, pip_activate
  output_custom_dir='', app_name=const.default_app_name, github_token='', manifest_file_name=[],
  direct=True):
  ret = True
- package_sheet_list = []
+ package_dep_item_list = []
  cover_comment = ''
 
  if package_manager_name == const.PYPI:
@@ -60,7 +60,7 @@ def analyze_dependency(package_manager_name, input_dir, output_dir, pip_activate
  else:
  logger.error(f"Not supported package manager name: {package_manager_name}")
  ret = False
- return ret, package_sheet_list
+ return ret, package_dep_item_list
 
  if manifest_file_name:
  package_manager.set_manifest_file(manifest_file_name)
@@ -76,7 +76,8 @@ def analyze_dependency(package_manager_name, input_dir, output_dir, pip_activate
  logger.info(f"Parse oss information with file: {f_name}")
 
  if os.path.isfile(f_name):
- package_sheet_list.extend(package_manager.parse_oss_information(f_name))
+ package_manager.parse_oss_information(f_name)
+ package_dep_item_list.extend(package_manager.dep_items)
  else:
  logger.error(f"Failed to open input file: {f_name}")
  ret = False
@@ -90,4 +91,4 @@ def analyze_dependency(package_manager_name, input_dir, output_dir, pip_activate
 
  del package_manager
 
- return ret, package_sheet_list, cover_comment
+ return ret, package_dep_item_list, cover_comment

src/fosslight_dependency/_graph_convertor.py

@@ -8,24 +8,23 @@
 
 
 class GraphConvertor:
- def __init__(self, package_list: Optional[list] = None):
+ def __init__(self, dep_items: Optional[list] = None):
  self._verticies = {}
  self._edges = []
- if package_list:
- self.init_list(package_list)
+ if dep_items:
+ self.init_list(dep_items)
 
- def init_list(self, package_list: list):
+ def init_list(self, dep_items: list):
  """
- Initialize package_list to self._verticies and self._edges
+ Initialize dep_items to self._verticies and self._edges
 
  Args:
- package_list (list): List containing package information
+ dep_items : List containing package information
  """
  depend_on_package_dict = {}
- for idx, package_info in enumerate(package_list):
- package_name = package_info[0]
- depend_on_packages_str = package_info[-1]
- depend_on_packages = list(map((lambda x: x.strip()), depend_on_packages_str.split(",")))
+ for idx, file_item in enumerate(dep_items):
+ package_name = file_item.purl
+ depend_on_packages = file_item.depends_on
  self._verticies[package_name] = idx
  depend_on_package_dict[package_name] = depend_on_packages
  else:

src/fosslight_dependency/_package_manager.py

@@ -49,8 +49,8 @@ def __init__(self, package_manager_name, dn_url, input_dir, output_dir):
  self.manifest_file_name = []
  self.relation_tree = {}
  self.package_name = ''
- self.purl_dict = {}
  self.cover_comment = ''
+ self.dep_items = []
 
  self.platform = platform.system()
  self.license_scanner_bin = check_license_scanner(self.platform)
@@ -67,6 +67,7 @@ def __del__(self):
  self.manifest_file_name = []
  self.relation_tree = {}
  self.package_name = ''
+ self.dep_items = []
 
  def run_plugin(self):
  ret = True
@@ -257,18 +258,6 @@ def parse_dependency_tree(self, f_name):
  except Exception as e:
  logger.warning(f'Fail to parse gradle dependency tree:{e}')
 
- def change_dep_to_purl(self, sheet_list):
- for oss_item in sheet_list:
- try:
- if len(oss_item) < 10:
- break
- deps_list = oss_item[9]
- deps_purl = list(filter(None, map(lambda x: self.purl_dict.get(x, ''), deps_list)))
- oss_item[9] = ','.join(deps_purl)
- except Exception as e:
- logger.warning(f'Fail to change depend_on to purl:{e}')
- return sheet_list
-
 
 def get_url_to_purl(url, pkg_manager, oss_name='', oss_version=''):
  purl_prefix = f'pkg:{pkg_manager}'

src/fosslight_dependency/dependency_item.py

@@ -0,0 +1,103 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright (c) 2024 LG Electronics Inc.
+# SPDX-License-Identifier: Apache-2.0
+
+import logging
+from fosslight_util.constant import LOGGER_NAME
+from fosslight_util.oss_item import FileItem
+
+_logger = logging.getLogger(LOGGER_NAME)
+
+
+class DependencyItem(FileItem):
+ def __init__(self):
+ super().__init__("")
+
+ self._depends_on_raw = [] # name(version) format
+ self._depends_on = [] # purl format
+ self.purl = ""
+
+ def __del__(self):
+ pass
+
+ @property
+ def depends_on(self):
+ return self._depends_on
+
+ @depends_on.setter
+ def depends_on(self, value):
+ if not value:
+ self._depends_on = []
+ else:
+ if not isinstance(value, list):
+ value = value.split(",")
+ self._depends_on.extend(value)
+ self._depends_on = [item.strip() for item in self._depends_on]
+ self._depends_on = list(set(self._depends_on))
+
+ @property
+ def depends_on_raw(self):
+ return self._depends_on_raw
+
+ @depends_on_raw.setter
+ def depends_on_raw(self, value):
+ if not value:
+ self._depends_on_raw = []
+ else:
+ if not isinstance(value, list):
+ value = value.split(",")
+ self._depends_on_raw.extend(value)
+ self._depends_on_raw = [item.strip() for item in self._depends_on_raw]
+ self._depends_on_raw = list(set(self._depends_on_raw))
+
+ def get_print_array(self):
+ items = []
+ for oss in self.oss_items:
+ exclude = "Exclude" if self.exclude or oss.exclude else ""
+ lic = ",".join(oss.license)
+ depends_on = ",".join(self.depends_on) if len(self.depends_on) > 0 else ""
+
+ oss_item = [self.purl, oss.name, oss.version, lic, oss.download_location, oss.homepage,
+ oss.copyright, exclude, oss.comment, depends_on]
+ items.append(oss_item)
+
+ return items
+
+ def get_print_json(self):
+ items = []
+ for oss in self.oss_items:
+ json_item = {}
+ json_item["name"] = oss.name
+ json_item["version"] = oss.version
+
+ if self.purl != "":
+ json_item["package url"] = self.purl
+ if len(oss.license) > 0:
+ json_item["license"] = oss.license
+ if oss.download_location != "":
+ json_item["download location"] = oss.download_location
+ if oss.homepage != "":
+ json_item["homepage"] = oss.homepage
+ if oss.copyright != "":
+ json_item["copyright text"] = oss.copyright
+ if self.exclude or oss.exclude:
+ json_item["exclude"] = True
+ if oss.comment != "":
+ json_item["comment"] = oss.comment
+ if len(self.depends_on) > 0:
+ json_item["depends on"] = self.depends_on
+
+ items.append(json_item)
+
+ return items
+
+
+def change_dependson_to_purl(purl_dict, dep_items):
+ for dep_item in dep_items:
+ try:
+ dep_item.depends_on = list(filter(None, map(lambda x: purl_dict.get(x, ''), dep_item.depends_on_raw)))
+
+ except Exception as e:
+ _logger.warning(f'Fail to change depend_on to purl:{e}')
+ return dep_items

src/fosslight_dependency/package_manager/Android.py

@@ -8,6 +8,8 @@
 import fosslight_util.constant as constant
 import fosslight_dependency.constant as const
 from fosslight_dependency._package_manager import PackageManager, get_url_to_purl
+from fosslight_dependency.dependency_item import DependencyItem, change_dependson_to_purl
+from fosslight_util.oss_item import OssItem
 
 logger = logging.getLogger(constant.LOGGER_NAME)
 
@@ -40,43 +42,41 @@ def check_input_path(self):
 
  def parse_oss_information(self, f_name):
  with open(f_name, 'r', encoding='utf8') as input_fp:
- sheet_list = []
-
+ purl_dict = {}
  for i, line in enumerate(input_fp.readlines()):
- comment = ''
+ dep_item = DependencyItem()
+ oss_item = OssItem()
  split_str = line.strip().split("\t")
  if i < 2:
  continue
 
- if len(split_str) == 9:
- idx, manifest_file, oss_name, oss_version, license_name, dn_loc, homepage, NA, NA = split_str
- elif len(split_str) == 7:
- idx, manifest_file, oss_name, oss_version, license_name, dn_loc, homepage = split_str
+ if len(split_str) == 9 or len(split_str) == 7:
+ _, _, oss_item.name, oss_item.version, oss_item.license, \
+ oss_item.download_location, oss_item.homepage = split_str[:7]
  else:
  continue
- purl = get_url_to_purl(dn_loc, 'maven')
- self.purl_dict[f'{oss_name}({oss_version})'] = purl
+ dep_item.purl = get_url_to_purl(oss_item.download_location, 'maven')
+ purl_dict[f'{oss_item.name}({oss_item.version})'] = dep_item.purl
 
- comment_list = []
- deps_list = []
  if self.direct_dep:
- dep_key = f"{oss_name}({oss_version})"
+ dep_key = f"{oss_item.name}({oss_item.version})"
  if self.total_dep_list:
  if dep_key not in self.total_dep_list:
  continue
  if dep_key in self.direct_dep_list:
- comment_list.append('direct')
+ oss_item.comment = 'direct'
  else:
- comment_list.append('transitive')
+ oss_item.comment = 'transitive'
  try:
  if dep_key in self.relation_tree:
- deps_list.extend(self.relation_tree[dep_key])
+ dep_item.depends_on_raw = self.relation_tree[dep_key]
  except Exception as e:
  logger.error(f"Fail to find oss scope in dependency tree: {e}")
- comment = ','.join(comment_list)
 
- sheet_list.append([purl, oss_name, oss_version, license_name, dn_loc, homepage,
- '', '', comment, deps_list])
- sheet_list = self.change_dep_to_purl(sheet_list)
+ dep_item.oss_items.append(oss_item)
+ self.dep_items.append(dep_item)
+
+ if self.direct_dep:
+ self.dep_items = change_dependson_to_purl(purl_dict, self.dep_items)
 
- return sheet_list
+ return

src/fosslight_dependency/package_manager/Carthage.py

@@ -11,6 +11,8 @@
 from fosslight_dependency._package_manager import PackageManager
 from fosslight_dependency._package_manager import connect_github, get_github_license, check_and_run_license_scanner
 from fosslight_dependency._package_manager import get_url_to_purl
+from fosslight_dependency.dependency_item import DependencyItem
+from fosslight_util.oss_item import OssItem
 
 logger = logging.getLogger(constant.LOGGER_NAME)
 
@@ -35,9 +37,8 @@ def __init__(self, input_dir, output_dir, github_token):
  def parse_oss_information(self, f_name):
  github = "github"
  checkout_dir_list = get_checkout_dirname()
- comment = ''
+
  with open(f_name, 'r', encoding='utf8') as input_fp:
- sheet_list = []
  g = ''
  if not checkout_dir_list:
  g = connect_github(self.github_token)
@@ -47,21 +48,24 @@ def parse_oss_information(self, f_name):
  # Ref. https://github.com/Carthage/Carthage/blob/master/Documentation/Artifacts.md
  re_result = re.findall(r'(github|git)[\s]\"(\S*)\"[\s]\"(\S*)\"', line)
  try:
+ dep_item = DependencyItem()
+ oss_item = OssItem()
  repo = re_result[0][0]
  oss_path = re_result[0][1]
  if oss_path.endswith('.git'):
  oss_path = oss_path[:-4]
  oss_origin_name = oss_path.split('/')[-1]
- oss_name = self.package_manager_name + ":" + oss_origin_name
+ oss_item.name = self.package_manager_name + ":" + oss_origin_name
 
  if repo == github:
- homepage = self.dn_url + oss_path
+ oss_item.homepage = self.dn_url + oss_path
  else:
- homepage = oss_path
- dn_loc = homepage
- oss_version = re_result[0][2]
+ oss_item.homepage = oss_path
+ oss_item.download_location = oss_item.homepage
+ oss_item.version = re_result[0][2]
 
- purl = get_url_to_purl(homepage, self.package_manager_name, oss_origin_name, oss_version)
+ dep_item.purl = get_url_to_purl(oss_item.homepage, self.package_manager_name,
+ oss_origin_name, oss_item.version)
 
  license_name = ''
  find_license = False
@@ -89,20 +93,17 @@ def parse_oss_information(self, f_name):
  except Exception as e:
  logger.warning(f"Failed to get license with github api: {e}")
  license_name == ''
-
+ oss_item.license = license_name
  if self.direct_dep_list:
  if oss_origin_name in self.direct_dep_list:
- comment = 'direct'
+ oss_item.comment = 'direct'
  else:
- comment = 'transitive'
-
- sheet_list.append([purl, oss_name, oss_version, license_name, dn_loc, homepage,
- '', '', comment, ''])
-
+ oss_item.comment = 'transitive'
+ dep_item.oss_items.append(oss_item)
+ self.dep_items.append(dep_item)
  except Exception as e:
  logger.warning(f"Failed to parse oss information: {e}")
-
- return sheet_list
+ return
 
  def parse_direct_dependencies(self):
  self.direct_dep = True