Print the dependencies of each package in comment (#128)

Signed-off-by: Jiyeong Seok <[email protected]>

src/fosslight_dependency/_package_manager.py

@@ -11,7 +11,6 @@
 import base64
 import subprocess
 import shutil
-import copy
 import fosslight_util.constant as constant
 import fosslight_dependency.constant as const
 
@@ -47,6 +46,8 @@ def __init__(self, package_manager_name, dn_url, input_dir, output_dir):
  self.output_dir = output_dir
  self.dn_url = dn_url
  self.manifest_file_name = []
+ self.relation_tree = {}
+ self.package_name = ''
 
  self.platform = platform.system()
  self.license_scanner_bin = check_license_scanner(self.platform)
@@ -61,6 +62,8 @@ def __del__(self):
  self.output_dir = ''
  self.dn_url = ''
  self.manifest_file_name = []
+ self.relation_tree = {}
+ self.package_name = ''
 
  def run_plugin(self):
  if self.package_manager_name == const.GRADLE or self.package_manager_name == const.ANDROID:
@@ -87,23 +90,20 @@ def run_gradle_task(self):
 
  shutil.copy(const.SUPPORT_PACKAE.get(self.package_manager_name), gradle_backup)
  ret = self.add_allDeps_in_gradle()
- if not ret:
- return
-
- if os.path.isfile('gradlew') or os.path.isfile('gradlew.bat'):
- if self.platform == const.WINDOWS:
- cmd_gradle = "gradlew.bat"
- else:
- cmd_gradle = "./gradlew"
- else:
- return 1
- cmd = f"{cmd_gradle} allDeps"
- ret = subprocess.check_output(cmd, shell=True, encoding='utf-8')
- if ret != 0:
- self.parse_dependency_tree(ret)
- else:
- self.set_direct_dependencies(False)
- logger.warning("Failed to run allDeps task.")
+ if ret:
+ if os.path.isfile('gradlew') or os.path.isfile('gradlew.bat'):
+ if self.platform == const.WINDOWS:
+ cmd_gradle = "gradlew.bat"
+ else:
+ cmd_gradle = "./gradlew"
+
+ cmd = f"{cmd_gradle} allDeps"
+ ret = subprocess.check_output(cmd, shell=True, encoding='utf-8')
+ if ret != 0:
+ self.parse_dependency_tree(ret)
+ else:
+ self.set_direct_dependencies(False)
+ logger.warning("Failed to run allDeps task.")
 
  if os.path.isfile(gradle_backup):
  os.remove(const.SUPPORT_PACKAE.get(self.package_manager_name))
@@ -132,26 +132,55 @@ def add_allDeps_in_gradle(self):
 
  return ret
 
- def parse_dependency_tree(self, dependency_tree_fname):
- config = android_config if self.package_manager_name == 'android' else gradle_config
+ def create_dep_stack(self, dep_line, config):
  packages_in_config = False
- for line in dependency_tree_fname.split('\n'):
+ dep_stack = []
+ cur_flag = ''
+ dep_level = -1
+ dep_level_plus = False
+ for line in dep_line.split('\n'):
  try:
- line_bk = copy.deepcopy(line)
  if not packages_in_config:
  filtered = next(filter(lambda c: re.findall(rf'^{c}\s\-', line), config), None)
  if filtered:
  packages_in_config = True
  else:
  if line == '':
  packages_in_config = False
- re_result = re.findall(r'\-\-\-\s([^\:\s]+\:[^\:\s]+)\:([^\:\s]+)', line)
+ prev_flag = cur_flag
+ prev_dep_level = dep_level
+ dep_level = line.count("|")
+
+ re_result = re.findall(r'([\+|\\])\-\-\-\s([^\:\s]+\:[^\:\s]+)\:([^\:\s]+)', line)
  if re_result:
- self.total_dep_list.append(re_result[0][0])
- if re.match(r'^[\+|\\]\-\-\-\s([^\:\s]+\:[^\:\s]+)\:([^\:\s]+)', line_bk):
- self.direct_dep_list.append(re_result[0][0])
+ cur_flag = re_result[0][0]
+ if (prev_flag == '\\') and (prev_dep_level == dep_level):
+ dep_level_plus = True
+ if dep_level_plus and (prev_flag == '\\') and (prev_dep_level != dep_level):
+ dep_level_plus = False
+ if dep_level_plus:
+ dep_level += 1
+ dep_name = f'{re_result[0][1]}({re_result[0][2]})'
+ dep_stack = dep_stack[:dep_level] + [dep_name]
+ yield dep_stack[:dep_level], dep_name
+ else:
+ cur_flag = ''
  except Exception as e:
- logger.error(f"Failed to parse dependency tree: {e}")
+ logger.warning(f"Failed to parse dependency tree: {e}")
+
+ def parse_dependency_tree(self, f_name):
+ config = android_config if self.package_manager_name == 'android' else gradle_config
+ try:
+ for stack, name in self.create_dep_stack(f_name, config):
+ self.total_dep_list.append(name)
+ if len(stack) == 0:
+ self.direct_dep_list.append(name)
+ else:
+ if stack[-1] not in self.relation_tree:
+ self.relation_tree[stack[-1]] = []
+ self.relation_tree[stack[-1]].append(name)
+ except Exception as e:
+ logger.warning(f'Fail to parse gradle dependency tree:{e}')
 
 
 def version_refine(oss_version):

src/fosslight_dependency/package_manager/Android.py

@@ -50,15 +50,22 @@ def parse_oss_information(self, f_name):
  else:
  continue
 
- if self.total_dep_list:
- if oss_name not in self.total_dep_list:
- continue
-
+ comment_list = []
  if self.direct_dep:
- if oss_name in self.direct_dep_list:
- comment = 'direct'
+ dep_key = f"{oss_name}({oss_version})"
+ if self.total_dep_list:
+ if dep_key not in self.total_dep_list:
+ continue
+ if dep_key in self.direct_dep_list:
+ comment_list.append('direct')
  else:
- comment = 'transitive'
+ comment_list.append('transitive')
+ try:
+ if dep_key in self.relation_tree:
+ comment_list.extend(self.relation_tree[dep_key])
+ except Exception as e:
+ logger.error(f"Fail to find oss scope in dependency tree: {e}")
+ comment = ', '.join(comment_list)
 
  sheet_list.append([manifest_file, oss_name, oss_version, license_name, dn_loc, homepage, '', '', comment])
 

src/fosslight_dependency/package_manager/Gradle.py

@@ -51,8 +51,9 @@ def parse_oss_information(self, f_name):
  oss_name, oss_ini_version = parse_oss_name_version_in_filename(filename)
  used_filename = True
 
+ dep_key = f"{oss_name}({oss_ini_version})"
  if self.total_dep_list:
- if oss_name not in self.total_dep_list:
+ if dep_key not in self.total_dep_list:
  continue
 
  oss_version = version_refine(oss_ini_version)
@@ -73,11 +74,19 @@ def parse_oss_information(self, f_name):
  dn_loc = f"{self.dn_url}{group_id}/{artifact_id}/{oss_ini_version}"
  homepage = f"{self.dn_url}{group_id}/{artifact_id}"
 
+ comment_list = []
  if self.direct_dep:
- if oss_name in self.direct_dep_list:
- comment = 'direct'
- else:
- comment = 'transitive'
+ if len(self.direct_dep_list) > 0:
+ if dep_key in self.direct_dep_list:
+ comment_list.append('direct')
+ else:
+ comment_list.append('transitive')
+ try:
+ if dep_key in self.relation_tree:
+ comment_list.extend(self.relation_tree[dep_key])
+ except Exception as e:
+ logger.error(f"Fail to find oss scope in dependency tree: {e}")
+ comment = ', '.join(comment_list)
 
  sheet_list.append([const.SUPPORT_PACKAE.get(self.package_manager_name),
  oss_name, oss_version, license_name, dn_loc, homepage, '', '', comment])

src/fosslight_dependency/package_manager/Maven.py

@@ -10,7 +10,6 @@
 from bs4 import BeautifulSoup as bs
 from xml.etree.ElementTree import parse
 import re
-import copy
 import fosslight_util.constant as constant
 import fosslight_dependency.constant as const
 from fosslight_dependency._package_manager import PackageManager
@@ -26,12 +25,10 @@ class Maven(PackageManager):
  input_file_name = os.path.join('target', 'generated-resources', 'licenses.xml')
  is_run_plugin = False
  output_custom_dir = ''
- dependency_tree = {}
 
  def __init__(self, input_dir, output_dir, output_custom_dir):
  super().__init__(self.package_manager_name, self.dn_url, input_dir, output_dir)
  self.is_run_plugin = False
- self.dependency_tree = {}
 
  if output_custom_dir:
  self.output_custom_dir = output_custom_dir
@@ -154,26 +151,60 @@ def run_maven_plugin(self):
  ret = subprocess.call(cmd, shell=True)
  if ret != 0:
  logger.error(f"Failed to run: {cmd}")
- self.set_direct_dependencies(True)
+ self.set_direct_dependencies(False)
  else:
  self.parse_dependency_tree(dependency_tree_fname)
  self.set_direct_dependencies(True)
  os.remove(dependency_tree_fname)
 
+ def create_dep_stack(self, dep_line):
+ dep_stack = []
+ cur_flag = ''
+ dep_level = -1
+ dep_level_plus = False
+ for line in dep_line.readlines():
+ try:
+ if not line.startswith('[INFO]'):
+ continue
+ if len(line) <= 7:
+ continue
+ line = line[7:]
+
+ prev_flag = cur_flag
+ prev_dep_level = dep_level
+ dep_level = line.count("|")
+
+ re_result = re.findall(r'([\+|\\]\-)\s([^\:\s]+\:[^\:\s]+)\:(?:[^\:\s]+)\:([^\:\s]+)\:([^\:\s]+)', line)
+ if re_result:
+ cur_flag = re_result[0][0]
+ if (prev_flag == '\\-') and (prev_dep_level == dep_level):
+ dep_level_plus = True
+ if dep_level_plus and (prev_flag == '\\-') and (prev_dep_level != dep_level):
+ dep_level_plus = False
+ if dep_level_plus:
+ dep_level += 1
+ if re_result[0][3] == 'test':
+ continue
+ dep_name = f'{re_result[0][1]}({re_result[0][2]})'
+ dep_stack = dep_stack[:dep_level] + [dep_name]
+ yield dep_stack[:dep_level], dep_name
+ else:
+ cur_flag = ''
+ except Exception as e:
+ logger.warning(f"Failed to parse dependency tree: {e}")
+
  def parse_dependency_tree(self, f_name):
  with open(f_name, 'r', encoding='utf8') as input_fp:
- for i, line in enumerate(input_fp.readlines()):
- try:
- line_bk = copy.deepcopy(line)
- re_result = re.findall(r'[\+|\\]\-\s([^\:\s]+\:[^\:\s]+)\:(?:[^\:\s]+)\:([^\:\s]+)\:([^\:\s]+)', line)
- if re_result:
- dependency_key = re_result[0][0] + ':' + re_result[0][1]
- if self.direct_dep:
- if re.match(r'^\[\w+\]\s[\+\\]\-', line_bk):
- self.direct_dep_list.append(re_result[0][0])
- self.dependency_tree[dependency_key] = re_result[0][2]
- except Exception as e:
- logger.error(f"Failed to parse dependency tree: {e}")
+ try:
+ for stack, name in self.create_dep_stack(input_fp):
+ if len(stack) == 0:
+ self.direct_dep_list.append(name)
+ else:
+ if stack[-1] not in self.relation_tree:
+ self.relation_tree[stack[-1]] = []
+ self.relation_tree[stack[-1]].append(name)
+ except Exception as e:
+ logger.warning(f'Fail to parse maven dependency tree:{e}')
 
  def parse_oss_information(self, f_name):
  with open(f_name, 'r', encoding='utf8') as input_fp:
@@ -183,6 +214,7 @@ def parse_oss_information(self, f_name):
  dependencies = root.find("dependencies")
 
  sheet_list = []
+ comment = ''
 
  for d in dependencies.iter("dependency"):
  groupid = d.findtext("groupId")
@@ -205,20 +237,18 @@ def parse_oss_information(self, f_name):
  # Case that doesn't include License tag value.
  license_name = ''
 
+ dep_key = f"{oss_name}({version})"
  comment_list = []
- try:
- dependency_tree_key = f"{oss_name}:{version}"
- if dependency_tree_key in self.dependency_tree.keys():
- comment_list.append(self.dependency_tree[dependency_tree_key])
- except Exception as e:
- logger.error(f"Fail to find oss scope in dependency tree: {e}")
-
  if self.direct_dep:
- if oss_name in self.direct_dep_list:
+ if dep_key in self.direct_dep_list:
  comment_list.append('direct')
  else:
  comment_list.append('transitive')
-
+ try:
+ if dep_key in self.relation_tree:
+ comment_list.extend(self.relation_tree[dep_key])
+ except Exception as e:
+ logger.error(f"Fail to find oss scope in dependency tree: {e}")
  comment = ', '.join(comment_list)
 
  sheet_list.append([const.SUPPORT_PACKAE.get(self.package_manager_name),