diff --git a/easy_annotate.py b/easy_annotate.py index 1631a6b..c571736 100755 --- a/easy_annotate.py +++ b/easy_annotate.py @@ -249,6 +249,9 @@ def main(): parser.add_argument( '-v', '--verbose', dest='verbose', required=False, action='store_true', help='Verbose; print lots of extra stuff, (default: false)') + parser.add_argument( + '--debug', dest='debug', required=False, action='store_true', + help='Debug add debug messages to stderr (default: false)') parser.add_argument('easyconfig', nargs='?') args = parser.parse_args() diff --git a/easy_update.py b/easy_update.py index 4e3ebb9..97af9d2 100755 --- a/easy_update.py +++ b/easy_update.py @@ -13,8 +13,8 @@ sys.stderr.write("ERROR: Python 3 required, found %s\n" % sys.version.split(' ')[0]) sys.exit(1) -__version__ = '2.1.4' -__date__ = 'May 20, 2021' +__version__ = '2.2.0' +__date__ = 'Aug 11, 2021' __maintainer__ = 'John Dey jfdey@fredhutch.org' @@ -27,6 +27,10 @@ """ """ Release Notes +2.2.0 Aug 11, 2020 - Dig deep to find all dependent Python libraries. Every dependency needs to be check + to determine if it contains Python modules. Inspect every dependency for PythonBundle or PythonPackage, + easyblock type. + 2.1.5 July 8, 2021 - fix bug in find_easyconfig_paths Add additonal headers 'SOURCE_WHL', 'SOURCE_PY3_WHL'; from caspar@SURFsara diff --git a/framework.py b/framework.py index 5a858b1..c2644cd 100755 --- a/framework.py +++ b/framework.py @@ -26,7 +26,7 @@ add logging.debug() 1.0.1 Aug, 15, 2019 - fix parse_dependencies + fix search_dependencies For the case of reading Python dependancies, conver the case of 'Biopython-1.74-foss-2016b-Python-3.7.4' Search dependcies for versionsuffix == '-Python-%(pyver)s' @@ -47,7 +47,8 @@ __maintainer__ = 'John Dey jfdey@fredhutch.org' __date__ = 'Nov 21, 2020' -logging.basicConfig(level=logging.WARN) +logging.basicConfig(format='%(levelname)s [%(filename)s:%(lineno)-4d] %(message)s', + level=logging.WARN) class FrameWork: """provide access to EasyBuild Config file variables @@ -57,7 +58,8 @@ class FrameWork: """ def __init__(self, args): if args.debug: - logging.basicConfig(level=logging.DEBUG) + logging.getLogger().setLevel(logging.DEBUG) + logging.debug('debug enabled') self.verbose = args.verbose self.code = None self.pyver = None @@ -71,7 +73,7 @@ def __init__(self, args): self.dep_exts = [] self.find_easyconfig_paths(args.easyconfig) - logging.debug("EB search path: {}".format(self.base_path)) + logging.debug("EasyConfig search paths: {}".format(self.base_paths)) # update EasyConfig exts_list @@ -89,9 +91,7 @@ def __init__(self, args): except AttributeError: self.defaultclass = None self.detect_language(eb) - if not self.lang: - logging.debug('Wow language is unknown!') - self.parse_dependencies(eb, self.lang) + self.search_dependencies(eb, self.lang) try: self.versionsuffix = eb.versionsuffix % self.interpolate except AttributeError: @@ -174,7 +174,7 @@ def detect_language(self, eb): elif self.defaultclass: self.lang = eb.exts_defaultclass.replace('Package', '') else: - logging.warn('can not determin either R or Python?') + logging.warn('Can not determine languge of module. R or Python?') def find_easyconfig_paths(self, filename): """find the paths to EasyConfigs, search within the path given by the easyconfig @@ -183,32 +183,27 @@ def find_easyconfig_paths(self, filename): userPath = os.path.expanduser(filename) fullPath = os.path.abspath(userPath) (head, tail) = os.path.split(fullPath) - self.base_path = None + self.base_paths = [] while tail: if 'easyconfig' in tail: - self.base_path = os.path.join(head, tail) - logging.debug('path to easyconfigs: {}'.format(self.base_path)) - return + self.base_paths.append(os.path.join(head, tail)) + logging.debug('local path to easyconfigs: {}'.format(self.base_paths[-1])) + break (head, tail) = os.path.split(head) - eb_path = shutil.which("eb") + eb_path = os.getenv('EBROOTEASYBUILD') if eb_path: - logging.debug('EB binary path: {}'.format(eb_path)) - ec_path = eb_path.replace('bin/eb', 'easybuild/easyconfigs') - if self.base_path: - self.base_path += ':' + ec_path - else: - self.base_path = ec_path + self.base_paths.append(os.path.join(eb_path, 'easybuild/easyconfigs')) else: sys.stderr.writelines('Could not fine path to EasyBuild, EasyBuild module must be loaded') sys.exit(1) def find_easyconfig(self, name, easyconfigs): - """ search base_path for easyconfig filename """ + """ search base_paths for easyconfig filename """ found = None first_letter = easyconfigs[0][0].lower() - for easyconfig in easyconfigs: - for ec_dir in self.base_path.split(':'): + for easyconfig in easyconfigs: + for ec_dir in self.base_paths: narrow = ec_dir + '/' + first_letter + '/' + name logging.debug('find_easyconfig: search for {} in {} '.format(easyconfig, narrow)) for r,d,f in os.walk(narrow): @@ -258,8 +253,8 @@ def build_dep_filename(self, eb, dep): logging.debug('build_dep_filename {}'.format(dep_filenames)) return dep_filenames - - def parse_dependencies(self, eb, lang): + + def search_dependencies(self, eb, lang): """ inspect dependencies for R and Python easyconfigs, if found add the exts_list to the list of dependent exts @@ -269,35 +264,33 @@ def parse_dependencies(self, eb, lang): except NameError: return None for dep in dependencies: - easyconfig = None - if dep[0] in ['R', 'Python']: - if dep[0] == 'Python': - self.interpolate['pyver'] = dep[1] - self.pyver = dep[1] - logging.debug('primary language Python {}'.format(self.pyver)) - if dep[0] == 'R': - self.interpolate['rver'] = dep[1] - self.rver = dep[1] - logging.debug('primary language R-{}'.format(self.rver)) - dep_filenames = self.build_dep_filename(eb, dep) - logging.debug('language file: {}'.format(dep_filenames)) + if self.pyver is None and dep[0] == 'Python': + self.interpolate['pyver'] = dep[1] + self.pyver = dep[1] + logging.debug('primary language Python {}'.format(self.pyver)) + if self.rver is None and dep[0] == 'R': + self.interpolate['rver'] = dep[1] + self.rver = dep[1] + logging.debug('primary language R-{}'.format(self.rver)) + dep_filenames = self.build_dep_filename(eb, dep) + logging.debug('dependency file names: {}'.format(dep_filenames)) + easyconfig = self.find_easyconfig(dep[0], dep_filenames) + for dep_filename in dep_filenames: easyconfig = self.find_easyconfig(dep[0], dep_filenames) - else: - dep_filenames = self.build_dep_filename(eb, dep) - for dep_filename in dep_filenames: - if 'R-' == dep_filename[0:2] or '-R-' in dep_filename or ( - 'Python-' == dep_filename[0:6] or '-Python-' in dep_filename): - logging.debug('deubug - language dependencie: {}'.format(dep_filename)) - easyconfig = self.find_easyconfig(dep[0], dep_filenames) - break - if easyconfig: - if self.verbose: - print('== reading extensions from: {}'.format(os.path.basename(easyconfig))) - eb = self.parse_eb(str(easyconfig), False) - try: - self.dep_exts.extend(eb.exts_list) - except (AttributeError, NameError): - self.dep_exts.extend([dep]) + if easyconfig: + if self.verbose: + print('== reading extensions from: {}'.format(os.path.basename(easyconfig))) + eb = self.parse_eb(str(easyconfig), False) + if hasattr(eb, 'easyblock') and eb.easyblock in ['PythonBundle', 'PythonPackage']: + self.dep_exts.extend([eb.name, eb.version]) + print('== adding {}'.format(eb.name)) + try: + self.dep_exts.extend(eb.exts_list) + except (AttributeError, NameError): + self.dep_exts.extend([dep]) + break + else: + print('== Not Found: {}'.format(dep_filename)) def parse_python_deps(self, eb):