diff --git a/easybuild/tools/module_naming_scheme/generation_mns.py b/easybuild/tools/module_naming_scheme/generation_mns.py index 06a414e4aa..35e3a0f620 100644 --- a/easybuild/tools/module_naming_scheme/generation_mns.py +++ b/easybuild/tools/module_naming_scheme/generation_mns.py @@ -25,34 +25,91 @@ """ Implementation of a different generation specific module naming scheme using release dates. :author: Thomas Eylenbosch (Gluo N.V.) +:author: Thomas Soenen (B-square IT services) """ import os import json -from pkgutil import get_data from easybuild.tools.module_naming_scheme.mns import ModuleNamingScheme -# from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version +from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.robot import search_easyconfigs +from easybuild.framework.easyconfig.easyconfig import get_toolchain_hierarchy DUMMY_TOOLCHAIN_NAME = 'dummy' -DUMMY_TOOLCHAIN_VERSION = 'dummy' - SYSTEM_TOOLCHAIN_NAME = 'system' -# Lookup table for toolchain versions and generations -GENERATION_LOOKUP = json.loads(get_data(__package__, 'generation_lookup_table.json')) +GMNS_ENV = "GENERATION_MODULE_NAMING_SCHEME_LOOKUP_TABLE" +GMNS_PATH = os.environ.get(GMNS_ENV) class GenerationModuleNamingScheme(ModuleNamingScheme): - """Class implementing the categorized module naming scheme.""" + """Class implementing the generational module naming scheme.""" REQUIRED_KEYS = ['name', 'version', 'versionsuffix', 'toolchain'] - def det_full_module_name(self, ec): + def __init__(self): """ - Determine short module name, i.e. the name under which modules will be exposed to users. - Examples: GCC/4.8.3, OpenMPI/1.6.5, OpenBLAS/0.2.9, HPL/2.1, Python/2.7.5 + Generate lookup table that maps toolchains on foss generations. Generations (e.g. 2018a, + 2020b) are fetched from the foss easyconfigs and dynamically mapped on toolchains using + get_toolchain_hierarchy. The lookup table can be extended by the user by providing a file. + + Lookup table is a dict with toolchain-generation key-value pairs:{(GCC, 4.8.2): 2016a}, + with toolchains resembled as a tuple. + + json format of file with custom mappings: + { + "2018b": [{"name": "GCC", "version": "5.2.0"}, {"name": "GCC", "version": "4.8.2"}], + "2019b": [{"name": "GCC", "version": "5.2.4"}, {"name": "GCC", "version": "4.8.4"}], + } """ + super().__init__() + + self.lookup_table = {} + + # Get all generations + foss_filenames = search_easyconfigs("^foss-20[0-9]{2}[a-z]\.eb", + filename_only=True, + print_result=False) + self.generations = [x.split('-')[1].split('.')[0] for x in foss_filenames] + + # map generations on toolchains + for generation in self.generations: + for tc in get_toolchain_hierarchy({'name':'foss', 'version':generation}): + self.lookup_table[(tc['name'], tc['version'])] = generation + # include (foss, ) as a toolchain aswell + self.lookup_table[('foss', generation)] = generation + + # users can provide custom generation-toolchain mapping through a file + if GMNS_PATH: + if not os.path.isfile(GMNS_PATH): + msg = "value of ENV {} ({}) should be a valid filepath" + raise EasyBuildError(msg.format(GMNS_ENV, GMNS_PATH)) + with open(GMNS_PATH, 'r') as hc_lookup: + try: + hc_lookup_data = json.loads(hc_lookup.read()) + except json.decoder.JSONDecodeError: + raise EasyBuildError("{} can't be decoded as json".format(GMNS_PATH)) + if not isinstance(hc_lookup_data, dict): + raise EasyBuildError("{} should contain a dict".format(GMNS_PATH)) + if not set(hc_lookup_data.keys()) <= set(self.generations): + raise EasyBuildError("Keys of {} should be generations".format(GMNS_PATH)) + for generation, toolchains in hc_lookup_data.items(): + if not isinstance(toolchains, list): + raise EasyBuildError("Values of {} should be lists".format(GMNS_PATH)) + for tc in toolchains: + if not isinsance(tc, dict): + msg = "Toolchains in {} should be of type dict" + raise EasyBuildError(msg.format(GMNS_PATH)) + if set(tc.keys()) != set('name', 'version'): + msg = "Toolchains in {} should have two keys ('name', 'version')" + raise EasyBuildError(msg.format(GMNS_PATH)) + self.lookup_table[(tc['name'], tc['version'])] = generation + def det_full_module_name(self, ec): + """ + Determine full module name, relative to the top of the module path. + Examples: General/GCC/4.8.3, Releases/2018b/OpenMPI/1.6.5 + """ return os.path.join(self.det_module_subdir(ec), self.det_short_module_name(ec)) def det_short_module_name(self, ec): @@ -69,22 +126,30 @@ def det_full_version(self, ec): return versionprefix + ec['version'] + ec['versionsuffix'] def det_module_subdir(self, ec): - + """ + Determine subdirectory for module file in $MODULEPATH. This determines the separation + between module names exposed to users, and what's part of the $MODULEPATH. subdirectory + is determined by mapping toolchain on a generation. + """ release = 'releases' release_date = '' if ec['toolchain']['name'] in [DUMMY_TOOLCHAIN_NAME, SYSTEM_TOOLCHAIN_NAME]: release = 'General' - - elif ec['toolchain']['version'] in GENERATION_LOOKUP['releases']: - release_date = ec['toolchain']['version'] - - elif ec['toolchain']['name'] in GENERATION_LOOKUP: - release_date = GENERATION_LOOKUP[ec['toolchain']['name']].get(ec['toolchain']['version'], 'NOTFOUND') - else: - release_date = 'NOTFOUND' - - subdir = os.path.join(release, release_date).rstrip('/') - - return subdir + if self.lookup_table.get((ec['toolchain']['name'], ec['toolchain']['version'])): + release_date = self.lookup_table[(ec['toolchain']['name'], ec['toolchain']['version'])] + else: + tc_hierarchy = get_toolchain_hierarchy({'name': ec['toolchain']['name'], + 'version': ec['toolchain']['version']}) + for tc in tc_hierarchy: + if self.lookup_table.get((tc['name'], tc['version'])): + release_date = self.lookup_table.get((tc['name'], tc['version'])) + break + + if release_date == '': + msg = "Couldn't map software version ({}, {}) to a generation. Provide a custom" \ + "toolchain mapping through {}" + raise EasyBuildError(msg.format(ec['name'], ec['version'], GMNS_ENV)) + + return os.path.join(release, release_date).rstrip('/')