diff --git a/easybuild/tools/module_naming_scheme/generation_lookup_table.json b/easybuild/tools/module_naming_scheme/generation_lookup_table.json new file mode 100644 index 0000000000..c4d376b16e --- /dev/null +++ b/easybuild/tools/module_naming_scheme/generation_lookup_table.json @@ -0,0 +1,371 @@ +{ + "releases": [ + "2014a", + "2014b", + "2015a", + "2015b", + "2016a", + "2016b", + "2017a", + "2017b", + "2018a", + "2018b", + "2019a", + "2019b", + "2020a", + "2020b", + "2021a" + ], + "GCCcore": { + "4.8.3": "2014b", + "4.9.2": "2015a", + "4.9.3": "2016a", + "4.9.4": "TBD", + "5.3.0": "TBD", + "5.4.0": "2016b", + "5.5.0": "TBD", + "6.1.0": "TBD", + "6.2.0": "TBD", + "6.3.0": "2017a", + "6.4.0": "2018a", + "7.1.0": "TBD", + "7.2.0": "TBD", + "7.3.0": "2018b", + "7.4.0": "TBD", + "8.1.0": "TBD", + "8.2.0": "2019a", + "8.3.0": "2019b", + "9.1.0": "TBD", + "9.2.0": "TBD", + "9.3.0": "2020a", + "10.1.0": "TBD", + "10.2.0": "2020b", + "10.3.0": "TBD", + "11.1.0": "TBD" + }, + "gcccuda": { + "2016.08": "TBD", + "2016.10": "TBD", + "2017.01": "TBD", + "2017.02": "TBD" + }, + "GCC": { + "4.9.3-2.25": "2016a", + "4.9.4-2.25": "TBD", + "5.3.0-2.26": "TBD", + "5.4.0-2.26": "2016b", + "6.1.0-2.27": "TBD", + "6.2.0-2.27": "TBD", + "6.3.0-2.27": "2017a", + "6.3.0-2.28": "TBD", + "6.4.0-2.28": "2018a", + "7.2.0-2.29": "TBD", + "7.3.0-2.30": "2018b", + "8.2.0-2.31.1": "2019a", + "8.3.0-2.32": "2019b", + "9.2.0-2.32": "TBD", + "4.9.2-binutils-2.25": "TBD", + "4.9.3-binutils-2.25": "2015b", + "5.1.0-binutils-2.25": "TBD", + "4.6.3": "TBD", + "4.6.4": "TBD", + "4.7.2": "TBD", + "4.7.3": "TBD", + "4.8.1": "TBD", + "4.8.2": "TBD", + "4.8.3": "2014b", + "4.8.4": "TBD", + "4.9.2": "2015a", + "4.9.3": "2015b", + "5.2.0": "TBD", + "8.3.0": "2019b", + "9.2.0": "TBD", + "9.3.0": "2020a", + "10.2.0": "2020b", + "10.3.0": "TBD" + }, + "iccifort": { + "2013_sp1.1.106": "TBD", + "2013_sp1.2.144": "TBD", + "2013_sp1.4.211": "TBD", + "2013.2.146": "TBD", + "2013.3.163": "TBD", + "2013.4.183": "TBD", + "2013.5.192-GCC-4.8.3": "2014b", + "2013.5.192": "2014b", + "2015.0.090": "TBD", + "2015.0.090-GCC-4.9.2": "TBD", + "2015.1.133": "2015a", + "2015.1.133-GCC-4.9.2": "2015a", + "2015.2.164-GCC-4.9.2": "TBD", + "2015.3.187": "2015b", + "2015.3.187-GNU-4.9.3-2.25": "2015b", + "2015.5.223-GCC-4.9.3-2.25": "TBD", + "2016.0.109-GCC-4.9.3-2.25": "TBD", + "2016.1.150-GCC-4.9.3-2.25": "TBD", + "2016.2.181-GCC-4.9.3-2.25": "TBD", + "2016.2.181-GCC-5.3.0-2.26": "TBD", + "2016.3.210-GCC-4.9.3-2.25": "TBD", + "2016.3.210-GCC-5.3.0-2.26": "TBD", + "2016.3.210-GCC-5.4.0-2.26": "2016b", + "2017.0.098-GCC-5.4.0-2.26": "TBD", + "2017.1.132-GCC-5.4.0-2.26": "TBD", + "2017.1.132-GCC-6.3.0-2.27": "2017a", + "2017.2.174-GCC-6.3.0-2.27": "TBD", + "2017.4.196-GCC-6.4.0-2.28": "2017b", + "2017.5.239-GCC-6.4.0-2.28": "TBD", + "2018.0.128-GCC-6.4.0-2.28": "TBD", + "2018.1.163-GCC-6.4.0-2.28": "2018a", + "2018.2.199-GCC-6.4.0-2.28": "TBD", + "2018.3.222-GCC-7.3.0-2.30": "2018b", + "2018.5.274-GCC-7.3.0-2.30": "TBD", + "2019.0.117-GCC-8.2.0-2.31.1": "TBD", + "2019.1.144-GCC-8.2.0-2.31.1": "2019a", + "2019.2.187-GCC-8.2.0-2.31.1": "TBD", + "2019.3.199-GCC-8.3.0-2.32": "TBD", + "2019.5.281": "2019b", + "2020.0.166-GCC-9.2.0": "TBD", + "2020.1.217": "2020a", + "2020.4.304": "2020b", + "system-GCC-system-2.29": "TBD" + }, + "GNU": { + "4.9.2-2.25": "TBD", + "4.9.3-2.25": "2015b", + "5.1.0-2.25": "TBD" + }, + "PGI": { + "16.3-GCC-4.9.3-2.25": "2015b", + "16.4-GCC-5.3.0-2.26": "TBD", + "16.7-GCC-5.4.0-2.26": "2016b", + "17.4-GCC-6.4.0-2.28": "2017b", + "18.4-GCC-6.4.0-2.28": "2018a" + }, + "iccifortcuda": { + "2016.1.150": "2016a", + "2016.10": "TBD", + "2017.4.196-GCC-6.4.0-2.28": "2017b" + }, + "foss": { + "2015.05": "TBD", + "2016.04": "TBD", + "2016.06": "TBD", + "2016.07": "TBD", + "2016.09": "TBD", + "2018.08": "TBD" + }, + "intel": { + "2014.06": "TBD", + "2014.10": "TBD", + "2014.11": "TBD", + "2015.02": "TBD", + "2015.08": "TBD", + "2016.00": "TBD", + "2016.01": "TBD", + "2016.02-GCC-4.9": "TBD", + "2016.02-GCC-5.3": "TBD", + "2016.03-GCC-4.9": "TBD", + "2016.03-GCC-5.3": "TBD", + "2016.03-GCC-5.4": "TBD", + "2017.00": "TBD", + "2017.01": "TBD", + "2017.02": "TBD", + "2017.09": "TBD", + "2018.00": "TBD", + "2018.01": "TBD", + "2018.02": "TBD", + "2018.04": "TBD", + "2019.00": "TBD", + "2019.01": "TBD", + "2019.02": "TBD", + "2019.03": "TBD", + "2020.00": "TBD", + "2020.06-impi-18.5": "TBD" + }, + "gompic": { + "2016.08": "TBD", + "2016.10": "TBD", + "2017.01": "TBD", + "2017.02": "TBD" + }, + "iimpi": { + "5.2.0": "TBD", + "5.3.0": "TBD", + "5.4.0": "TBD", + "5.5.0-GCC-4.8.3": "TBD", + "5.5.0": "TBD", + "5.5.3-GCC-4.8.3": "TBD", + "6.1.5": "TBD", + "6.2.5": "TBD", + "7.1.2-GCC-4.9.2": "TBD", + "7.1.2": "TBD", + "7.2.3-GCC-4.9.2": "TBD", + "7.2.5-GCC-4.9.2": "TBD", + "7.3.5": "TBD", + "7.3.5-GNU-4.9.3-2.25": "TBD", + "7.5.5-GCC-4.9.3-2.25": "TBD", + "8.1.5-GCC-4.9.3-2.25": "TBD", + "2016.00-GCC-4.9.3-2.25": "TBD", + "2016.01-GCC-4.9.3-2.25": "TBD", + "2016.02-GCC-4.9.3-2.25": "TBD", + "2016.02-GCC-5.3.0-2.26": "TBD", + "2016.03-GCC-4.9.3-2.25": "TBD", + "2016.03-GCC-5.3.0-2.26": "TBD", + "2016.03-GCC-5.4.0-2.26": "TBD", + "2017.00-GCC-5.4.0-2.26": "TBD", + "2017.01-GCC-5.4.0-2.26": "TBD", + "2017.02-GCC-6.3.0-2.27": "TBD", + "2017.09": "TBD", + "2018.01": "TBD", + "2018.04": "TBD", + "2019.01": "TBD", + "2020.00": "TBD", + "2018.00": "TBD", + "2018.02": "TBD", + "2019.00": "TBD", + "2019.02": "TBD", + "2019.03": "TBD", + "2019.08": "TBD", + "2020.06-impi-18.5": "TBD" + }, + "gompi": { + "2015.05": "TBD", + "2016.04": "TBD", + "2016.06": "TBD", + "2016.07": "TBD", + "2016.09": "TBD", + "2018.08": "TBD", + "1.4.10": "TBD", + "1.5.14": "TBD", + "1.6.10": "TBD", + "1.5.16": "TBD", + "1.7.20": "TBD" + }, + "iimpic": { + "2016.10": "TBD" + }, + "fosscuda": { + }, + "gimkl": { + "2.11.5": "TBD" + }, + "iomkl": { + "2015.01": "TBD", + "2015.02": "TBD", + "2015.03": "TBD", + "2016.07": "TBD", + "2016.09-GCC-4.9.3-2.25": "TBD", + "2016.09-GCC-5.4.0-2.26": "TBD", + "2017.01": "TBD", + "2018.02": "TBD", + "2019.01": "TBD" + }, + "gmvapich2": { + "1.7.20": "TBD" + }, + "intelcuda": { + "2016.10": "TBD" + }, + "gmpolf": { + "2017.10": "TBD" + }, + "gimpi": { + "2.11.5": "TBD" + }, + "gmpich": { + "2017.08": "TBD" + }, + "gimpic": { + }, + "gomkl": { + }, + "giolf": { + }, + "ictce": { + "5.2.0": "TBD", + "5.3.0": "TBD", + "5.4.0": "TBD", + "5.5.0": "TBD", + "6.1.5": "TBD", + "6.2.5": "TBD", + "7.1.2": "TBD", + "7.3.5": "TBD" + }, + "goolf": { + "1.4.10": "TBD", + "1.5.14": "TBD", + "1.5.16": "TBD", + "1.6.10": "TBD", + "1.7.20": "TBD" + }, + "CrayGNU": { + "2015.06": "TBD", + "2015.11": "TBD", + "2016.03": "TBD", + "2016.04": "TBD", + "2016.06": "TBD", + "19.06": "TBD" + }, + "goolfc": { + "2016.08": "TBD", + "2016.10": "TBD", + "2017.01": "TBD", + "2017.02": "TBD" + }, + "CrayIntel": { + "2015.06": "TBD", + "2015.11": "TBD", + "2016.06": "TBD", + "19.06": "TBD" + }, + "CrayCCE": { + "2015.06": "TBD", + "2015.11": "TBD", + "19.06": "TBD" + }, + "iompi": { + "2015.01": "TBD", + "2015.02": "TBD", + "2015.03": "TBD", + "2016.07": "TBD", + "2016.09-GCC-4.9.3-2.25": "TBD", + "2016.09-GCC-5.4.0-2.26": "TBD", + "2017.01": "TBD", + "2018.02": "TBD", + "2019.01": "TBD" + }, + "golf": { + }, + "iimkl": { + }, + "pomkl": { + "2016.03": "TBD", + "2016.04": "TBD", + "2016.09": "TBD" + }, + "gmvolf": { + "1.7.20": "TBD" + }, + "giolfc": { + }, + "goblf": { + }, + "iibff": { + }, + "gobff": { + "2020.06-amd": "TBD", + "2020.11": "TBD" + }, + "intel-compilers": { + "2021.1.2": "TBD", + "2021.2.0": "TBD" + }, + "pompi": { + "2016.03": "TBD", + "2016.04": "TBD", + "2016.09": "TBD" + }, + "dummy": { + "dummy": "TBD" + } +} \ No newline at end of file diff --git a/easybuild/tools/module_naming_scheme/generation_mns.py b/easybuild/tools/module_naming_scheme/generation_mns.py index 71b6fe76c6..06a414e4aa 100644 --- a/easybuild/tools/module_naming_scheme/generation_mns.py +++ b/easybuild/tools/module_naming_scheme/generation_mns.py @@ -28,6 +28,8 @@ """ import os +import json +from pkgutil import get_data from easybuild.tools.module_naming_scheme.mns import ModuleNamingScheme # from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version @@ -37,6 +39,8 @@ SYSTEM_TOOLCHAIN_NAME = 'system' +# Lookup table for toolchain versions and generations +GENERATION_LOOKUP = json.loads(get_data(__package__, 'generation_lookup_table.json')) class GenerationModuleNamingScheme(ModuleNamingScheme): """Class implementing the categorized module naming scheme.""" @@ -69,47 +73,18 @@ def det_module_subdir(self, ec): release = 'releases' release_date = '' - if ec['toolchain']['name'] == 'foss': - release_date = ec['toolchain']['version'] - elif ec['toolchain']['name'] == 'GCCcore': - # please add a new GCCcore version - # if you want to use a new toolchain version. - if ec['toolchain']['version'] == '7.3.0': - release_date = '2018b' - elif ec['toolchain']['version'] == '6.3.0': - release_date = '2017a' - elif ec['toolchain']['version'] == '5.4.0': - release_date = '2016b' - elif ec['toolchain']['version'] == '4.9.3': - release_date = '2016a' - elif ec['toolchain']['version'] == '8.3.0': - release_date = '2019b' - elif ec['toolchain']['version'] == '10.2.0': - release_date = '2020b' - elif ec['toolchain']['name'] == 'GCC': - # please add a new GCC version - # if you want to use a new toolchain version. - if ec['toolchain']['version'] == '7.3.0-2.30': - release_date = '2018b' - elif ec['toolchain']['version'] == '6.3.0-2.27': - release_date = '2017a' - elif ec['toolchain']['version'] == '5.4.0-2.26': - release_date = '2016b' - elif ec['toolchain']['version'] == '4.9.3-2.25': - release_date = '2016a' - elif ec['toolchain']['version'] == '8.3.0-2.32': - release_date = '2019b' - elif ec['toolchain']['version'] == '10.2.0': - release_date = '2020b' - elif ec['toolchain']['name'] == 'gompi': - release_date = ec['toolchain']['version'] - elif ec['toolchain']['name'] == 'fosscuda': - release_date = ec['toolchain']['version'] - elif ec['toolchain']['name'] in [DUMMY_TOOLCHAIN_NAME, SYSTEM_TOOLCHAIN_NAME]: - release_date = '' + if ec['toolchain']['name'] in [DUMMY_TOOLCHAIN_NAME, SYSTEM_TOOLCHAIN_NAME]: release = 'General' + + elif ec['toolchain']['version'] in GENERATION_LOOKUP['releases']: + release_date = ec['toolchain']['version'] + + elif ec['toolchain']['name'] in GENERATION_LOOKUP: + release_date = GENERATION_LOOKUP[ec['toolchain']['name']].get(ec['toolchain']['version'], 'NOTFOUND') + else: - release_date = "NOTFOUND" + release_date = 'NOTFOUND' + + subdir = os.path.join(release, release_date).rstrip('/') - subdir = os.path.join(release, release_date) return subdir diff --git a/test/framework/module_generator.py b/test/framework/module_generator.py index 4a8cea46f5..17911b8d05 100644 --- a/test/framework/module_generator.py +++ b/test/framework/module_generator.py @@ -1367,6 +1367,62 @@ def test_ec(ecfile, short_modname, mod_subdir, modpath_exts, user_modpath_exts, for ecfile, mns_vals in test_ecs.items(): test_ec(ecfile, *mns_vals) + def test_generation_mns(self): + """Test generation module naming scheme.""" + + moduleclasses = ['base', 'compiler', 'mpi', 'numlib', 'system', 'toolchain'] + ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') + all_stops = [x[0] for x in EasyBlock.get_steps()] + build_options = { + 'check_osdeps': False, + 'robot_path': [ecs_dir], + 'valid_stops': all_stops, + 'validate': False, + 'valid_module_classes': moduleclasses, + } + + def test_ec(ecfile, short_modname, mod_subdir, modpath_exts, user_modpath_exts, init_modpaths): + """Test whether active module naming scheme returns expected values.""" + ec = EasyConfig(glob.glob(os.path.join(ecs_dir, '*', '*', ecfile))[0]) + self.assertEqual(ActiveMNS().det_full_module_name(ec), os.path.join(mod_subdir, short_modname)) + self.assertEqual(ActiveMNS().det_short_module_name(ec), short_modname) + self.assertEqual(ActiveMNS().det_module_subdir(ec), mod_subdir) + self.assertEqual(ActiveMNS().det_modpath_extensions(ec), modpath_exts) + self.assertEqual(ActiveMNS().det_user_modpath_extensions(ec), user_modpath_exts) + self.assertEqual(ActiveMNS().det_init_modulepaths(ec), init_modpaths) + + os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = 'GenerationModuleNamingScheme' + init_config(build_options=build_options) + + # format: easyconfig_file: (short_mod_name, mod_subdir, modpath_exts, user_modpath_exts, init_modpaths) + test_ecs = { + 'GCCcore-4.9.3.eb': ('GCCcore/4.9.3', 'General', [], [], []), + 'gcccuda-2018a.eb': ('gcccuda/2018a', 'General', [], [], []), + 'binutils-2.25-GCCcore-4.9.3.eb': ('binutils/2.25', 'releases/2016a', [], [], []), + 'GCC-6.4.0-2.28.eb': ('GCC/6.4.0-2.28', 'General', [], [], []), + 'OpenMPI-2.1.2-GCC-6.4.0-2.28.eb': ('OpenMPI/2.1.2', 'releases/2018a', [], [], []), + 'gzip-1.5-foss-2018a.eb': ('gzip/1.5', 'releases/2018a', [], [], []), + 'gzip-1.5-intel-2018a.eb': ('gzip/1.5', 'releases/2018a', [], [], []), + 'foss-2018a.eb': ('foss/2018a', 'General', [], [], []), + 'ifort-2016.1.150.eb': ('ifort/2016.1.150', 'General', [], [], []), + 'iccifort-2019.4.243.eb': ('iccifort/2019.4.243', 'General', [], [], []), + 'imkl-2019.4.243-iimpi-2019.08.eb': ('imkl/2019.4.243', 'releases/TBD', [], [], []), + 'CUDA-9.1.85-GCC-6.4.0-2.28.eb': ('CUDA/9.1.85', 'releases/2018a', [], [], []), + 'CUDA-5.5.22.eb': ('CUDA/5.5.22', 'General', [], [], []), + 'CUDA-5.5.22-iccifort-2016.1.150-GCC-4.9.3-2.25.eb': ('CUDA/5.5.22', 'releases/TBD', [], [], []), + 'CUDA-10.1.243-iccifort-2019.4.243.eb': ('CUDA/10.1.243', 'releases/NOTFOUND', [], [], []), + 'impi-5.1.2.150-iccifortcuda-2016.1.150.eb': ('impi/5.1.2.150', 'releases/2016a', [], [], []), + 'CrayCCE-5.1.29.eb': ('CrayCCE/5.1.29', 'General', [], [], []), + 'HPL-2.1-CrayCCE-5.1.29.eb': ('HPL/2.1', 'releases/NOTFOUND', [], [], []), + 'FFTW-3.3.7-gompi-2018a.eb': ('FFTW/3.3.7', 'releases/2018a', [], [], []), + 'FFTW-3.3.7-gompic-2018a.eb': ('FFTW/3.3.7', 'releases/2018a', [], [], []), + 'hwloc-1.8-gcccuda-2018a.eb': ('hwloc/1.8', 'releases/2018a', [], [], []), + 'imkl-2019.4.243-iimpi-2019.08.eb': ('imkl/2019.4.243', 'releases/TBD', [], [], []) + } + + for ecfile, mns_vals in test_ecs.items(): + test_ec(ecfile, *mns_vals) + def test_dependencies_for(self): """Test for dependencies_for function.""" expected = [