From b76277908e4b92c85184f322c52a466cdefd1443 Mon Sep 17 00:00:00 2001 From: Thomas Eylenbosch Date: Thu, 14 Jan 2021 11:15:11 +0100 Subject: [PATCH 1/7] add Generation module naming scheme --- .../module_naming_scheme/generation_mns.py | 113 ++++++++++++++++++ 1 file changed, 113 insertions(+) create mode 100644 easybuild/tools/module_naming_scheme/generation_mns.py diff --git a/easybuild/tools/module_naming_scheme/generation_mns.py b/easybuild/tools/module_naming_scheme/generation_mns.py new file mode 100644 index 0000000000..57b04675f3 --- /dev/null +++ b/easybuild/tools/module_naming_scheme/generation_mns.py @@ -0,0 +1,113 @@ +## +# Copyright 2016-2021 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +Implementation of a different generation specific module naming scheme using release dates. +:author: Thomas Eylenbosch (Gluo N.V.) +""" + +import os +import re + +from easybuild.tools.module_naming_scheme.mns import ModuleNamingScheme +#from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version + +DUMMY_TOOLCHAIN_NAME = 'dummy' +DUMMY_TOOLCHAIN_VERSION = 'dummy' + +SYSTEM_TOOLCHAIN_NAME = 'system' + +class GenerationModuleNamingScheme(ModuleNamingScheme): + """Class implementing the categorized module naming scheme.""" + + REQUIRED_KEYS = ['name', 'version', 'versionsuffix', 'toolchain'] + + def det_full_module_name(self, ec): + """ + Determine short module name, i.e. the name under which modules will be exposed to users. + Examples: GCC/4.8.3, OpenMPI/1.6.5, OpenBLAS/0.2.9, HPL/2.1, Python/2.7.5 + """ + + return os.path.join(self.det_module_subdir(ec), self.det_short_module_name(ec)) + + def det_short_module_name(self, ec): + """ + Determine short module name, i.e. the name under which modules will be exposed to users. + Examples: GCC/4.8.3, OpenMPI/1.6.5, OpenBLAS/0.2.9, HPL/2.1, Python/2.7.5 + """ + return os.path.join(ec['name'], self.det_full_version(ec)) + + + def det_full_version(self, ec): + """Determine full version, taking into account version prefix/suffix.""" + # versionprefix is not always available (e.g., for toolchains) + versionprefix = ec.get('versionprefix', '') + return versionprefix + ec['version'] + ec['versionsuffix'] + + def det_module_subdir(self, ec): + + release = 'releases' + release_date = '' + + if ec['toolchain']['name'] == 'foss': + release_date = ec['toolchain']['version'] + elif ec['toolchain']['name'] == 'GCCcore': # please add a new GCCcore version if you want to use a new toolchain version. + if ec['toolchain']['version'] == '7.3.0': + release_date = '2018b' + elif ec['toolchain']['version'] == '6.3.0': + release_date = '2017a' + elif ec['toolchain']['version'] == '5.4.0': + release_date = '2016b' + elif ec['toolchain']['version'] == '4.9.3': + release_date = '2016a' + elif ec['toolchain']['version'] == '8.3.0': + release_date = '2019b' + elif ec['toolchain']['version'] == '10.2.0': + release_date = '2020b' + elif ec['toolchain']['name'] == 'GCC': # please add a new GCC version if you want to use a new toolchain version. + if ec['toolchain']['version'] == '7.3.0-2.30': + release_date = '2018b' + elif ec['toolchain']['version'] == '6.3.0-2.27': + release_date = '2017a' + elif ec['toolchain']['version'] == '5.4.0-2.26': + release_date = '2016b' + elif ec['toolchain']['version'] == '4.9.3-2.25': + release_date = '2016a' + elif ec['toolchain']['version'] == '8.3.0-2.32': + release_date = '2019b' + elif ec['toolchain']['version'] == '10.2.0': + release_date = '2020b' + elif ec['toolchain']['name'] == 'gompi': + release_date = ec['toolchain']['version'] + elif ec['toolchain']['name'] == 'fosscuda': + release_date = ec['toolchain']['version'] + elif ec['toolchain']['name'] in [DUMMY_TOOLCHAIN_NAME, SYSTEM_TOOLCHAIN_NAME]: + release_date = '' + release = 'General' + else: + release_date = "NOTFOUND" + + subdir = os.path.join(release, release_date) + return subdir + From 3477e9ba5dd60997331a06b07ba0c461cbe05db0 Mon Sep 17 00:00:00 2001 From: Thomas Eylenbosch Date: Fri, 15 Jan 2021 10:23:48 +0100 Subject: [PATCH 2/7] add changes from hound --- .../tools/module_naming_scheme/generation_mns.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/easybuild/tools/module_naming_scheme/generation_mns.py b/easybuild/tools/module_naming_scheme/generation_mns.py index 57b04675f3..fb4c5e619a 100644 --- a/easybuild/tools/module_naming_scheme/generation_mns.py +++ b/easybuild/tools/module_naming_scheme/generation_mns.py @@ -28,16 +28,16 @@ """ import os -import re from easybuild.tools.module_naming_scheme.mns import ModuleNamingScheme -#from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version +# from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version DUMMY_TOOLCHAIN_NAME = 'dummy' DUMMY_TOOLCHAIN_VERSION = 'dummy' SYSTEM_TOOLCHAIN_NAME = 'system' + class GenerationModuleNamingScheme(ModuleNamingScheme): """Class implementing the categorized module naming scheme.""" @@ -58,7 +58,6 @@ def det_short_module_name(self, ec): """ return os.path.join(ec['name'], self.det_full_version(ec)) - def det_full_version(self, ec): """Determine full version, taking into account version prefix/suffix.""" # versionprefix is not always available (e.g., for toolchains) @@ -72,7 +71,8 @@ def det_module_subdir(self, ec): if ec['toolchain']['name'] == 'foss': release_date = ec['toolchain']['version'] - elif ec['toolchain']['name'] == 'GCCcore': # please add a new GCCcore version if you want to use a new toolchain version. + elif ec['toolchain']['name'] == 'GCCcore': + # please add a new GCCcore version if you want to use a new toolchain version. if ec['toolchain']['version'] == '7.3.0': release_date = '2018b' elif ec['toolchain']['version'] == '6.3.0': @@ -85,7 +85,8 @@ def det_module_subdir(self, ec): release_date = '2019b' elif ec['toolchain']['version'] == '10.2.0': release_date = '2020b' - elif ec['toolchain']['name'] == 'GCC': # please add a new GCC version if you want to use a new toolchain version. + elif ec['toolchain']['name'] == 'GCC': + # please add a new GCC version if you want to use a new toolchain version. if ec['toolchain']['version'] == '7.3.0-2.30': release_date = '2018b' elif ec['toolchain']['version'] == '6.3.0-2.27': @@ -110,4 +111,3 @@ def det_module_subdir(self, ec): subdir = os.path.join(release, release_date) return subdir - From 7f5a05d74e0b5efbc0be1e17e6e319dd94695e78 Mon Sep 17 00:00:00 2001 From: Thomas Eylenbosch Date: Fri, 15 Jan 2021 10:27:20 +0100 Subject: [PATCH 3/7] add changes from hound --- easybuild/tools/module_naming_scheme/generation_mns.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/easybuild/tools/module_naming_scheme/generation_mns.py b/easybuild/tools/module_naming_scheme/generation_mns.py index fb4c5e619a..71b6fe76c6 100644 --- a/easybuild/tools/module_naming_scheme/generation_mns.py +++ b/easybuild/tools/module_naming_scheme/generation_mns.py @@ -71,8 +71,9 @@ def det_module_subdir(self, ec): if ec['toolchain']['name'] == 'foss': release_date = ec['toolchain']['version'] - elif ec['toolchain']['name'] == 'GCCcore': - # please add a new GCCcore version if you want to use a new toolchain version. + elif ec['toolchain']['name'] == 'GCCcore': + # please add a new GCCcore version + # if you want to use a new toolchain version. if ec['toolchain']['version'] == '7.3.0': release_date = '2018b' elif ec['toolchain']['version'] == '6.3.0': @@ -85,8 +86,9 @@ def det_module_subdir(self, ec): release_date = '2019b' elif ec['toolchain']['version'] == '10.2.0': release_date = '2020b' - elif ec['toolchain']['name'] == 'GCC': - # please add a new GCC version if you want to use a new toolchain version. + elif ec['toolchain']['name'] == 'GCC': + # please add a new GCC version + # if you want to use a new toolchain version. if ec['toolchain']['version'] == '7.3.0-2.30': release_date = '2018b' elif ec['toolchain']['version'] == '6.3.0-2.27': From 6b93ea6e49d02f7a32cbda5a1737ccabf706b420 Mon Sep 17 00:00:00 2001 From: Thomas Soenen Date: Mon, 17 May 2021 18:05:29 +0200 Subject: [PATCH 4/7] refactoring and unittest generation mns --- .../generation_lookup_table.json | 371 ++++++++++++++++++ .../module_naming_scheme/generation_mns.py | 55 +-- test/framework/module_generator.py | 56 +++ 3 files changed, 442 insertions(+), 40 deletions(-) create mode 100644 easybuild/tools/module_naming_scheme/generation_lookup_table.json diff --git a/easybuild/tools/module_naming_scheme/generation_lookup_table.json b/easybuild/tools/module_naming_scheme/generation_lookup_table.json new file mode 100644 index 0000000000..c4d376b16e --- /dev/null +++ b/easybuild/tools/module_naming_scheme/generation_lookup_table.json @@ -0,0 +1,371 @@ +{ + "releases": [ + "2014a", + "2014b", + "2015a", + "2015b", + "2016a", + "2016b", + "2017a", + "2017b", + "2018a", + "2018b", + "2019a", + "2019b", + "2020a", + "2020b", + "2021a" + ], + "GCCcore": { + "4.8.3": "2014b", + "4.9.2": "2015a", + "4.9.3": "2016a", + "4.9.4": "TBD", + "5.3.0": "TBD", + "5.4.0": "2016b", + "5.5.0": "TBD", + "6.1.0": "TBD", + "6.2.0": "TBD", + "6.3.0": "2017a", + "6.4.0": "2018a", + "7.1.0": "TBD", + "7.2.0": "TBD", + "7.3.0": "2018b", + "7.4.0": "TBD", + "8.1.0": "TBD", + "8.2.0": "2019a", + "8.3.0": "2019b", + "9.1.0": "TBD", + "9.2.0": "TBD", + "9.3.0": "2020a", + "10.1.0": "TBD", + "10.2.0": "2020b", + "10.3.0": "TBD", + "11.1.0": "TBD" + }, + "gcccuda": { + "2016.08": "TBD", + "2016.10": "TBD", + "2017.01": "TBD", + "2017.02": "TBD" + }, + "GCC": { + "4.9.3-2.25": "2016a", + "4.9.4-2.25": "TBD", + "5.3.0-2.26": "TBD", + "5.4.0-2.26": "2016b", + "6.1.0-2.27": "TBD", + "6.2.0-2.27": "TBD", + "6.3.0-2.27": "2017a", + "6.3.0-2.28": "TBD", + "6.4.0-2.28": "2018a", + "7.2.0-2.29": "TBD", + "7.3.0-2.30": "2018b", + "8.2.0-2.31.1": "2019a", + "8.3.0-2.32": "2019b", + "9.2.0-2.32": "TBD", + "4.9.2-binutils-2.25": "TBD", + "4.9.3-binutils-2.25": "2015b", + "5.1.0-binutils-2.25": "TBD", + "4.6.3": "TBD", + "4.6.4": "TBD", + "4.7.2": "TBD", + "4.7.3": "TBD", + "4.8.1": "TBD", + "4.8.2": "TBD", + "4.8.3": "2014b", + "4.8.4": "TBD", + "4.9.2": "2015a", + "4.9.3": "2015b", + "5.2.0": "TBD", + "8.3.0": "2019b", + "9.2.0": "TBD", + "9.3.0": "2020a", + "10.2.0": "2020b", + "10.3.0": "TBD" + }, + "iccifort": { + "2013_sp1.1.106": "TBD", + "2013_sp1.2.144": "TBD", + "2013_sp1.4.211": "TBD", + "2013.2.146": "TBD", + "2013.3.163": "TBD", + "2013.4.183": "TBD", + "2013.5.192-GCC-4.8.3": "2014b", + "2013.5.192": "2014b", + "2015.0.090": "TBD", + "2015.0.090-GCC-4.9.2": "TBD", + "2015.1.133": "2015a", + "2015.1.133-GCC-4.9.2": "2015a", + "2015.2.164-GCC-4.9.2": "TBD", + "2015.3.187": "2015b", + "2015.3.187-GNU-4.9.3-2.25": "2015b", + "2015.5.223-GCC-4.9.3-2.25": "TBD", + "2016.0.109-GCC-4.9.3-2.25": "TBD", + "2016.1.150-GCC-4.9.3-2.25": "TBD", + "2016.2.181-GCC-4.9.3-2.25": "TBD", + "2016.2.181-GCC-5.3.0-2.26": "TBD", + "2016.3.210-GCC-4.9.3-2.25": "TBD", + "2016.3.210-GCC-5.3.0-2.26": "TBD", + "2016.3.210-GCC-5.4.0-2.26": "2016b", + "2017.0.098-GCC-5.4.0-2.26": "TBD", + "2017.1.132-GCC-5.4.0-2.26": "TBD", + "2017.1.132-GCC-6.3.0-2.27": "2017a", + "2017.2.174-GCC-6.3.0-2.27": "TBD", + "2017.4.196-GCC-6.4.0-2.28": "2017b", + "2017.5.239-GCC-6.4.0-2.28": "TBD", + "2018.0.128-GCC-6.4.0-2.28": "TBD", + "2018.1.163-GCC-6.4.0-2.28": "2018a", + "2018.2.199-GCC-6.4.0-2.28": "TBD", + "2018.3.222-GCC-7.3.0-2.30": "2018b", + "2018.5.274-GCC-7.3.0-2.30": "TBD", + "2019.0.117-GCC-8.2.0-2.31.1": "TBD", + "2019.1.144-GCC-8.2.0-2.31.1": "2019a", + "2019.2.187-GCC-8.2.0-2.31.1": "TBD", + "2019.3.199-GCC-8.3.0-2.32": "TBD", + "2019.5.281": "2019b", + "2020.0.166-GCC-9.2.0": "TBD", + "2020.1.217": "2020a", + "2020.4.304": "2020b", + "system-GCC-system-2.29": "TBD" + }, + "GNU": { + "4.9.2-2.25": "TBD", + "4.9.3-2.25": "2015b", + "5.1.0-2.25": "TBD" + }, + "PGI": { + "16.3-GCC-4.9.3-2.25": "2015b", + "16.4-GCC-5.3.0-2.26": "TBD", + "16.7-GCC-5.4.0-2.26": "2016b", + "17.4-GCC-6.4.0-2.28": "2017b", + "18.4-GCC-6.4.0-2.28": "2018a" + }, + "iccifortcuda": { + "2016.1.150": "2016a", + "2016.10": "TBD", + "2017.4.196-GCC-6.4.0-2.28": "2017b" + }, + "foss": { + "2015.05": "TBD", + "2016.04": "TBD", + "2016.06": "TBD", + "2016.07": "TBD", + "2016.09": "TBD", + "2018.08": "TBD" + }, + "intel": { + "2014.06": "TBD", + "2014.10": "TBD", + "2014.11": "TBD", + "2015.02": "TBD", + "2015.08": "TBD", + "2016.00": "TBD", + "2016.01": "TBD", + "2016.02-GCC-4.9": "TBD", + "2016.02-GCC-5.3": "TBD", + "2016.03-GCC-4.9": "TBD", + "2016.03-GCC-5.3": "TBD", + "2016.03-GCC-5.4": "TBD", + "2017.00": "TBD", + "2017.01": "TBD", + "2017.02": "TBD", + "2017.09": "TBD", + "2018.00": "TBD", + "2018.01": "TBD", + "2018.02": "TBD", + "2018.04": "TBD", + "2019.00": "TBD", + "2019.01": "TBD", + "2019.02": "TBD", + "2019.03": "TBD", + "2020.00": "TBD", + "2020.06-impi-18.5": "TBD" + }, + "gompic": { + "2016.08": "TBD", + "2016.10": "TBD", + "2017.01": "TBD", + "2017.02": "TBD" + }, + "iimpi": { + "5.2.0": "TBD", + "5.3.0": "TBD", + "5.4.0": "TBD", + "5.5.0-GCC-4.8.3": "TBD", + "5.5.0": "TBD", + "5.5.3-GCC-4.8.3": "TBD", + "6.1.5": "TBD", + "6.2.5": "TBD", + "7.1.2-GCC-4.9.2": "TBD", + "7.1.2": "TBD", + "7.2.3-GCC-4.9.2": "TBD", + "7.2.5-GCC-4.9.2": "TBD", + "7.3.5": "TBD", + "7.3.5-GNU-4.9.3-2.25": "TBD", + "7.5.5-GCC-4.9.3-2.25": "TBD", + "8.1.5-GCC-4.9.3-2.25": "TBD", + "2016.00-GCC-4.9.3-2.25": "TBD", + "2016.01-GCC-4.9.3-2.25": "TBD", + "2016.02-GCC-4.9.3-2.25": "TBD", + "2016.02-GCC-5.3.0-2.26": "TBD", + "2016.03-GCC-4.9.3-2.25": "TBD", + "2016.03-GCC-5.3.0-2.26": "TBD", + "2016.03-GCC-5.4.0-2.26": "TBD", + "2017.00-GCC-5.4.0-2.26": "TBD", + "2017.01-GCC-5.4.0-2.26": "TBD", + "2017.02-GCC-6.3.0-2.27": "TBD", + "2017.09": "TBD", + "2018.01": "TBD", + "2018.04": "TBD", + "2019.01": "TBD", + "2020.00": "TBD", + "2018.00": "TBD", + "2018.02": "TBD", + "2019.00": "TBD", + "2019.02": "TBD", + "2019.03": "TBD", + "2019.08": "TBD", + "2020.06-impi-18.5": "TBD" + }, + "gompi": { + "2015.05": "TBD", + "2016.04": "TBD", + "2016.06": "TBD", + "2016.07": "TBD", + "2016.09": "TBD", + "2018.08": "TBD", + "1.4.10": "TBD", + "1.5.14": "TBD", + "1.6.10": "TBD", + "1.5.16": "TBD", + "1.7.20": "TBD" + }, + "iimpic": { + "2016.10": "TBD" + }, + "fosscuda": { + }, + "gimkl": { + "2.11.5": "TBD" + }, + "iomkl": { + "2015.01": "TBD", + "2015.02": "TBD", + "2015.03": "TBD", + "2016.07": "TBD", + "2016.09-GCC-4.9.3-2.25": "TBD", + "2016.09-GCC-5.4.0-2.26": "TBD", + "2017.01": "TBD", + "2018.02": "TBD", + "2019.01": "TBD" + }, + "gmvapich2": { + "1.7.20": "TBD" + }, + "intelcuda": { + "2016.10": "TBD" + }, + "gmpolf": { + "2017.10": "TBD" + }, + "gimpi": { + "2.11.5": "TBD" + }, + "gmpich": { + "2017.08": "TBD" + }, + "gimpic": { + }, + "gomkl": { + }, + "giolf": { + }, + "ictce": { + "5.2.0": "TBD", + "5.3.0": "TBD", + "5.4.0": "TBD", + "5.5.0": "TBD", + "6.1.5": "TBD", + "6.2.5": "TBD", + "7.1.2": "TBD", + "7.3.5": "TBD" + }, + "goolf": { + "1.4.10": "TBD", + "1.5.14": "TBD", + "1.5.16": "TBD", + "1.6.10": "TBD", + "1.7.20": "TBD" + }, + "CrayGNU": { + "2015.06": "TBD", + "2015.11": "TBD", + "2016.03": "TBD", + "2016.04": "TBD", + "2016.06": "TBD", + "19.06": "TBD" + }, + "goolfc": { + "2016.08": "TBD", + "2016.10": "TBD", + "2017.01": "TBD", + "2017.02": "TBD" + }, + "CrayIntel": { + "2015.06": "TBD", + "2015.11": "TBD", + "2016.06": "TBD", + "19.06": "TBD" + }, + "CrayCCE": { + "2015.06": "TBD", + "2015.11": "TBD", + "19.06": "TBD" + }, + "iompi": { + "2015.01": "TBD", + "2015.02": "TBD", + "2015.03": "TBD", + "2016.07": "TBD", + "2016.09-GCC-4.9.3-2.25": "TBD", + "2016.09-GCC-5.4.0-2.26": "TBD", + "2017.01": "TBD", + "2018.02": "TBD", + "2019.01": "TBD" + }, + "golf": { + }, + "iimkl": { + }, + "pomkl": { + "2016.03": "TBD", + "2016.04": "TBD", + "2016.09": "TBD" + }, + "gmvolf": { + "1.7.20": "TBD" + }, + "giolfc": { + }, + "goblf": { + }, + "iibff": { + }, + "gobff": { + "2020.06-amd": "TBD", + "2020.11": "TBD" + }, + "intel-compilers": { + "2021.1.2": "TBD", + "2021.2.0": "TBD" + }, + "pompi": { + "2016.03": "TBD", + "2016.04": "TBD", + "2016.09": "TBD" + }, + "dummy": { + "dummy": "TBD" + } +} \ No newline at end of file diff --git a/easybuild/tools/module_naming_scheme/generation_mns.py b/easybuild/tools/module_naming_scheme/generation_mns.py index 71b6fe76c6..06a414e4aa 100644 --- a/easybuild/tools/module_naming_scheme/generation_mns.py +++ b/easybuild/tools/module_naming_scheme/generation_mns.py @@ -28,6 +28,8 @@ """ import os +import json +from pkgutil import get_data from easybuild.tools.module_naming_scheme.mns import ModuleNamingScheme # from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version @@ -37,6 +39,8 @@ SYSTEM_TOOLCHAIN_NAME = 'system' +# Lookup table for toolchain versions and generations +GENERATION_LOOKUP = json.loads(get_data(__package__, 'generation_lookup_table.json')) class GenerationModuleNamingScheme(ModuleNamingScheme): """Class implementing the categorized module naming scheme.""" @@ -69,47 +73,18 @@ def det_module_subdir(self, ec): release = 'releases' release_date = '' - if ec['toolchain']['name'] == 'foss': - release_date = ec['toolchain']['version'] - elif ec['toolchain']['name'] == 'GCCcore': - # please add a new GCCcore version - # if you want to use a new toolchain version. - if ec['toolchain']['version'] == '7.3.0': - release_date = '2018b' - elif ec['toolchain']['version'] == '6.3.0': - release_date = '2017a' - elif ec['toolchain']['version'] == '5.4.0': - release_date = '2016b' - elif ec['toolchain']['version'] == '4.9.3': - release_date = '2016a' - elif ec['toolchain']['version'] == '8.3.0': - release_date = '2019b' - elif ec['toolchain']['version'] == '10.2.0': - release_date = '2020b' - elif ec['toolchain']['name'] == 'GCC': - # please add a new GCC version - # if you want to use a new toolchain version. - if ec['toolchain']['version'] == '7.3.0-2.30': - release_date = '2018b' - elif ec['toolchain']['version'] == '6.3.0-2.27': - release_date = '2017a' - elif ec['toolchain']['version'] == '5.4.0-2.26': - release_date = '2016b' - elif ec['toolchain']['version'] == '4.9.3-2.25': - release_date = '2016a' - elif ec['toolchain']['version'] == '8.3.0-2.32': - release_date = '2019b' - elif ec['toolchain']['version'] == '10.2.0': - release_date = '2020b' - elif ec['toolchain']['name'] == 'gompi': - release_date = ec['toolchain']['version'] - elif ec['toolchain']['name'] == 'fosscuda': - release_date = ec['toolchain']['version'] - elif ec['toolchain']['name'] in [DUMMY_TOOLCHAIN_NAME, SYSTEM_TOOLCHAIN_NAME]: - release_date = '' + if ec['toolchain']['name'] in [DUMMY_TOOLCHAIN_NAME, SYSTEM_TOOLCHAIN_NAME]: release = 'General' + + elif ec['toolchain']['version'] in GENERATION_LOOKUP['releases']: + release_date = ec['toolchain']['version'] + + elif ec['toolchain']['name'] in GENERATION_LOOKUP: + release_date = GENERATION_LOOKUP[ec['toolchain']['name']].get(ec['toolchain']['version'], 'NOTFOUND') + else: - release_date = "NOTFOUND" + release_date = 'NOTFOUND' + + subdir = os.path.join(release, release_date).rstrip('/') - subdir = os.path.join(release, release_date) return subdir diff --git a/test/framework/module_generator.py b/test/framework/module_generator.py index 4a8cea46f5..17911b8d05 100644 --- a/test/framework/module_generator.py +++ b/test/framework/module_generator.py @@ -1367,6 +1367,62 @@ def test_ec(ecfile, short_modname, mod_subdir, modpath_exts, user_modpath_exts, for ecfile, mns_vals in test_ecs.items(): test_ec(ecfile, *mns_vals) + def test_generation_mns(self): + """Test generation module naming scheme.""" + + moduleclasses = ['base', 'compiler', 'mpi', 'numlib', 'system', 'toolchain'] + ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') + all_stops = [x[0] for x in EasyBlock.get_steps()] + build_options = { + 'check_osdeps': False, + 'robot_path': [ecs_dir], + 'valid_stops': all_stops, + 'validate': False, + 'valid_module_classes': moduleclasses, + } + + def test_ec(ecfile, short_modname, mod_subdir, modpath_exts, user_modpath_exts, init_modpaths): + """Test whether active module naming scheme returns expected values.""" + ec = EasyConfig(glob.glob(os.path.join(ecs_dir, '*', '*', ecfile))[0]) + self.assertEqual(ActiveMNS().det_full_module_name(ec), os.path.join(mod_subdir, short_modname)) + self.assertEqual(ActiveMNS().det_short_module_name(ec), short_modname) + self.assertEqual(ActiveMNS().det_module_subdir(ec), mod_subdir) + self.assertEqual(ActiveMNS().det_modpath_extensions(ec), modpath_exts) + self.assertEqual(ActiveMNS().det_user_modpath_extensions(ec), user_modpath_exts) + self.assertEqual(ActiveMNS().det_init_modulepaths(ec), init_modpaths) + + os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = 'GenerationModuleNamingScheme' + init_config(build_options=build_options) + + # format: easyconfig_file: (short_mod_name, mod_subdir, modpath_exts, user_modpath_exts, init_modpaths) + test_ecs = { + 'GCCcore-4.9.3.eb': ('GCCcore/4.9.3', 'General', [], [], []), + 'gcccuda-2018a.eb': ('gcccuda/2018a', 'General', [], [], []), + 'binutils-2.25-GCCcore-4.9.3.eb': ('binutils/2.25', 'releases/2016a', [], [], []), + 'GCC-6.4.0-2.28.eb': ('GCC/6.4.0-2.28', 'General', [], [], []), + 'OpenMPI-2.1.2-GCC-6.4.0-2.28.eb': ('OpenMPI/2.1.2', 'releases/2018a', [], [], []), + 'gzip-1.5-foss-2018a.eb': ('gzip/1.5', 'releases/2018a', [], [], []), + 'gzip-1.5-intel-2018a.eb': ('gzip/1.5', 'releases/2018a', [], [], []), + 'foss-2018a.eb': ('foss/2018a', 'General', [], [], []), + 'ifort-2016.1.150.eb': ('ifort/2016.1.150', 'General', [], [], []), + 'iccifort-2019.4.243.eb': ('iccifort/2019.4.243', 'General', [], [], []), + 'imkl-2019.4.243-iimpi-2019.08.eb': ('imkl/2019.4.243', 'releases/TBD', [], [], []), + 'CUDA-9.1.85-GCC-6.4.0-2.28.eb': ('CUDA/9.1.85', 'releases/2018a', [], [], []), + 'CUDA-5.5.22.eb': ('CUDA/5.5.22', 'General', [], [], []), + 'CUDA-5.5.22-iccifort-2016.1.150-GCC-4.9.3-2.25.eb': ('CUDA/5.5.22', 'releases/TBD', [], [], []), + 'CUDA-10.1.243-iccifort-2019.4.243.eb': ('CUDA/10.1.243', 'releases/NOTFOUND', [], [], []), + 'impi-5.1.2.150-iccifortcuda-2016.1.150.eb': ('impi/5.1.2.150', 'releases/2016a', [], [], []), + 'CrayCCE-5.1.29.eb': ('CrayCCE/5.1.29', 'General', [], [], []), + 'HPL-2.1-CrayCCE-5.1.29.eb': ('HPL/2.1', 'releases/NOTFOUND', [], [], []), + 'FFTW-3.3.7-gompi-2018a.eb': ('FFTW/3.3.7', 'releases/2018a', [], [], []), + 'FFTW-3.3.7-gompic-2018a.eb': ('FFTW/3.3.7', 'releases/2018a', [], [], []), + 'hwloc-1.8-gcccuda-2018a.eb': ('hwloc/1.8', 'releases/2018a', [], [], []), + 'imkl-2019.4.243-iimpi-2019.08.eb': ('imkl/2019.4.243', 'releases/TBD', [], [], []) + } + + for ecfile, mns_vals in test_ecs.items(): + test_ec(ecfile, *mns_vals) + def test_dependencies_for(self): """Test for dependencies_for function.""" expected = [ From 009ca004b802a67d14fb7bbf07c075a4d6050219 Mon Sep 17 00:00:00 2001 From: Thomas Soenen Date: Tue, 3 May 2022 18:34:00 +0200 Subject: [PATCH 5/7] dynamic lookup table for generation mns --- .../module_naming_scheme/generation_mns.py | 111 ++++++++++++++---- 1 file changed, 88 insertions(+), 23 deletions(-) diff --git a/easybuild/tools/module_naming_scheme/generation_mns.py b/easybuild/tools/module_naming_scheme/generation_mns.py index 06a414e4aa..35e3a0f620 100644 --- a/easybuild/tools/module_naming_scheme/generation_mns.py +++ b/easybuild/tools/module_naming_scheme/generation_mns.py @@ -25,34 +25,91 @@ """ Implementation of a different generation specific module naming scheme using release dates. :author: Thomas Eylenbosch (Gluo N.V.) +:author: Thomas Soenen (B-square IT services) """ import os import json -from pkgutil import get_data from easybuild.tools.module_naming_scheme.mns import ModuleNamingScheme -# from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version +from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.robot import search_easyconfigs +from easybuild.framework.easyconfig.easyconfig import get_toolchain_hierarchy DUMMY_TOOLCHAIN_NAME = 'dummy' -DUMMY_TOOLCHAIN_VERSION = 'dummy' - SYSTEM_TOOLCHAIN_NAME = 'system' -# Lookup table for toolchain versions and generations -GENERATION_LOOKUP = json.loads(get_data(__package__, 'generation_lookup_table.json')) +GMNS_ENV = "GENERATION_MODULE_NAMING_SCHEME_LOOKUP_TABLE" +GMNS_PATH = os.environ.get(GMNS_ENV) class GenerationModuleNamingScheme(ModuleNamingScheme): - """Class implementing the categorized module naming scheme.""" + """Class implementing the generational module naming scheme.""" REQUIRED_KEYS = ['name', 'version', 'versionsuffix', 'toolchain'] - def det_full_module_name(self, ec): + def __init__(self): """ - Determine short module name, i.e. the name under which modules will be exposed to users. - Examples: GCC/4.8.3, OpenMPI/1.6.5, OpenBLAS/0.2.9, HPL/2.1, Python/2.7.5 + Generate lookup table that maps toolchains on foss generations. Generations (e.g. 2018a, + 2020b) are fetched from the foss easyconfigs and dynamically mapped on toolchains using + get_toolchain_hierarchy. The lookup table can be extended by the user by providing a file. + + Lookup table is a dict with toolchain-generation key-value pairs:{(GCC, 4.8.2): 2016a}, + with toolchains resembled as a tuple. + + json format of file with custom mappings: + { + "2018b": [{"name": "GCC", "version": "5.2.0"}, {"name": "GCC", "version": "4.8.2"}], + "2019b": [{"name": "GCC", "version": "5.2.4"}, {"name": "GCC", "version": "4.8.4"}], + } """ + super().__init__() + + self.lookup_table = {} + + # Get all generations + foss_filenames = search_easyconfigs("^foss-20[0-9]{2}[a-z]\.eb", + filename_only=True, + print_result=False) + self.generations = [x.split('-')[1].split('.')[0] for x in foss_filenames] + + # map generations on toolchains + for generation in self.generations: + for tc in get_toolchain_hierarchy({'name':'foss', 'version':generation}): + self.lookup_table[(tc['name'], tc['version'])] = generation + # include (foss, ) as a toolchain aswell + self.lookup_table[('foss', generation)] = generation + + # users can provide custom generation-toolchain mapping through a file + if GMNS_PATH: + if not os.path.isfile(GMNS_PATH): + msg = "value of ENV {} ({}) should be a valid filepath" + raise EasyBuildError(msg.format(GMNS_ENV, GMNS_PATH)) + with open(GMNS_PATH, 'r') as hc_lookup: + try: + hc_lookup_data = json.loads(hc_lookup.read()) + except json.decoder.JSONDecodeError: + raise EasyBuildError("{} can't be decoded as json".format(GMNS_PATH)) + if not isinstance(hc_lookup_data, dict): + raise EasyBuildError("{} should contain a dict".format(GMNS_PATH)) + if not set(hc_lookup_data.keys()) <= set(self.generations): + raise EasyBuildError("Keys of {} should be generations".format(GMNS_PATH)) + for generation, toolchains in hc_lookup_data.items(): + if not isinstance(toolchains, list): + raise EasyBuildError("Values of {} should be lists".format(GMNS_PATH)) + for tc in toolchains: + if not isinsance(tc, dict): + msg = "Toolchains in {} should be of type dict" + raise EasyBuildError(msg.format(GMNS_PATH)) + if set(tc.keys()) != set('name', 'version'): + msg = "Toolchains in {} should have two keys ('name', 'version')" + raise EasyBuildError(msg.format(GMNS_PATH)) + self.lookup_table[(tc['name'], tc['version'])] = generation + def det_full_module_name(self, ec): + """ + Determine full module name, relative to the top of the module path. + Examples: General/GCC/4.8.3, Releases/2018b/OpenMPI/1.6.5 + """ return os.path.join(self.det_module_subdir(ec), self.det_short_module_name(ec)) def det_short_module_name(self, ec): @@ -69,22 +126,30 @@ def det_full_version(self, ec): return versionprefix + ec['version'] + ec['versionsuffix'] def det_module_subdir(self, ec): - + """ + Determine subdirectory for module file in $MODULEPATH. This determines the separation + between module names exposed to users, and what's part of the $MODULEPATH. subdirectory + is determined by mapping toolchain on a generation. + """ release = 'releases' release_date = '' if ec['toolchain']['name'] in [DUMMY_TOOLCHAIN_NAME, SYSTEM_TOOLCHAIN_NAME]: release = 'General' - - elif ec['toolchain']['version'] in GENERATION_LOOKUP['releases']: - release_date = ec['toolchain']['version'] - - elif ec['toolchain']['name'] in GENERATION_LOOKUP: - release_date = GENERATION_LOOKUP[ec['toolchain']['name']].get(ec['toolchain']['version'], 'NOTFOUND') - else: - release_date = 'NOTFOUND' - - subdir = os.path.join(release, release_date).rstrip('/') - - return subdir + if self.lookup_table.get((ec['toolchain']['name'], ec['toolchain']['version'])): + release_date = self.lookup_table[(ec['toolchain']['name'], ec['toolchain']['version'])] + else: + tc_hierarchy = get_toolchain_hierarchy({'name': ec['toolchain']['name'], + 'version': ec['toolchain']['version']}) + for tc in tc_hierarchy: + if self.lookup_table.get((tc['name'], tc['version'])): + release_date = self.lookup_table.get((tc['name'], tc['version'])) + break + + if release_date == '': + msg = "Couldn't map software version ({}, {}) to a generation. Provide a custom" \ + "toolchain mapping through {}" + raise EasyBuildError(msg.format(ec['name'], ec['version'], GMNS_ENV)) + + return os.path.join(release, release_date).rstrip('/') From 0066a71667ff8eb860008ec8f17ac6597945404a Mon Sep 17 00:00:00 2001 From: Thomas Soenen Date: Fri, 17 Jun 2022 12:11:37 +0200 Subject: [PATCH 6/7] Removed circular reference when building dynamic mapping --- .../generation_lookup_table.json | 371 ------------------ .../module_naming_scheme/generation_mns.py | 52 +-- test/framework/module_generator.py | 58 +-- 3 files changed, 64 insertions(+), 417 deletions(-) delete mode 100644 easybuild/tools/module_naming_scheme/generation_lookup_table.json diff --git a/easybuild/tools/module_naming_scheme/generation_lookup_table.json b/easybuild/tools/module_naming_scheme/generation_lookup_table.json deleted file mode 100644 index c4d376b16e..0000000000 --- a/easybuild/tools/module_naming_scheme/generation_lookup_table.json +++ /dev/null @@ -1,371 +0,0 @@ -{ - "releases": [ - "2014a", - "2014b", - "2015a", - "2015b", - "2016a", - "2016b", - "2017a", - "2017b", - "2018a", - "2018b", - "2019a", - "2019b", - "2020a", - "2020b", - "2021a" - ], - "GCCcore": { - "4.8.3": "2014b", - "4.9.2": "2015a", - "4.9.3": "2016a", - "4.9.4": "TBD", - "5.3.0": "TBD", - "5.4.0": "2016b", - "5.5.0": "TBD", - "6.1.0": "TBD", - "6.2.0": "TBD", - "6.3.0": "2017a", - "6.4.0": "2018a", - "7.1.0": "TBD", - "7.2.0": "TBD", - "7.3.0": "2018b", - "7.4.0": "TBD", - "8.1.0": "TBD", - "8.2.0": "2019a", - "8.3.0": "2019b", - "9.1.0": "TBD", - "9.2.0": "TBD", - "9.3.0": "2020a", - "10.1.0": "TBD", - "10.2.0": "2020b", - "10.3.0": "TBD", - "11.1.0": "TBD" - }, - "gcccuda": { - "2016.08": "TBD", - "2016.10": "TBD", - "2017.01": "TBD", - "2017.02": "TBD" - }, - "GCC": { - "4.9.3-2.25": "2016a", - "4.9.4-2.25": "TBD", - "5.3.0-2.26": "TBD", - "5.4.0-2.26": "2016b", - "6.1.0-2.27": "TBD", - "6.2.0-2.27": "TBD", - "6.3.0-2.27": "2017a", - "6.3.0-2.28": "TBD", - "6.4.0-2.28": "2018a", - "7.2.0-2.29": "TBD", - "7.3.0-2.30": "2018b", - "8.2.0-2.31.1": "2019a", - "8.3.0-2.32": "2019b", - "9.2.0-2.32": "TBD", - "4.9.2-binutils-2.25": "TBD", - "4.9.3-binutils-2.25": "2015b", - "5.1.0-binutils-2.25": "TBD", - "4.6.3": "TBD", - "4.6.4": "TBD", - "4.7.2": "TBD", - "4.7.3": "TBD", - "4.8.1": "TBD", - "4.8.2": "TBD", - "4.8.3": "2014b", - "4.8.4": "TBD", - "4.9.2": "2015a", - "4.9.3": "2015b", - "5.2.0": "TBD", - "8.3.0": "2019b", - "9.2.0": "TBD", - "9.3.0": "2020a", - "10.2.0": "2020b", - "10.3.0": "TBD" - }, - "iccifort": { - "2013_sp1.1.106": "TBD", - "2013_sp1.2.144": "TBD", - "2013_sp1.4.211": "TBD", - "2013.2.146": "TBD", - "2013.3.163": "TBD", - "2013.4.183": "TBD", - "2013.5.192-GCC-4.8.3": "2014b", - "2013.5.192": "2014b", - "2015.0.090": "TBD", - "2015.0.090-GCC-4.9.2": "TBD", - "2015.1.133": "2015a", - "2015.1.133-GCC-4.9.2": "2015a", - "2015.2.164-GCC-4.9.2": "TBD", - "2015.3.187": "2015b", - "2015.3.187-GNU-4.9.3-2.25": "2015b", - "2015.5.223-GCC-4.9.3-2.25": "TBD", - "2016.0.109-GCC-4.9.3-2.25": "TBD", - "2016.1.150-GCC-4.9.3-2.25": "TBD", - "2016.2.181-GCC-4.9.3-2.25": "TBD", - "2016.2.181-GCC-5.3.0-2.26": "TBD", - "2016.3.210-GCC-4.9.3-2.25": "TBD", - "2016.3.210-GCC-5.3.0-2.26": "TBD", - "2016.3.210-GCC-5.4.0-2.26": "2016b", - "2017.0.098-GCC-5.4.0-2.26": "TBD", - "2017.1.132-GCC-5.4.0-2.26": "TBD", - "2017.1.132-GCC-6.3.0-2.27": "2017a", - "2017.2.174-GCC-6.3.0-2.27": "TBD", - "2017.4.196-GCC-6.4.0-2.28": "2017b", - "2017.5.239-GCC-6.4.0-2.28": "TBD", - "2018.0.128-GCC-6.4.0-2.28": "TBD", - "2018.1.163-GCC-6.4.0-2.28": "2018a", - "2018.2.199-GCC-6.4.0-2.28": "TBD", - "2018.3.222-GCC-7.3.0-2.30": "2018b", - "2018.5.274-GCC-7.3.0-2.30": "TBD", - "2019.0.117-GCC-8.2.0-2.31.1": "TBD", - "2019.1.144-GCC-8.2.0-2.31.1": "2019a", - "2019.2.187-GCC-8.2.0-2.31.1": "TBD", - "2019.3.199-GCC-8.3.0-2.32": "TBD", - "2019.5.281": "2019b", - "2020.0.166-GCC-9.2.0": "TBD", - "2020.1.217": "2020a", - "2020.4.304": "2020b", - "system-GCC-system-2.29": "TBD" - }, - "GNU": { - "4.9.2-2.25": "TBD", - "4.9.3-2.25": "2015b", - "5.1.0-2.25": "TBD" - }, - "PGI": { - "16.3-GCC-4.9.3-2.25": "2015b", - "16.4-GCC-5.3.0-2.26": "TBD", - "16.7-GCC-5.4.0-2.26": "2016b", - "17.4-GCC-6.4.0-2.28": "2017b", - "18.4-GCC-6.4.0-2.28": "2018a" - }, - "iccifortcuda": { - "2016.1.150": "2016a", - "2016.10": "TBD", - "2017.4.196-GCC-6.4.0-2.28": "2017b" - }, - "foss": { - "2015.05": "TBD", - "2016.04": "TBD", - "2016.06": "TBD", - "2016.07": "TBD", - "2016.09": "TBD", - "2018.08": "TBD" - }, - "intel": { - "2014.06": "TBD", - "2014.10": "TBD", - "2014.11": "TBD", - "2015.02": "TBD", - "2015.08": "TBD", - "2016.00": "TBD", - "2016.01": "TBD", - "2016.02-GCC-4.9": "TBD", - "2016.02-GCC-5.3": "TBD", - "2016.03-GCC-4.9": "TBD", - "2016.03-GCC-5.3": "TBD", - "2016.03-GCC-5.4": "TBD", - "2017.00": "TBD", - "2017.01": "TBD", - "2017.02": "TBD", - "2017.09": "TBD", - "2018.00": "TBD", - "2018.01": "TBD", - "2018.02": "TBD", - "2018.04": "TBD", - "2019.00": "TBD", - "2019.01": "TBD", - "2019.02": "TBD", - "2019.03": "TBD", - "2020.00": "TBD", - "2020.06-impi-18.5": "TBD" - }, - "gompic": { - "2016.08": "TBD", - "2016.10": "TBD", - "2017.01": "TBD", - "2017.02": "TBD" - }, - "iimpi": { - "5.2.0": "TBD", - "5.3.0": "TBD", - "5.4.0": "TBD", - "5.5.0-GCC-4.8.3": "TBD", - "5.5.0": "TBD", - "5.5.3-GCC-4.8.3": "TBD", - "6.1.5": "TBD", - "6.2.5": "TBD", - "7.1.2-GCC-4.9.2": "TBD", - "7.1.2": "TBD", - "7.2.3-GCC-4.9.2": "TBD", - "7.2.5-GCC-4.9.2": "TBD", - "7.3.5": "TBD", - "7.3.5-GNU-4.9.3-2.25": "TBD", - "7.5.5-GCC-4.9.3-2.25": "TBD", - "8.1.5-GCC-4.9.3-2.25": "TBD", - "2016.00-GCC-4.9.3-2.25": "TBD", - "2016.01-GCC-4.9.3-2.25": "TBD", - "2016.02-GCC-4.9.3-2.25": "TBD", - "2016.02-GCC-5.3.0-2.26": "TBD", - "2016.03-GCC-4.9.3-2.25": "TBD", - "2016.03-GCC-5.3.0-2.26": "TBD", - "2016.03-GCC-5.4.0-2.26": "TBD", - "2017.00-GCC-5.4.0-2.26": "TBD", - "2017.01-GCC-5.4.0-2.26": "TBD", - "2017.02-GCC-6.3.0-2.27": "TBD", - "2017.09": "TBD", - "2018.01": "TBD", - "2018.04": "TBD", - "2019.01": "TBD", - "2020.00": "TBD", - "2018.00": "TBD", - "2018.02": "TBD", - "2019.00": "TBD", - "2019.02": "TBD", - "2019.03": "TBD", - "2019.08": "TBD", - "2020.06-impi-18.5": "TBD" - }, - "gompi": { - "2015.05": "TBD", - "2016.04": "TBD", - "2016.06": "TBD", - "2016.07": "TBD", - "2016.09": "TBD", - "2018.08": "TBD", - "1.4.10": "TBD", - "1.5.14": "TBD", - "1.6.10": "TBD", - "1.5.16": "TBD", - "1.7.20": "TBD" - }, - "iimpic": { - "2016.10": "TBD" - }, - "fosscuda": { - }, - "gimkl": { - "2.11.5": "TBD" - }, - "iomkl": { - "2015.01": "TBD", - "2015.02": "TBD", - "2015.03": "TBD", - "2016.07": "TBD", - "2016.09-GCC-4.9.3-2.25": "TBD", - "2016.09-GCC-5.4.0-2.26": "TBD", - "2017.01": "TBD", - "2018.02": "TBD", - "2019.01": "TBD" - }, - "gmvapich2": { - "1.7.20": "TBD" - }, - "intelcuda": { - "2016.10": "TBD" - }, - "gmpolf": { - "2017.10": "TBD" - }, - "gimpi": { - "2.11.5": "TBD" - }, - "gmpich": { - "2017.08": "TBD" - }, - "gimpic": { - }, - "gomkl": { - }, - "giolf": { - }, - "ictce": { - "5.2.0": "TBD", - "5.3.0": "TBD", - "5.4.0": "TBD", - "5.5.0": "TBD", - "6.1.5": "TBD", - "6.2.5": "TBD", - "7.1.2": "TBD", - "7.3.5": "TBD" - }, - "goolf": { - "1.4.10": "TBD", - "1.5.14": "TBD", - "1.5.16": "TBD", - "1.6.10": "TBD", - "1.7.20": "TBD" - }, - "CrayGNU": { - "2015.06": "TBD", - "2015.11": "TBD", - "2016.03": "TBD", - "2016.04": "TBD", - "2016.06": "TBD", - "19.06": "TBD" - }, - "goolfc": { - "2016.08": "TBD", - "2016.10": "TBD", - "2017.01": "TBD", - "2017.02": "TBD" - }, - "CrayIntel": { - "2015.06": "TBD", - "2015.11": "TBD", - "2016.06": "TBD", - "19.06": "TBD" - }, - "CrayCCE": { - "2015.06": "TBD", - "2015.11": "TBD", - "19.06": "TBD" - }, - "iompi": { - "2015.01": "TBD", - "2015.02": "TBD", - "2015.03": "TBD", - "2016.07": "TBD", - "2016.09-GCC-4.9.3-2.25": "TBD", - "2016.09-GCC-5.4.0-2.26": "TBD", - "2017.01": "TBD", - "2018.02": "TBD", - "2019.01": "TBD" - }, - "golf": { - }, - "iimkl": { - }, - "pomkl": { - "2016.03": "TBD", - "2016.04": "TBD", - "2016.09": "TBD" - }, - "gmvolf": { - "1.7.20": "TBD" - }, - "giolfc": { - }, - "goblf": { - }, - "iibff": { - }, - "gobff": { - "2020.06-amd": "TBD", - "2020.11": "TBD" - }, - "intel-compilers": { - "2021.1.2": "TBD", - "2021.2.0": "TBD" - }, - "pompi": { - "2016.03": "TBD", - "2016.04": "TBD", - "2016.09": "TBD" - }, - "dummy": { - "dummy": "TBD" - } -} \ No newline at end of file diff --git a/easybuild/tools/module_naming_scheme/generation_mns.py b/easybuild/tools/module_naming_scheme/generation_mns.py index 35e3a0f620..3f6e1123d1 100644 --- a/easybuild/tools/module_naming_scheme/generation_mns.py +++ b/easybuild/tools/module_naming_scheme/generation_mns.py @@ -26,6 +26,7 @@ Implementation of a different generation specific module naming scheme using release dates. :author: Thomas Eylenbosch (Gluo N.V.) :author: Thomas Soenen (B-square IT services) +:author: Alan O'Cais (CECAM) """ import os @@ -34,13 +35,11 @@ from easybuild.tools.module_naming_scheme.mns import ModuleNamingScheme from easybuild.tools.build_log import EasyBuildError from easybuild.tools.robot import search_easyconfigs +from easybuild.tools.config import ConfigurationVariables from easybuild.framework.easyconfig.easyconfig import get_toolchain_hierarchy - -DUMMY_TOOLCHAIN_NAME = 'dummy' -SYSTEM_TOOLCHAIN_NAME = 'system' +from easybuild.tools.toolchain.toolchain import is_system_toolchain GMNS_ENV = "GENERATION_MODULE_NAMING_SCHEME_LOOKUP_TABLE" -GMNS_PATH = os.environ.get(GMNS_ENV) class GenerationModuleNamingScheme(ModuleNamingScheme): """Class implementing the generational module naming scheme.""" @@ -72,6 +71,11 @@ def __init__(self): print_result=False) self.generations = [x.split('-')[1].split('.')[0] for x in foss_filenames] + # get_toolchain_hierarchy() depends on ActiveMNS(), which can't point to + # GenerationModuleNamingScheme to prevent circular reference errors. For that purpose, the MNS + # that ActiveMNS() points to is tweaked while get_toolchain_hierarchy() is used. + ConfigurationVariables()._FrozenDict__dict['module_naming_scheme'] = 'EasyBuildMNS' + # map generations on toolchains for generation in self.generations: for tc in get_toolchain_hierarchy({'name':'foss', 'version':generation}): @@ -79,30 +83,34 @@ def __init__(self): # include (foss, ) as a toolchain aswell self.lookup_table[('foss', generation)] = generation + # Force config to point to other MNS + ConfigurationVariables()._FrozenDict__dict['module_naming_scheme'] = 'GenerationModuleNamingScheme' + # users can provide custom generation-toolchain mapping through a file - if GMNS_PATH: - if not os.path.isfile(GMNS_PATH): + path = os.environ.get(GMNS_ENV) + if path: + if not os.path.isfile(path): msg = "value of ENV {} ({}) should be a valid filepath" - raise EasyBuildError(msg.format(GMNS_ENV, GMNS_PATH)) - with open(GMNS_PATH, 'r') as hc_lookup: + raise EasyBuildError(msg.format(GMNS_ENV, path)) + with open(path, 'r') as hc_lookup: try: hc_lookup_data = json.loads(hc_lookup.read()) except json.decoder.JSONDecodeError: - raise EasyBuildError("{} can't be decoded as json".format(GMNS_PATH)) + raise EasyBuildError("{} can't be decoded as json".format(path)) if not isinstance(hc_lookup_data, dict): - raise EasyBuildError("{} should contain a dict".format(GMNS_PATH)) + raise EasyBuildError("{} should contain a dict".format(path)) if not set(hc_lookup_data.keys()) <= set(self.generations): - raise EasyBuildError("Keys of {} should be generations".format(GMNS_PATH)) + raise EasyBuildError("Keys of {} should be generations".format(path)) for generation, toolchains in hc_lookup_data.items(): if not isinstance(toolchains, list): - raise EasyBuildError("Values of {} should be lists".format(GMNS_PATH)) + raise EasyBuildError("Values of {} should be lists".format(path)) for tc in toolchains: - if not isinsance(tc, dict): + if not isinstance(tc, dict): msg = "Toolchains in {} should be of type dict" - raise EasyBuildError(msg.format(GMNS_PATH)) - if set(tc.keys()) != set('name', 'version'): + raise EasyBuildError(msg.format(path)) + if set(tc.keys()) != {'name', 'version'}: msg = "Toolchains in {} should have two keys ('name', 'version')" - raise EasyBuildError(msg.format(GMNS_PATH)) + raise EasyBuildError(msg.format(path)) self.lookup_table[(tc['name'], tc['version'])] = generation def det_full_module_name(self, ec): @@ -132,24 +140,24 @@ def det_module_subdir(self, ec): is determined by mapping toolchain on a generation. """ release = 'releases' - release_date = '' + release_version = '' - if ec['toolchain']['name'] in [DUMMY_TOOLCHAIN_NAME, SYSTEM_TOOLCHAIN_NAME]: + if is_system_toolchain(ec['toolchain']['name']): release = 'General' else: if self.lookup_table.get((ec['toolchain']['name'], ec['toolchain']['version'])): - release_date = self.lookup_table[(ec['toolchain']['name'], ec['toolchain']['version'])] + release_version = self.lookup_table[(ec['toolchain']['name'], ec['toolchain']['version'])] else: tc_hierarchy = get_toolchain_hierarchy({'name': ec['toolchain']['name'], 'version': ec['toolchain']['version']}) for tc in tc_hierarchy: if self.lookup_table.get((tc['name'], tc['version'])): - release_date = self.lookup_table.get((tc['name'], tc['version'])) + release_version = self.lookup_table.get((tc['name'], tc['version'])) break - if release_date == '': + if release_version == '': msg = "Couldn't map software version ({}, {}) to a generation. Provide a custom" \ "toolchain mapping through {}" raise EasyBuildError(msg.format(ec['name'], ec['version'], GMNS_ENV)) - return os.path.join(release, release_date).rstrip('/') + return os.path.join(release, release_version).rstrip('/') diff --git a/test/framework/module_generator.py b/test/framework/module_generator.py index 4862f8ff79..b5d1983805 100644 --- a/test/framework/module_generator.py +++ b/test/framework/module_generator.py @@ -32,6 +32,7 @@ import os import re import sys +import json import tempfile from distutils.version import LooseVersion from unittest import TextTestRunner, TestSuite @@ -48,7 +49,6 @@ from easybuild.tools.utilities import quote_str from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, find_full_path, init_config - class ModuleGeneratorTest(EnhancedTestCase): """Tests for module_generator module.""" @@ -1456,9 +1456,20 @@ def test_generation_mns(self): 'valid_module_classes': moduleclasses, } + os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = 'GenerationModuleNamingScheme' + os.environ['GENERATION_MODULE_NAMING_SCHEME_LOOKUP_TABLE'] = '/tmp/gmns_hardcoded_data.json' + + gmns_hardcoded_data = {"2018a": [{"name": "GCC", "version": "4.9.2"}]} + with open('/tmp/gmns_hardcoded_data.json', 'w') as f: + f.write(json.dumps(gmns_hardcoded_data)) + f.close() + + init_config(build_options=build_options) + def test_ec(ecfile, short_modname, mod_subdir, modpath_exts, user_modpath_exts, init_modpaths): """Test whether active module naming scheme returns expected values.""" ec = EasyConfig(glob.glob(os.path.join(ecs_dir, '*', '*', ecfile))[0]) + self.assertEqual(ActiveMNS().det_full_module_name(ec), os.path.join(mod_subdir, short_modname)) self.assertEqual(ActiveMNS().det_short_module_name(ec), short_modname) self.assertEqual(ActiveMNS().det_module_subdir(ec), mod_subdir) @@ -1466,38 +1477,37 @@ def test_ec(ecfile, short_modname, mod_subdir, modpath_exts, user_modpath_exts, self.assertEqual(ActiveMNS().det_user_modpath_extensions(ec), user_modpath_exts) self.assertEqual(ActiveMNS().det_init_modulepaths(ec), init_modpaths) - os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = 'GenerationModuleNamingScheme' - init_config(build_options=build_options) - + # test examples that are resolved by the dynamically generated generation lookup table # format: easyconfig_file: (short_mod_name, mod_subdir, modpath_exts, user_modpath_exts, init_modpaths) test_ecs = { + 'OpenMPI-2.1.2-GCC-6.4.0-2.28.eb': ('OpenMPI/2.1.2', 'releases/2018a', [], [], []), 'GCCcore-4.9.3.eb': ('GCCcore/4.9.3', 'General', [], [], []), 'gcccuda-2018a.eb': ('gcccuda/2018a', 'General', [], [], []), - 'binutils-2.25-GCCcore-4.9.3.eb': ('binutils/2.25', 'releases/2016a', [], [], []), - 'GCC-6.4.0-2.28.eb': ('GCC/6.4.0-2.28', 'General', [], [], []), - 'OpenMPI-2.1.2-GCC-6.4.0-2.28.eb': ('OpenMPI/2.1.2', 'releases/2018a', [], [], []), - 'gzip-1.5-foss-2018a.eb': ('gzip/1.5', 'releases/2018a', [], [], []), - 'gzip-1.5-intel-2018a.eb': ('gzip/1.5', 'releases/2018a', [], [], []), - 'foss-2018a.eb': ('foss/2018a', 'General', [], [], []), - 'ifort-2016.1.150.eb': ('ifort/2016.1.150', 'General', [], [], []), - 'iccifort-2019.4.243.eb': ('iccifort/2019.4.243', 'General', [], [], []), - 'imkl-2019.4.243-iimpi-2019.08.eb': ('imkl/2019.4.243', 'releases/TBD', [], [], []), - 'CUDA-9.1.85-GCC-6.4.0-2.28.eb': ('CUDA/9.1.85', 'releases/2018a', [], [], []), - 'CUDA-5.5.22.eb': ('CUDA/5.5.22', 'General', [], [], []), - 'CUDA-5.5.22-iccifort-2016.1.150-GCC-4.9.3-2.25.eb': ('CUDA/5.5.22', 'releases/TBD', [], [], []), - 'CUDA-10.1.243-iccifort-2019.4.243.eb': ('CUDA/10.1.243', 'releases/NOTFOUND', [], [], []), - 'impi-5.1.2.150-iccifortcuda-2016.1.150.eb': ('impi/5.1.2.150', 'releases/2016a', [], [], []), - 'CrayCCE-5.1.29.eb': ('CrayCCE/5.1.29', 'General', [], [], []), - 'HPL-2.1-CrayCCE-5.1.29.eb': ('HPL/2.1', 'releases/NOTFOUND', [], [], []), - 'FFTW-3.3.7-gompi-2018a.eb': ('FFTW/3.3.7', 'releases/2018a', [], [], []), - 'FFTW-3.3.7-gompic-2018a.eb': ('FFTW/3.3.7', 'releases/2018a', [], [], []), - 'hwloc-1.8-gcccuda-2018a.eb': ('hwloc/1.8', 'releases/2018a', [], [], []), - 'imkl-2019.4.243-iimpi-2019.08.eb': ('imkl/2019.4.243', 'releases/TBD', [], [], []) + 'toy-0.0-gompi-2018a.eb': ('toy/0.0', 'releases/2018a', [], [], []), + 'foss-2018a.eb': ('foss/2018a', 'General', [], [], []) } for ecfile, mns_vals in test_ecs.items(): test_ec(ecfile, *mns_vals) + # test error for examples without toolchain-generation mapping in lookup table. EasyConfig() calls + # det_module_subdir() of the generationModuleNamingScheme object for the toolchain (binutils) + with self.assertRaises(EasyBuildError) as cm: + ec = EasyConfig(glob.glob(os.path.join(ecs_dir, '*', '*', 'hwloc-1.6.2-GCC-4.9.3-2.26.eb'))[0]) + + msg = "Couldn't map software version (binutils, 2.26) to a generation. Provide a customtoolchain " \ + "mapping through GENERATION_MODULE_NAMING_SCHEME_LOOKUP_TABLE" + self.assertIn(msg, cm.exception.args[0]) + + # test lookup table extension with user-provided input. User-provided input (GCC 4.9.2 maps on 2018a) + # is provided through a file during setup at the start of the test case. + test_ecs_2 = { + 'bzip2-1.0.6-GCC-4.9.2.eb': ('bzip2/1.0.6', 'releases/2018a', [], [], []) + } + + for ecfile, mns_vals in test_ecs_2.items(): + test_ec(ecfile, *mns_vals) + def test_dependencies_for(self): """Test for dependencies_for function.""" expected = [ From 9df685d93ccaafe82448512b4d8016501207e8f5 Mon Sep 17 00:00:00 2001 From: Thomas Soenen Date: Fri, 17 Jun 2022 14:08:34 +0200 Subject: [PATCH 7/7] hound fixes --- easybuild/tools/module_naming_scheme/generation_mns.py | 5 +++-- test/framework/module_generator.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/easybuild/tools/module_naming_scheme/generation_mns.py b/easybuild/tools/module_naming_scheme/generation_mns.py index 3f6e1123d1..0d98168295 100644 --- a/easybuild/tools/module_naming_scheme/generation_mns.py +++ b/easybuild/tools/module_naming_scheme/generation_mns.py @@ -41,6 +41,7 @@ GMNS_ENV = "GENERATION_MODULE_NAMING_SCHEME_LOOKUP_TABLE" + class GenerationModuleNamingScheme(ModuleNamingScheme): """Class implementing the generational module naming scheme.""" @@ -71,14 +72,14 @@ def __init__(self): print_result=False) self.generations = [x.split('-')[1].split('.')[0] for x in foss_filenames] - # get_toolchain_hierarchy() depends on ActiveMNS(), which can't point to + # get_toolchain_hierarchy() depends on ActiveMNS(), which can't point to # GenerationModuleNamingScheme to prevent circular reference errors. For that purpose, the MNS # that ActiveMNS() points to is tweaked while get_toolchain_hierarchy() is used. ConfigurationVariables()._FrozenDict__dict['module_naming_scheme'] = 'EasyBuildMNS' # map generations on toolchains for generation in self.generations: - for tc in get_toolchain_hierarchy({'name':'foss', 'version':generation}): + for tc in get_toolchain_hierarchy({'name': 'foss', 'version': generation}): self.lookup_table[(tc['name'], tc['version'])] = generation # include (foss, ) as a toolchain aswell self.lookup_table[('foss', generation)] = generation diff --git a/test/framework/module_generator.py b/test/framework/module_generator.py index b5d1983805..3dc3bbe2bc 100644 --- a/test/framework/module_generator.py +++ b/test/framework/module_generator.py @@ -1493,7 +1493,7 @@ def test_ec(ecfile, short_modname, mod_subdir, modpath_exts, user_modpath_exts, # test error for examples without toolchain-generation mapping in lookup table. EasyConfig() calls # det_module_subdir() of the generationModuleNamingScheme object for the toolchain (binutils) with self.assertRaises(EasyBuildError) as cm: - ec = EasyConfig(glob.glob(os.path.join(ecs_dir, '*', '*', 'hwloc-1.6.2-GCC-4.9.3-2.26.eb'))[0]) + EasyConfig(glob.glob(os.path.join(ecs_dir, '*', '*', 'hwloc-1.6.2-GCC-4.9.3-2.26.eb'))[0]) msg = "Couldn't map software version (binutils, 2.26) to a generation. Provide a customtoolchain " \ "mapping through GENERATION_MODULE_NAMING_SCHEME_LOOKUP_TABLE"