From 1d18c7eda2fd8183e2d760c3902e4e1660e94081 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Andr=C3=A9=20Reuter?= Date: Wed, 13 Nov 2024 21:28:31 +0100 Subject: [PATCH 1/4] EasyBlock: Allow to pass logfile from other EasyBlock MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In the case an EasyBlock creates another instance of an EasyBlock (for example when dealing with Bundles), each instance will create new log file, which may stay open and cause issues down the line. To prevent this, add an option to the EasyBlock constructor to pass a logfile from another EasyBlock. Signed-off-by: Jan André Reuter --- easybuild/framework/easyblock.py | 16 ++++++++---- test/framework/easyblock.py | 42 ++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 5 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index df1866b6db..80db82bb06 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -40,6 +40,7 @@ * Maxime Boissonneault (Compute Canada) * Davide Vanzo (Vanderbilt University) * Caspar van Leeuwen (SURF) +* Jan Andre Reuter (Juelich Supercomputing Centre) """ import copy @@ -141,10 +142,11 @@ def extra_options(extra=None): # # INIT # - def __init__(self, ec): + def __init__(self, ec, logfile=None): """ Initialize the EasyBlock instance. :param ec: a parsed easyconfig file (EasyConfig instance) + :param logfile: pass logfile from other EasyBlock. If not passed, create logfile (optional) """ # keep track of original working directory, so we can go back there @@ -214,7 +216,8 @@ def __init__(self, ec): # logging self.log = None - self.logfile = None + self.logfile = logfile + self.external_logfile = logfile is not None self.logdebug = build_option('debug') self.postmsg = '' # allow a post message to be set, which can be shown as last output self.current_step = None @@ -303,11 +306,11 @@ def _init_log(self): if self.log is not None: return - self.logfile = get_log_filename(self.name, self.version, add_salt=True) - fancylogger.logToFile(self.logfile, max_bytes=0) + if self.logfile is None: + self.logfile = get_log_filename(self.name, self.version, add_salt=True) + fancylogger.logToFile(self.logfile, max_bytes=0) self.log = fancylogger.getLogger(name=self.__class__.__name__, fname=False) - self.log.info(this_is_easybuild()) this_module = inspect.getmodule(self) @@ -323,6 +326,9 @@ def close_log(self): """ Shutdown the logger. """ + # only close log if we created a logfile + if self.external_logfile: + return self.log.info("Closing log for application name %s version %s" % (self.name, self.version)) fancylogger.logToFile(self.logfile, enable=False) diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 19218eca6a..25207ea2f6 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -28,6 +28,7 @@ @author: Jens Timmerman (Ghent University) @author: Kenneth Hoste (Ghent University) @author: Maxime Boissonneault (Compute Canada) +@author: Jan Andre Reuter (Juelich Supercomputing Centre) """ import os import re @@ -2895,6 +2896,47 @@ def run_sanity_check_step(sanity_check_paths, enhance_sanity_check): run_sanity_check_step({}, False) run_sanity_check_step({}, True) + def test_create_easyblock_without_logfile(self): + """ + Test creating an EasyBlock without a logfile. + This represents scenarios found in Bundle and QuantumESPRESSO, where an EasyBlock is + created within another EasyBlock. + """ + self.contents = '\n'.join([ + 'easyblock = "ConfigureMake"', + 'name = "pi"', + 'version = "3.14"', + 'homepage = "http://example.com"', + 'description = "test easyconfig"', + 'toolchain = SYSTEM', + ]) + self.writeEC() + # Ensure that the default case works as expected + eb = EasyBlock(EasyConfig(self.eb_file)) + self.assertNotEqual(eb.log, None) + self.assertNotEqual(eb.logfile, None) + # Get reference to the actual log instance and ensure that it works + # This is NOT eb.log, which represents a separate logger with a separate name. + file_log = fancylogger.getLogger(name=None) + self.assertNotEqual(getattr(file_log, 'logtofile_%s' % eb.logfile), False) + + # Now, create another EasyBlock by passing logfile from first EasyBlock. + eb_external_logfile = EasyBlock(EasyConfig(self.eb_file), logfile=eb.logfile) + self.assertNotEqual(eb_external_logfile.log, None) + self.assertTrue(eb_external_logfile.external_logfile) + self.assertEqual(eb_external_logfile.logfile, eb.logfile) + # Try to log something in it. + eb_external_logfile.log.info("Test message") + + # Try to close EasyBlock with external logfile. This should not affect the logger. + eb_external_logfile.close_log() + self.assertNotEqual(getattr(file_log, 'logtofile_%s' % eb.logfile), False) + # Then close the log from creating EasyBlock. This should work as expected. + eb.close_log() + self.assertEqual(getattr(file_log, 'logtofile_%s' % eb.logfile), False) + + os.remove(eb.logfile) + def suite(): """ return all the tests in this file """ From 7e5a69ce52ebe51d41a912f111f763d9d6d8e5ad Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Fri, 22 Nov 2024 10:31:26 +0100 Subject: [PATCH 2/4] avoid checking loaded module twice in findUpdatedEcs.sh The environment variables for the GCC and GCCcore ($EBROOT*) both point to the GCCcore directory which leads to checking that twice. This is true for all ECs using `altroot`. To avoid outputting the same change multiple times make the folders to check unique first. Also error if `--short` and `--diff` are both given at the same time. --- easybuild/scripts/findUpdatedEcs.sh | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/easybuild/scripts/findUpdatedEcs.sh b/easybuild/scripts/findUpdatedEcs.sh index 66b7e34bd8..918e9b2ca5 100755 --- a/easybuild/scripts/findUpdatedEcs.sh +++ b/easybuild/scripts/findUpdatedEcs.sh @@ -8,7 +8,7 @@ YELLOW='\033[0;33m' NC='\033[0m' function printError { - echo -e "${RED}$@${NC}" + echo -e "${RED}$*${NC}" } verbose=0 @@ -23,11 +23,11 @@ function checkModule { moduleStr="$moduleName/$moduleVersion" printVerbose "Processing $moduleStr" ec_glob=( "$moduleFolder/easybuild/"*.eb ) - if [[ ! -e "${ec_glob[@]}" ]]; then + if [[ ! -e "${ec_glob[0]}" ]]; then printError "=== Did not find installed EC for $moduleStr" return fi - ec_installed="$ec_glob" + ec_installed="${ec_glob[0]}" ec_filename=$(basename "$ec_installed") # Try with most likely location first for speed first_letter=${ec_filename:0:1} @@ -67,15 +67,15 @@ if path=$(which eb 2>/dev/null); then fi function usage { - echo "Usage: $(basename "$0") [--verbose] [--diff] --loaded|--modules INSTALLPATH --easyconfigs EC-FOLDER" + echo "Usage: $(basename "$0") [--verbose] [--short] [--diff] --loaded|--modules INSTALLPATH --easyconfigs EC-FOLDER" echo echo "Check installed modules against the source EasyConfig (EC) files to determine which have changed." echo "Can either check the currently loaded modules or all modules installed in a specific location" echo echo "--verbose Verbose status output while checking" - echo "--loaded Check only currently loaded modules" echo "--short Only show filename of changed ECs" echo "--diff Show diff of changed module files" + echo "--loaded Check only currently loaded modules" echo "--modules INSTALLPATH Check all modules in the specified (software) installpath, i.e. the root of module-binaries" echo "--easyconfigs EC-FOLDER Path to the folder containg the current/updated EasyConfigs. ${ecDefaultFolder:+Defaults to $ecDefaultFolder}" exit 0 @@ -115,6 +115,7 @@ done if [ -z "$easyconfigFolder" ]; then printError "Folder to easyconfigs not given!" && exit 1 fi + if [ -z "$modulesFolder" ]; then if (( checkLoadedModules == 0 )); then printError "Need either --modules or --loaded to specify what to check!" && exit 1 @@ -123,13 +124,18 @@ elif (( checkLoadedModules == 1 )); then printError "Cannot specify --modules and --loaded!" && exit 1 fi +if (( showDiff == 1 && short == 1 )); then + printError "Cannot specify --diff and --short" && exit 1 +fi + if [ -d "$easyconfigFolder/easybuild/easyconfigs" ]; then easyconfigFolder="$easyconfigFolder/easybuild/easyconfigs" fi if (( checkLoadedModules == 1 )); then - for varname in $(compgen -A variable | grep '^EBROOT'); do - checkModule "${!varname}" + IFS=$'\n' read -r -d '' -a unique_module_paths <<< "$(for varname in $(compgen -A variable | grep '^EBROOT'); do echo "${!varname}"; done | sort -u )" || true + for module in "${unique_module_paths[@]}"; do + checkModule "$module" done else for module in "$modulesFolder"/*/*/easybuild; do From bf8d7683baccd19fbc058eb087dd293e196a3f6d Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Mon, 2 Dec 2024 13:52:11 +0100 Subject: [PATCH 3/4] Fix GitHub Actions for CentOS container Node 16 support is removed and Node 20 requires a newer GLibc. Install an unofficial Node 20 that is build against GLibc 2.17 --- .github/workflows/end2end.yml | 11 +++++++++-- .github/workflows/unit_tests_python2.yml | 11 +++++++++-- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/.github/workflows/end2end.yml b/.github/workflows/end2end.yml index e74b65cc79..7951137a1b 100644 --- a/.github/workflows/end2end.yml +++ b/.github/workflows/end2end.yml @@ -18,10 +18,17 @@ jobs: fail-fast: false container: image: ghcr.io/easybuilders/${{ matrix.container }}-amd64 - env: {ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true} # Allow using Node16 actions + volumes: + - /node20217:/node20217:rw,rshared + - ${{ matrix.container == 'centos-7.9' && '/node20217:/__e/node20:ro,rshared' || ' ' }} steps: + - name: install nodejs20glibc2.17 + if: ${{ matrix.container == 'centos-7.9' }} + run: | + curl -LO https://unofficial-builds.nodejs.org/download/release/v20.9.0/node-v20.9.0-linux-x64-glibc-217.tar.xz + tar -xf node-v20.9.0-linux-x64-glibc-217.tar.xz --strip-components 1 -C /node20217 - name: Check out the repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: download and unpack easyblocks and easyconfigs repositories run: | diff --git a/.github/workflows/unit_tests_python2.yml b/.github/workflows/unit_tests_python2.yml index aa00f9a2fd..9ed3ae25d0 100644 --- a/.github/workflows/unit_tests_python2.yml +++ b/.github/workflows/unit_tests_python2.yml @@ -16,9 +16,16 @@ jobs: # CentOS 7.9 container that already includes Lmod & co, # see https://github.com/easybuilders/easybuild-containers image: ghcr.io/easybuilders/centos-7.9-amd64 - env: {ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true} # Allow using Node16 actions + volumes: + - '/node20217:/node20217:rw,rshared' + - '/node20217:/__e/node20:ro,rshared' steps: - - uses: actions/checkout@v3 + - name: install nodejs20glibc2.17 + run: | + curl -LO https://unofficial-builds.nodejs.org/download/release/v20.9.0/node-v20.9.0-linux-x64-glibc-217.tar.xz + tar -xf node-v20.9.0-linux-x64-glibc-217.tar.xz --strip-components 1 -C /node20217 + + - uses: actions/checkout@v4 - name: install Python packages run: | From 57a3e4ee22cec3b9c338704ad6f5a05309789694 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Mon, 2 Dec 2024 14:50:03 +0100 Subject: [PATCH 4/4] Ignore deprecation warning for cryptography on Python 3.7 --- .github/workflows/unit_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 8a8b13af96..1546d20a97 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -195,7 +195,7 @@ jobs: IGNORE_PATTERNS+="|skipping SvnRepository test" IGNORE_PATTERNS+="|requires Lmod as modules tool" IGNORE_PATTERNS+="|stty: 'standard input': Inappropriate ioctl for device" - IGNORE_PATTERNS+="|CryptographyDeprecationWarning: Python 3.[56]" + IGNORE_PATTERNS+="|CryptographyDeprecationWarning: Python 3.[567]" IGNORE_PATTERNS+="|from cryptography.* import " IGNORE_PATTERNS+="|CryptographyDeprecationWarning: Python 2" IGNORE_PATTERNS+="|Blowfish"