diff --git a/.github/workflows/end2end.yml b/.github/workflows/end2end.yml index ed6098de60..14092eac65 100644 --- a/.github/workflows/end2end.yml +++ b/.github/workflows/end2end.yml @@ -17,10 +17,17 @@ jobs: fail-fast: false container: image: ghcr.io/easybuilders/${{ matrix.container }}-amd64 - env: {ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true} # Allow using Node16 actions + volumes: + - /node20217:/node20217:rw,rshared + - ${{ matrix.container == 'centos-7.9' && '/node20217:/__e/node20:ro,rshared' || ' ' }} steps: + - name: install nodejs20glibc2.17 + if: ${{ matrix.container == 'centos-7.9' }} + run: | + curl -LO https://unofficial-builds.nodejs.org/download/release/v20.9.0/node-v20.9.0-linux-x64-glibc-217.tar.xz + tar -xf node-v20.9.0-linux-x64-glibc-217.tar.xz --strip-components 1 -C /node20217 - name: Check out the repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: download and unpack easyblocks and easyconfigs repositories run: | diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index e7b5e2b50d..d39994361d 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -197,7 +197,7 @@ jobs: IGNORE_PATTERNS+="|skipping SvnRepository test" IGNORE_PATTERNS+="|requires Lmod as modules tool" IGNORE_PATTERNS+="|stty: 'standard input': Inappropriate ioctl for device" - IGNORE_PATTERNS+="|CryptographyDeprecationWarning: Python 3.[56]" + IGNORE_PATTERNS+="|CryptographyDeprecationWarning: Python 3.[567]" IGNORE_PATTERNS+="|from cryptography.* import " IGNORE_PATTERNS+="|CryptographyDeprecationWarning: Python 2" IGNORE_PATTERNS+="|Blowfish" diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 9ee91b48d6..b6baf5d55e 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -40,6 +40,7 @@ * Maxime Boissonneault (Compute Canada) * Davide Vanzo (Vanderbilt University) * Caspar van Leeuwen (SURF) +* Jan Andre Reuter (Juelich Supercomputing Centre) """ import concurrent import copy @@ -143,10 +144,11 @@ def extra_options(extra=None): # # INIT # - def __init__(self, ec): + def __init__(self, ec, logfile=None): """ Initialize the EasyBlock instance. :param ec: a parsed easyconfig file (EasyConfig instance) + :param logfile: pass logfile from other EasyBlock. If not passed, create logfile (optional) """ # keep track of original working directory, so we can go back there @@ -216,7 +218,8 @@ def __init__(self, ec): # logging self.log = None - self.logfile = None + self.logfile = logfile + self.external_logfile = logfile is not None self.logdebug = build_option('debug') self.postmsg = '' # allow a post message to be set, which can be shown as last output self.current_step = None @@ -305,11 +308,11 @@ def _init_log(self): if self.log is not None: return - self.logfile = get_log_filename(self.name, self.version, add_salt=True) - fancylogger.logToFile(self.logfile, max_bytes=0) + if self.logfile is None: + self.logfile = get_log_filename(self.name, self.version, add_salt=True) + fancylogger.logToFile(self.logfile, max_bytes=0) self.log = fancylogger.getLogger(name=self.__class__.__name__, fname=False) - self.log.info(this_is_easybuild()) this_module = inspect.getmodule(self) @@ -325,6 +328,9 @@ def close_log(self): """ Shutdown the logger. """ + # only close log if we created a logfile + if self.external_logfile: + return self.log.info("Closing log for application name %s version %s" % (self.name, self.version)) fancylogger.logToFile(self.logfile, enable=False) diff --git a/easybuild/scripts/findUpdatedEcs.sh b/easybuild/scripts/findUpdatedEcs.sh index 66b7e34bd8..918e9b2ca5 100755 --- a/easybuild/scripts/findUpdatedEcs.sh +++ b/easybuild/scripts/findUpdatedEcs.sh @@ -8,7 +8,7 @@ YELLOW='\033[0;33m' NC='\033[0m' function printError { - echo -e "${RED}$@${NC}" + echo -e "${RED}$*${NC}" } verbose=0 @@ -23,11 +23,11 @@ function checkModule { moduleStr="$moduleName/$moduleVersion" printVerbose "Processing $moduleStr" ec_glob=( "$moduleFolder/easybuild/"*.eb ) - if [[ ! -e "${ec_glob[@]}" ]]; then + if [[ ! -e "${ec_glob[0]}" ]]; then printError "=== Did not find installed EC for $moduleStr" return fi - ec_installed="$ec_glob" + ec_installed="${ec_glob[0]}" ec_filename=$(basename "$ec_installed") # Try with most likely location first for speed first_letter=${ec_filename:0:1} @@ -67,15 +67,15 @@ if path=$(which eb 2>/dev/null); then fi function usage { - echo "Usage: $(basename "$0") [--verbose] [--diff] --loaded|--modules INSTALLPATH --easyconfigs EC-FOLDER" + echo "Usage: $(basename "$0") [--verbose] [--short] [--diff] --loaded|--modules INSTALLPATH --easyconfigs EC-FOLDER" echo echo "Check installed modules against the source EasyConfig (EC) files to determine which have changed." echo "Can either check the currently loaded modules or all modules installed in a specific location" echo echo "--verbose Verbose status output while checking" - echo "--loaded Check only currently loaded modules" echo "--short Only show filename of changed ECs" echo "--diff Show diff of changed module files" + echo "--loaded Check only currently loaded modules" echo "--modules INSTALLPATH Check all modules in the specified (software) installpath, i.e. the root of module-binaries" echo "--easyconfigs EC-FOLDER Path to the folder containg the current/updated EasyConfigs. ${ecDefaultFolder:+Defaults to $ecDefaultFolder}" exit 0 @@ -115,6 +115,7 @@ done if [ -z "$easyconfigFolder" ]; then printError "Folder to easyconfigs not given!" && exit 1 fi + if [ -z "$modulesFolder" ]; then if (( checkLoadedModules == 0 )); then printError "Need either --modules or --loaded to specify what to check!" && exit 1 @@ -123,13 +124,18 @@ elif (( checkLoadedModules == 1 )); then printError "Cannot specify --modules and --loaded!" && exit 1 fi +if (( showDiff == 1 && short == 1 )); then + printError "Cannot specify --diff and --short" && exit 1 +fi + if [ -d "$easyconfigFolder/easybuild/easyconfigs" ]; then easyconfigFolder="$easyconfigFolder/easybuild/easyconfigs" fi if (( checkLoadedModules == 1 )); then - for varname in $(compgen -A variable | grep '^EBROOT'); do - checkModule "${!varname}" + IFS=$'\n' read -r -d '' -a unique_module_paths <<< "$(for varname in $(compgen -A variable | grep '^EBROOT'); do echo "${!varname}"; done | sort -u )" || true + for module in "${unique_module_paths[@]}"; do + checkModule "$module" done else for module in "$modulesFolder"/*/*/easybuild; do diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 2e4953b8bc..2eb654ecae 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -28,6 +28,7 @@ @author: Jens Timmerman (Ghent University) @author: Kenneth Hoste (Ghent University) @author: Maxime Boissonneault (Compute Canada) +@author: Jan Andre Reuter (Juelich Supercomputing Centre) """ import os import re @@ -2994,6 +2995,47 @@ def run_sanity_check_step(sanity_check_paths, enhance_sanity_check): run_sanity_check_step({}, False) run_sanity_check_step({}, True) + def test_create_easyblock_without_logfile(self): + """ + Test creating an EasyBlock without a logfile. + This represents scenarios found in Bundle and QuantumESPRESSO, where an EasyBlock is + created within another EasyBlock. + """ + self.contents = '\n'.join([ + 'easyblock = "ConfigureMake"', + 'name = "pi"', + 'version = "3.14"', + 'homepage = "http://example.com"', + 'description = "test easyconfig"', + 'toolchain = SYSTEM', + ]) + self.writeEC() + # Ensure that the default case works as expected + eb = EasyBlock(EasyConfig(self.eb_file)) + self.assertNotEqual(eb.log, None) + self.assertNotEqual(eb.logfile, None) + # Get reference to the actual log instance and ensure that it works + # This is NOT eb.log, which represents a separate logger with a separate name. + file_log = fancylogger.getLogger(name=None) + self.assertNotEqual(getattr(file_log, 'logtofile_%s' % eb.logfile), False) + + # Now, create another EasyBlock by passing logfile from first EasyBlock. + eb_external_logfile = EasyBlock(EasyConfig(self.eb_file), logfile=eb.logfile) + self.assertNotEqual(eb_external_logfile.log, None) + self.assertTrue(eb_external_logfile.external_logfile) + self.assertEqual(eb_external_logfile.logfile, eb.logfile) + # Try to log something in it. + eb_external_logfile.log.info("Test message") + + # Try to close EasyBlock with external logfile. This should not affect the logger. + eb_external_logfile.close_log() + self.assertNotEqual(getattr(file_log, 'logtofile_%s' % eb.logfile), False) + # Then close the log from creating EasyBlock. This should work as expected. + eb.close_log() + self.assertEqual(getattr(file_log, 'logtofile_%s' % eb.logfile), False) + + os.remove(eb.logfile) + def suite(): """ return all the tests in this file """