From 3da393aa2ce8cb82af3b10c0bdda139461899873 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Fri, 20 Jan 2023 13:21:52 +0100 Subject: [PATCH 01/44] reduce number of CI jobs We run into rate limits installing the dependencies. We can reduce the number of CI jobs to almost half by running the tests in a loop for each supported module syntax. --- .github/workflows/unit_tests.yml | 70 +++++++++++++------------------- 1 file changed, 29 insertions(+), 41 deletions(-) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 5ff4a24521..d2f8ec0ae9 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -34,47 +34,25 @@ jobs: - ${{needs.setup.outputs.modulesTcl}} - ${{needs.setup.outputs.modules3}} - ${{needs.setup.outputs.modules4}} - module_syntax: [Lua, Tcl] lc_all: [""] - # don't test with Lua module syntax (only supported in Lmod) - exclude: - - modules_tool: ${{needs.setup.outputs.modulesTcl}} - module_syntax: Lua - - modules_tool: ${{needs.setup.outputs.modules3}} - module_syntax: Lua - - modules_tool: ${{needs.setup.outputs.modules4}} - module_syntax: Lua include: - # Test different Python 3 versions with Lmod 8.x (with both Lua and Tcl module syntax) + # Test different Python 3 versions with Lmod 8.x - python: 3.5 modules_tool: ${{needs.setup.outputs.lmod8}} - module_syntax: Lua - python: 3.7 modules_tool: ${{needs.setup.outputs.lmod8}} - module_syntax: Lua - python: 3.8 modules_tool: ${{needs.setup.outputs.lmod8}} - module_syntax: Lua - - python: 3.8 - modules_tool: ${{needs.setup.outputs.lmod8}} - module_syntax: Tcl - python: 3.9 modules_tool: ${{needs.setup.outputs.lmod8}} - module_syntax: Lua - python: '3.10' modules_tool: ${{needs.setup.outputs.lmod8}} - module_syntax: Lua - - python: '3.11' - modules_tool: ${{needs.setup.outputs.lmod8}} - module_syntax: Lua - python: '3.11' modules_tool: ${{needs.setup.outputs.lmod8}} - module_syntax: Tcl # There may be encoding errors in Python 3 which are hidden when an UTF-8 encoding is set # Hence run the tests (again) with LC_ALL=C and Python 3.6 (or any < 3.7) - python: 3.6 modules_tool: ${{needs.setup.outputs.lmod8}} - module_syntax: Lua lc_all: C fail-fast: false steps: @@ -129,7 +107,7 @@ jobs: # and only when testing with Lua as module syntax, # to avoid hitting GitHub rate limit; # tests that require a GitHub token are skipped automatically when no GitHub token is available - if [[ ! "${{matrix.modules_tool}}" =~ 'Lmod-7' ]] && [[ ! "${{matrix.modules_tool}}" =~ 'modules-' ]] && [[ "${{matrix.module_syntax}}" == 'Lua' ]]; then + if [[ ! "${{matrix.modules_tool}}" =~ 'Lmod-7' ]] && [[ ! "${{matrix.modules_tool}}" =~ 'modules-' ]]; then if [ ! -z $GITHUB_TOKEN ]; then if [ "x${{matrix.python}}" == 'x2.6' ]; then SET_KEYRING="keyring.set_keyring(keyring.backends.file.PlaintextKeyring())"; @@ -172,8 +150,6 @@ jobs: - name: run test suite env: EB_VERBOSE: 1 - EASYBUILD_MODULE_SYNTAX: ${{matrix.module_syntax}} - TEST_EASYBUILD_MODULE_SYNTAX: ${{matrix.module_syntax}} LC_ALL: ${{matrix.lc_all}} run: | # run tests *outside* of checked out easybuild-framework directory, @@ -199,18 +175,30 @@ jobs: export EASYBUILD_MODULES_TOOL=Lmod fi export TEST_EASYBUILD_MODULES_TOOL=$EASYBUILD_MODULES_TOOL - eb --show-config - # gather some useful info on test system - eb --show-system-info - # check GitHub configuration - eb --check-github --github-user=easybuild_test - # create file owned by root but writable by anyone (used by test_copy_file) - sudo touch /tmp/file_to_overwrite_for_easybuild_test_copy_file.txt - sudo chmod o+w /tmp/file_to_overwrite_for_easybuild_test_copy_file.txt - # run test suite - python -O -m test.framework.suite 2>&1 | tee test_framework_suite.log - # try and make sure output of running tests is clean (no printed messages/warnings) - IGNORE_PATTERNS="no GitHub token available|skipping SvnRepository test|requires Lmod as modules tool|stty: 'standard input': Inappropriate ioctl for device|CryptographyDeprecationWarning: Python 3.[56]|from cryptography.* import |CryptographyDeprecationWarning: Python 2|Blowfish|GC3Pie not available, skipping test" - # '|| true' is needed to avoid that Travis stops the job on non-zero exit of grep (i.e. when there are no matches) - PRINTED_MSG=$(egrep -v "${IGNORE_PATTERNS}" test_framework_suite.log | grep '\.\n*[A-Za-z]' || true) - test "x$PRINTED_MSG" = "x" || (echo "ERROR: Found printed messages in output of test suite" && echo "${PRINTED_MSG}" && exit 1) + + # Run tests with LUA and Tcl module syntax (where supported) + for module_syntax in Lua Tcl; do + # Only Lmod supports Lua + if [[ "$module_syntax" == "Lua" ]] && [[ "$EASYBUILD_MODULES_TOOL" != "Lmod" ]]; then + continue + fi + printf '\n\n=====================> Using $module_syntax module syntax <=====================\n\n' + export EASYBUILD_MODULE_SYNTAX="$module_syntax" + export TEST_EASYBUILD_MODULE_SYNTAX="$EASYBUILD_MODULE_SYNTAX" + + eb --show-config + # gather some useful info on test system + eb --show-system-info + # check GitHub configuration + eb --check-github --github-user=easybuild_test + # create file owned by root but writable by anyone (used by test_copy_file) + sudo touch /tmp/file_to_overwrite_for_easybuild_test_copy_file.txt + sudo chmod o+w /tmp/file_to_overwrite_for_easybuild_test_copy_file.txt + # run test suite + python -O -m test.framework.suite 2>&1 | tee test_framework_suite.log + # try and make sure output of running tests is clean (no printed messages/warnings) + IGNORE_PATTERNS="no GitHub token available|skipping SvnRepository test|requires Lmod as modules tool|stty: 'standard input': Inappropriate ioctl for device|CryptographyDeprecationWarning: Python 3.[56]|from cryptography.* import |CryptographyDeprecationWarning: Python 2|Blowfish|GC3Pie not available, skipping test" + # '|| true' is needed to avoid that Travis stops the job on non-zero exit of grep (i.e. when there are no matches) + PRINTED_MSG=$(egrep -v "${IGNORE_PATTERNS}" test_framework_suite.log | grep '\.\n*[A-Za-z]' || true) + test "x$PRINTED_MSG" = "x" || (echo "ERROR: Found printed messages in output of test suite" && echo "${PRINTED_MSG}" && exit 1) + done From 8eeae322b3779497f933abb03bf2ba7b6dc4cd2b Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 16 Aug 2023 09:36:06 +0200 Subject: [PATCH 02/44] run unit tests with python2 via CentOS 7.9 container --- .github/workflows/unit_tests_python2.yml | 86 ++++++++++++++++++++++++ 1 file changed, 86 insertions(+) create mode 100644 .github/workflows/unit_tests_python2.yml diff --git a/.github/workflows/unit_tests_python2.yml b/.github/workflows/unit_tests_python2.yml new file mode 100644 index 0000000000..d15b8c240c --- /dev/null +++ b/.github/workflows/unit_tests_python2.yml @@ -0,0 +1,86 @@ +# documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions +name: EasyBuild framework unit tests (python2) +on: [push, pull_request] + +permissions: + contents: read # to fetch code (actions/checkout) + +concurrency: + group: ${{format('{0}:{1}:{2}', github.repository, github.ref, github.workflow)}} + cancel-in-progress: true + +jobs: + test_python2: + runs-on: ubuntu-20.04 + container: + # CentOS 7.9 container that already includes Lmod & co, + # see https://github.com/easybuilders/easybuild-containers + image: ghcr.io/easybuilders/centos-7.9-amd64 + steps: + - uses: actions/checkout@v3 + + - name: install Python packages + run: | + # Python packages + python2 -V + python2 -m pip --version + python2 -m pip install --upgrade pip + python2 -m pip --version + python2 -m pip install -r requirements.txt + # git config is required to make actual git commits (cfr. tests for GitRepository) + git config --global user.name "Travis CI" + git config --global user.email "travis@travis-ci.org" + git config --get-regexp 'user.*' + + - name: install GitHub token (if available) + env: + # token (owned by @boegelbot) with gist permissions (required for some of the tests for GitHub integration); + # this token is not available in pull requests, so tests that require it are skipped in PRs, + # and are only run after the PR gets merged + GITHUB_TOKEN: ${{secrets.CI_UNIT_TESTS_GITHUB_TOKEN}} + run: | + # tests that require a GitHub token are skipped automatically when no GitHub token is available + if [ ! -z $GITHUB_TOKEN ]; then + python2 -c "import keyring; import keyrings.alt.file; keyring.set_keyring(keyrings.alt.file.PlaintextKeyring()); keyring.set_password('github_token', 'easybuild_test', '$GITHUB_TOKEN')"; + echo "GitHub token installed!" + else + echo "Installation of GitHub token skipped!" + fi + + - name: install sources + run: | + # install from source distribution tarball, to test release as published on PyPI + python2 setup.py sdist + ls dist + export PREFIX=/tmp/$USER/$GITHUB_SHA + python2 -m pip install --prefix $PREFIX dist/easybuild-framework*tar.gz + + - name: run test suite + env: + EB_VERBOSE: 1 + EB_PYTHON: python2 + run: | + # run tests *outside* of checked out easybuild-framework directory, + # to ensure we're testing installed version (see previous step) + cd $HOME + # initialize environment for modules tool + source /etc/profile.d/z00_lmod.sh + type module + module --version + # make sure 'eb' is available via $PATH, and that $PYTHONPATH is set (some tests expect that); + # also pick up changes to $PATH set by sourcing $MOD_INIT + export PREFIX=/tmp/$USER/$GITHUB_SHA + export PATH=$PREFIX/bin:$PATH + export PYTHONPATH=$PREFIX/lib/python2.7/site-packages:$PYTHONPATH + eb --version + # show active EasyBuild configuration + eb --show-config + # gather some useful info on test system + eb --show-system-info + # check GitHub configuration + eb --check-github --github-user=easybuild_test + # create file owned by root but writable by anyone (used by test_copy_file) + sudo touch /tmp/file_to_overwrite_for_easybuild_test_copy_file.txt + sudo chmod o+w /tmp/file_to_overwrite_for_easybuild_test_copy_file.txt + # run test suite + python2 -O -m test.framework.suite From cf81cec2adc6c17e395da6ebe25d428cf8b651d7 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 16 Aug 2023 09:40:08 +0200 Subject: [PATCH 03/44] strip out GC3Pie from requirements.txt before installing Python packages when testing with Python 2 --- .github/workflows/unit_tests_python2.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/unit_tests_python2.yml b/.github/workflows/unit_tests_python2.yml index d15b8c240c..92870724a7 100644 --- a/.github/workflows/unit_tests_python2.yml +++ b/.github/workflows/unit_tests_python2.yml @@ -26,6 +26,8 @@ jobs: python2 -m pip --version python2 -m pip install --upgrade pip python2 -m pip --version + # strip out GC3Pie since installation with ancient setuptools (0.9.8) fails + sed -i '/GC3Pie/d' requirements.txt python2 -m pip install -r requirements.txt # git config is required to make actual git commits (cfr. tests for GitRepository) git config --global user.name "Travis CI" From c335cc9e967d8d2e514791897273d6674a57fafb Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 16 Aug 2023 09:48:04 +0200 Subject: [PATCH 04/44] run command via easybuild user and login shell when running test suite with python2 --- .github/workflows/unit_tests_python2.yml | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/.github/workflows/unit_tests_python2.yml b/.github/workflows/unit_tests_python2.yml index 92870724a7..e508dfc5a6 100644 --- a/.github/workflows/unit_tests_python2.yml +++ b/.github/workflows/unit_tests_python2.yml @@ -43,7 +43,7 @@ jobs: run: | # tests that require a GitHub token are skipped automatically when no GitHub token is available if [ ! -z $GITHUB_TOKEN ]; then - python2 -c "import keyring; import keyrings.alt.file; keyring.set_keyring(keyrings.alt.file.PlaintextKeyring()); keyring.set_password('github_token', 'easybuild_test', '$GITHUB_TOKEN')"; + sudo -u easybuild python2 -c "import keyring; import keyrings.alt.file; keyring.set_keyring(keyrings.alt.file.PlaintextKeyring()); keyring.set_password('github_token', 'easybuild_test', '$GITHUB_TOKEN')"; echo "GitHub token installed!" else echo "Installation of GitHub token skipped!" @@ -65,24 +65,21 @@ jobs: # run tests *outside* of checked out easybuild-framework directory, # to ensure we're testing installed version (see previous step) cd $HOME - # initialize environment for modules tool - source /etc/profile.d/z00_lmod.sh - type module - module --version # make sure 'eb' is available via $PATH, and that $PYTHONPATH is set (some tests expect that); # also pick up changes to $PATH set by sourcing $MOD_INIT export PREFIX=/tmp/$USER/$GITHUB_SHA export PATH=$PREFIX/bin:$PATH export PYTHONPATH=$PREFIX/lib/python2.7/site-packages:$PYTHONPATH - eb --version + # run EasyBuild command via (non-root) easybuild user + login shell + sudo -u easybuild bash -l -c 'module --version; eb --version' # show active EasyBuild configuration - eb --show-config + sudo -u easybuild bash -l -c 'eb --show-config' # gather some useful info on test system - eb --show-system-info + sudo -u easybuild bash -l -c 'eb --show-system-info' # check GitHub configuration - eb --check-github --github-user=easybuild_test + sudo -u easybuild bash -l -c 'eb --check-github --github-user=easybuild_test' # create file owned by root but writable by anyone (used by test_copy_file) sudo touch /tmp/file_to_overwrite_for_easybuild_test_copy_file.txt sudo chmod o+w /tmp/file_to_overwrite_for_easybuild_test_copy_file.txt - # run test suite - python2 -O -m test.framework.suite + # run test suite (via easybuild user + login shell) + sudo -u easybuild bash -l -c "python2 -O -m test.framework.suite" From c941493fc0cc4679ee170ebf35fbd959d3f97ba0 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 16 Aug 2023 13:44:02 +0200 Subject: [PATCH 05/44] prepend updates to $PATH and $PYTHONPATH in commands being run via `sudo -u easybuild bash -l -c ...` --- .github/workflows/unit_tests_python2.yml | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/.github/workflows/unit_tests_python2.yml b/.github/workflows/unit_tests_python2.yml index e508dfc5a6..971ac4ca36 100644 --- a/.github/workflows/unit_tests_python2.yml +++ b/.github/workflows/unit_tests_python2.yml @@ -65,21 +65,19 @@ jobs: # run tests *outside* of checked out easybuild-framework directory, # to ensure we're testing installed version (see previous step) cd $HOME - # make sure 'eb' is available via $PATH, and that $PYTHONPATH is set (some tests expect that); - # also pick up changes to $PATH set by sourcing $MOD_INIT + # make sure 'eb' is available via $PATH, and that $PYTHONPATH is set (some tests expect that) export PREFIX=/tmp/$USER/$GITHUB_SHA - export PATH=$PREFIX/bin:$PATH - export PYTHONPATH=$PREFIX/lib/python2.7/site-packages:$PYTHONPATH + ENV_CMDS="export PATH=$PREFIX/bin:$PATH; export PYTHONPATH=$PREFIX/lib/python2.7/site-packages:$PYTHONPATH" # run EasyBuild command via (non-root) easybuild user + login shell - sudo -u easybuild bash -l -c 'module --version; eb --version' + sudo -u easybuild bash -l -c "${ENV_CMDS}; module --version; eb --version" # show active EasyBuild configuration - sudo -u easybuild bash -l -c 'eb --show-config' + sudo -u easybuild bash -l -c "${ENV_CMDS}; eb --show-config" # gather some useful info on test system - sudo -u easybuild bash -l -c 'eb --show-system-info' + sudo -u easybuild bash -l -c "${ENV_CMDS}; eb --show-system-info" # check GitHub configuration - sudo -u easybuild bash -l -c 'eb --check-github --github-user=easybuild_test' + sudo -u easybuild bash -l -c "${ENV_CMDS}; eb --check-github --github-user=easybuild_test" # create file owned by root but writable by anyone (used by test_copy_file) sudo touch /tmp/file_to_overwrite_for_easybuild_test_copy_file.txt sudo chmod o+w /tmp/file_to_overwrite_for_easybuild_test_copy_file.txt # run test suite (via easybuild user + login shell) - sudo -u easybuild bash -l -c "python2 -O -m test.framework.suite" + sudo -u easybuild bash -l -c "${ENV_CMDS}; python2 -O -m test.framework.suite" From a042318bb5de8cc2dfd93294dc69db1b11aeb050 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 16 Aug 2023 14:39:14 +0200 Subject: [PATCH 06/44] tweak eval_quoted_string helper function in test_quote_py_str to be compatible with both (old versions of) Python 2.7 & 3.x --- test/framework/easyconfig.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index e17f1c3ca7..6ce2d09650 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -2147,10 +2147,13 @@ def eval_quoted_string(quoted_val, val): """ globals = dict() try: - exec('res = %s' % quoted_val, globals) + # this is needlessly complicated because we can't use 'exec' here without potentially running + # into a SyntaxError bug in old Python 2.7 versions (for example when running the tests in CentOS 7.9) + # cfr. https://stackoverflow.com/questions/4484872/why-doesnt-exec-work-in-a-function-with-a-subfunction + edict = {}; eval(compile('res = %s' % quoted_val, '', 'exec'), globals, edict) except Exception as e: # pylint: disable=broad-except self.fail('Failed to evaluate %s (from %s): %s' % (quoted_val, val, e)) - return globals['res'] + return edict['res'] def assertEqual_unquoted(quoted_val, val): """Assert that evaluating the quoted_val yields the val""" From b93eb7ddf44eab7816d999ae7c73a2c70fc43ce1 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 16 Aug 2023 18:21:21 +0200 Subject: [PATCH 07/44] silence deprecation warning for using Python 2 in workflow to run tests with Python 2.7 in CentOS 7.9 container --- .github/workflows/unit_tests_python2.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/unit_tests_python2.yml b/.github/workflows/unit_tests_python2.yml index 971ac4ca36..06f3097436 100644 --- a/.github/workflows/unit_tests_python2.yml +++ b/.github/workflows/unit_tests_python2.yml @@ -58,9 +58,6 @@ jobs: python2 -m pip install --prefix $PREFIX dist/easybuild-framework*tar.gz - name: run test suite - env: - EB_VERBOSE: 1 - EB_PYTHON: python2 run: | # run tests *outside* of checked out easybuild-framework directory, # to ensure we're testing installed version (see previous step) @@ -68,6 +65,7 @@ jobs: # make sure 'eb' is available via $PATH, and that $PYTHONPATH is set (some tests expect that) export PREFIX=/tmp/$USER/$GITHUB_SHA ENV_CMDS="export PATH=$PREFIX/bin:$PATH; export PYTHONPATH=$PREFIX/lib/python2.7/site-packages:$PYTHONPATH" + ENV_CMDS="${ENV_CMDS}; export EB_VERBOSE=1; export EB_PYTHON=python2; export TEST_EASYBUILD_SILENCE_DEPRECATION_WARNINGS=python2" # run EasyBuild command via (non-root) easybuild user + login shell sudo -u easybuild bash -l -c "${ENV_CMDS}; module --version; eb --version" # show active EasyBuild configuration From 1e8dee8e8c0ee064f8f244c99c06a54669bce91e Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 16 Aug 2023 19:20:44 +0200 Subject: [PATCH 08/44] fix code style issues in helper function in test_quote_py_str --- test/framework/easyconfig.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 6ce2d09650..ab1924088d 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -2145,15 +2145,15 @@ def eval_quoted_string(quoted_val, val): Helper function to sanity check we can use the quoted string in Python contexts. Returns the evaluated (i.e. unquoted) string """ - globals = dict() + scope = dict() try: # this is needlessly complicated because we can't use 'exec' here without potentially running # into a SyntaxError bug in old Python 2.7 versions (for example when running the tests in CentOS 7.9) # cfr. https://stackoverflow.com/questions/4484872/why-doesnt-exec-work-in-a-function-with-a-subfunction - edict = {}; eval(compile('res = %s' % quoted_val, '', 'exec'), globals, edict) - except Exception as e: # pylint: disable=broad-except - self.fail('Failed to evaluate %s (from %s): %s' % (quoted_val, val, e)) - return edict['res'] + eval(compile('res = %s' % quoted_val, '', 'exec'), dict(), scope) + except Exception as err: # pylint: disable=broad-except + self.fail('Failed to evaluate %s (from %s): %s' % (quoted_val, val, err)) + return scope['res'] def assertEqual_unquoted(quoted_val, val): """Assert that evaluating the quoted_val yields the val""" From 905d919f8c420528b57a63711d4b482bc149fdaf Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 16 Aug 2023 20:49:10 +0200 Subject: [PATCH 09/44] configure Git for easybuild user in workflow for running test suite with Python 2 --- .github/workflows/unit_tests_python2.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/unit_tests_python2.yml b/.github/workflows/unit_tests_python2.yml index 06f3097436..1b921ee83c 100644 --- a/.github/workflows/unit_tests_python2.yml +++ b/.github/workflows/unit_tests_python2.yml @@ -30,9 +30,9 @@ jobs: sed -i '/GC3Pie/d' requirements.txt python2 -m pip install -r requirements.txt # git config is required to make actual git commits (cfr. tests for GitRepository) - git config --global user.name "Travis CI" - git config --global user.email "travis@travis-ci.org" - git config --get-regexp 'user.*' + sudo -u easybuild git config --global user.name "GitHub Actions" + sudo -u easybuild git config --global user.email "actions@github.com" + sudo -u easybuild git config --get-regexp 'user.*' - name: install GitHub token (if available) env: From 6c026b3ca7eff2b94e0e767ec46c6c48a2890d95 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 16 Aug 2023 20:49:27 +0200 Subject: [PATCH 10/44] fix :avail pattern in test_debug_lmod --- test/framework/options.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/options.py b/test/framework/options.py index 219aa4a39a..f4d331519a 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -5232,7 +5232,7 @@ def test_debug_lmod(self): init_config(build_options={'debug_lmod': True}) out = self.modtool.run_module('avail', return_output=True) - for pattern in [r"^Lmod version", r"^lmod\(--terse -D avail\)\{", "Master:avail"]: + for pattern in [r"^Lmod version", r"^lmod\(--terse -D avail\)\{", ":avail"]: regex = re.compile(pattern, re.M) self.assertTrue(regex.search(out), "Pattern '%s' found in: %s" % (regex.pattern, out)) else: From 5e834e96de31a39e2775235aac7eca1db6b06a67 Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Mon, 11 Sep 2023 12:57:49 +0100 Subject: [PATCH 11/44] Supress ==> Suppress --- easybuild/tools/options.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index b60e57513d..a55cea4bbf 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -501,7 +501,7 @@ def override_options(self): 'silence-deprecation-warnings': ( "Silence specified deprecation warnings out of (%s)" % ', '.join(all_deprecations), 'strlist', 'extend', []), - 'silence-hook-trigger': ("Supress printing of debug message every time a hook is triggered", + 'silence-hook-trigger': ("Suppress printing of debug message every time a hook is triggered", None, 'store_true', False), 'skip-extensions': ("Skip installation of extensions", None, 'store_true', False), 'skip-test-cases': ("Skip running test cases", None, 'store_true', False, 't'), From 1464fe029a9919968baa13a3d9c7b110c669974a Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 11 Sep 2023 16:52:19 +0200 Subject: [PATCH 12/44] bump version to 4.8.2dev --- easybuild/tools/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py index e0016710ae..e34584df36 100644 --- a/easybuild/tools/version.py +++ b/easybuild/tools/version.py @@ -45,7 +45,7 @@ # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like # UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0' # This causes problems further up the dependency chain... -VERSION = LooseVersion('4.8.1') +VERSION = LooseVersion('4.8.2.dev0') UNKNOWN = 'UNKNOWN' From 299a4511b65a462187e3cd6f55d24a24488a2461 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 11 Sep 2023 20:50:29 +0200 Subject: [PATCH 13/44] clean up stray regex.search in test_checksum_step --- test/framework/easyblock.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 6afc9db94f..d526479f52 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -2446,7 +2446,6 @@ def test_checksum_step(self): # make sure there's no error logged for not finding checksums.json, # see also https://github.com/easybuilders/easybuild-framework/issues/4301 regex = re.compile("ERROR .*Couldn't find file checksums.json anywhere", re.M) - regex.search(stdout) self.assertFalse(regex.search(stdout), "Pattern '%s' should not be found in log" % regex.pattern) # fiddle with checksum to check whether faulty checksum is catched From c3fc220401ec0dbc34b5ac531d226ffa01a54e42 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 13 Sep 2023 11:34:51 +0200 Subject: [PATCH 14/44] skip 4 broken tests when using Python 2, so other 850 tests can be run with Python 2 --- test/framework/easyblock.py | 7 +++++++ test/framework/toy_build.py | 19 +++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index d526479f52..819d74fc7a 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -1510,6 +1510,13 @@ def test_fetch_sources(self): def test_download_instructions(self): """Test use of download_instructions easyconfig parameter.""" + + # skip test when using Python 2, since it somehow fails then, + # cfr. https://github.com/easybuilders/easybuild-framework/pull/4333 + if sys.version_info[0] == 2: + print("Skipping test_download_instructions because Python 2.x is being used") + return + orig_test_ec = '\n'.join([ "easyblock = 'ConfigureMake'", "name = 'software_with_missing_sources'", diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index fb8ec907dc..757c73f30a 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -2109,6 +2109,13 @@ def test_package_skip(self): def test_regtest(self): """Test use of --regtest.""" + + # skip test when using Python 2, since it somehow fails then, + # cfr. https://github.com/easybuilders/easybuild-framework/pull/4333 + if sys.version_info[0] == 2: + print("Skipping test_regtest because Python 2.x is being used") + return + self.test_toy_build(extra_args=['--regtest', '--sequential'], verify=False) # just check whether module exists @@ -2125,6 +2132,12 @@ def test_minimal_toolchains(self): def test_reproducibility(self): """Test toy build produces expected reproducibility files""" + # skip test when using Python 2, since it somehow fails then, + # cfr. https://github.com/easybuilders/easybuild-framework/pull/4333 + if sys.version_info[0] == 2: + print("Skipping test_reproducibility because Python 2.x is being used") + return + # We need hooks for a complete test hooks_filename = 'my_hooks.py' hooks_file = os.path.join(self.test_prefix, hooks_filename) @@ -3594,6 +3607,12 @@ def __exit__(self, type, value, traceback): def test_toy_lock_cleanup_signals(self): """Test cleanup of locks after EasyBuild session gets a cancellation signal.""" + # skip test when using Python 2, since it somehow fails then, + # cfr. https://github.com/easybuilders/easybuild-framework/pull/4333 + if sys.version_info[0] == 2: + print("Skipping test_toy_lock_cleanup_signals because Python 2.x is being used") + return + orig_wd = os.getcwd() locks_dir = os.path.join(self.test_installpath, 'software', '.locks') From 40d81a0ccf1c0e9aac1f0f4ff6e12d8d40e1d03b Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 13 Sep 2023 14:29:36 +0200 Subject: [PATCH 15/44] relax number of expected 'waiting' messages in test_toy_build_lock --- test/framework/toy_build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 757c73f30a..9ccb3c08df 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -3562,7 +3562,7 @@ def __exit__(self, type, value, traceback): wait_matches = wait_regex.findall(stdout) # we can't rely on an exact number of 'waiting' messages, so let's go with a range... - self.assertIn(len(wait_matches), range(2, 5)) + self.assertIn(len(wait_matches), range(1, 5)) self.assertTrue(ok_regex.search(stdout), "Pattern '%s' found in: %s" % (ok_regex.pattern, stdout)) From e759c92de2a5804930977df124910af6b8f883ed Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 19 Sep 2023 12:28:02 +0200 Subject: [PATCH 16/44] add documentation for major version templates E.g. `%(cudamajver)s` (introduced by #2850) is available but not documented. --- easybuild/framework/easyconfig/templates.py | 5 +++-- easybuild/tools/docs.py | 5 +++++ test/framework/docs.py | 1 + test/framework/easyconfig.py | 2 +- test/framework/options.py | 2 ++ 5 files changed, 12 insertions(+), 3 deletions(-) diff --git a/easybuild/framework/easyconfig/templates.py b/easybuild/framework/easyconfig/templates.py index d0a209c192..465028c3ca 100644 --- a/easybuild/framework/easyconfig/templates.py +++ b/easybuild/framework/easyconfig/templates.py @@ -77,9 +77,9 @@ ('installdir', "Installation directory"), ('start_dir', "Directory in which the build process begins"), ] -# software names for which to define ver and shortver templates +# software names for which to define ver, majver and shortver templates TEMPLATE_SOFTWARE_VERSIONS = [ - # software name, prefix for *ver and *shortver + # software name, prefix for *ver, *majver and *shortver ('CUDA', 'cuda'), ('CUDAcore', 'cuda'), ('Java', 'java'), @@ -427,6 +427,7 @@ def template_documentation(): # step 2: add *ver/*shortver templates for software listed in TEMPLATE_SOFTWARE_VERSIONS doc.append("Template names/values for (short) software versions") for name, pref in TEMPLATE_SOFTWARE_VERSIONS: + doc.append("%s%%(%smajver)s: major version for %s" % (indent_l1, pref, name)) doc.append("%s%%(%sshortver)s: short version for %s (.)" % (indent_l1, pref, name)) doc.append("%s%%(%sver)s: full version for %s" % (indent_l1, pref, name)) diff --git a/easybuild/tools/docs.py b/easybuild/tools/docs.py index 1fcc6ece83..4f4bfca99c 100644 --- a/easybuild/tools/docs.py +++ b/easybuild/tools/docs.py @@ -440,6 +440,7 @@ def avail_easyconfig_templates_txt(): # step 2: add SOFTWARE_VERSIONS doc.append('Template names/values for (short) software versions') for name, pref in TEMPLATE_SOFTWARE_VERSIONS: + doc.append("%s%%(%smajver)s: major version for %s" % (INDENT_4SPACES, pref, name)) doc.append("%s%%(%sshortver)s: short version for %s (.)" % (INDENT_4SPACES, pref, name)) doc.append("%s%%(%sver)s: full version for %s" % (INDENT_4SPACES, pref, name)) doc.append('') @@ -494,8 +495,10 @@ def avail_easyconfig_templates_rst(): ver = [] ver_desc = [] for name, pref in TEMPLATE_SOFTWARE_VERSIONS: + ver.append('``%%(%smajver)s``' % pref) ver.append('``%%(%sshortver)s``' % pref) ver.append('``%%(%sver)s``' % pref) + ver_desc.append('major version for %s' % name) ver_desc.append('short version for %s (.)' % name) ver_desc.append('full version for %s' % name) table_values = [ver, ver_desc] @@ -557,8 +560,10 @@ def avail_easyconfig_templates_md(): ver = [] ver_desc = [] for name, pref in TEMPLATE_SOFTWARE_VERSIONS: + ver.append('``%%(%smajver)s``' % pref) ver.append('``%%(%sshortver)s``' % pref) ver.append('``%%(%sver)s``' % pref) + ver_desc.append('major version for %s' % name) ver_desc.append('short version for %s (``.``)' % name) ver_desc.append('full version for %s' % name) table_values = [ver, ver_desc] diff --git a/test/framework/docs.py b/test/framework/docs.py index 146d05b24c..84d862bd3c 100644 --- a/test/framework/docs.py +++ b/test/framework/docs.py @@ -785,6 +785,7 @@ def test_avail_easyconfig_templates(self): r"^Template names/values derived from easyconfig instance", r"^\s+%\(version_major\)s: Major version", r"^Template names/values for \(short\) software versions", + r"^\s+%\(pymajver\)s: major version for Python", r"^\s+%\(pyshortver\)s: short version for Python \(\.\)", r"^Template constants that can be used in easyconfigs", r"^\s+SOURCE_TAR_GZ: Source \.tar\.gz bundle \(%\(name\)s-%\(version\)s.tar.gz\)", diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index ab1924088d..14ed69084b 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -1290,7 +1290,7 @@ def test_templating_doc(self): # expected length: 1 per constant and 2 extra per constantgroup (title + empty line in between) temps = [ easyconfig.templates.TEMPLATE_NAMES_EASYCONFIG, - easyconfig.templates.TEMPLATE_SOFTWARE_VERSIONS * 2, + easyconfig.templates.TEMPLATE_SOFTWARE_VERSIONS * 3, easyconfig.templates.TEMPLATE_NAMES_CONFIG, easyconfig.templates.TEMPLATE_NAMES_LOWER, easyconfig.templates.TEMPLATE_NAMES_EASYBLOCK_RUN_STEP, diff --git a/test/framework/options.py b/test/framework/options.py index f4d331519a..51b67e39e5 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -565,6 +565,7 @@ def run_test(fmt=None): pattern_lines = [ r'^``%\(version_major\)s``\s+Major version\s*$', r'^``%\(cudaver\)s``\s+full version for CUDA\s*$', + r'^``%\(cudamajver\)s``\s+major version for CUDA\s*$', r'^``%\(pyshortver\)s``\s+short version for Python \(.\)\s*$', r'^\* ``%\(name\)s``$', r'^``%\(namelower\)s``\s+lower case of value of name\s*$', @@ -576,6 +577,7 @@ def run_test(fmt=None): pattern_lines = [ r'^\s+%\(version_major\)s: Major version$', r'^\s+%\(cudaver\)s: full version for CUDA$', + r'^\s+%\(cudamajver\)s: major version for CUDA$', r'^\s+%\(pyshortver\)s: short version for Python \(.\)$', r'^\s+%\(name\)s$', r'^\s+%\(namelower\)s: lower case of value of name$', From 5628b280ef144da1248f89aae019d22f09ff7ec6 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 27 Sep 2023 08:49:04 +0200 Subject: [PATCH 17/44] reset tempdir to avoid that tmpdir path gets progressively deeper with each easystack item (fixes #4291) --- easybuild/main.py | 4 +++- test/framework/easystack.py | 30 ++++++++++++++++++++++++++++-- 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/easybuild/main.py b/easybuild/main.py index 1a9aa736a7..73550e3998 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -42,6 +42,7 @@ import os import stat import sys +import tempfile import traceback # IMPORTANT this has to be the first easybuild import as it customises the logging @@ -254,8 +255,9 @@ def process_easystack(easystack_path, args, logfile, testing, init_session_state easyconfig._easyconfigs_cache.clear() easyconfig._easyconfig_files_cache.clear() - # restore environment + # restore environment and reset tempdir (to avoid tmpdir path getting progressively longer) restore_env(init_env) + tempfile.tempdir = None # If EasyConfig specific arguments were supplied in EasyStack file # merge arguments with original command line args diff --git a/test/framework/easystack.py b/test/framework/easystack.py index ac36d7a26f..6951857129 100644 --- a/test/framework/easystack.py +++ b/test/framework/easystack.py @@ -31,6 +31,7 @@ import os import re import sys +import tempfile from unittest import TextTestRunner import easybuild.tools.build_log @@ -129,11 +130,22 @@ def test_easystack_invalid_key2(self): self.assertErrorRegex(EasyBuildError, error_pattern, parse_easystack, test_easystack) def test_easystack_restore_env_after_each_build(self): - """Test that the build environment is reset for each easystack item""" + """Test that the build environment and tmpdir is reset for each easystack item""" + + orig_tmpdir_tempfile = tempfile.gettempdir() + orig_tmpdir_env = os.getenv('TMPDIR') + orig_tmpdir_tempfile_len = len(orig_tmpdir_env.split(os.path.sep)) + orig_tmpdir_env_len = len(orig_tmpdir_env.split(os.path.sep)) + test_es_txt = '\n'.join([ "easyconfigs:", " - toy-0.0-gompi-2018a.eb:", " - libtoy-0.0.eb:", + # also include a couple of easyconfigs for which a module is already available in test environment, + # see test/framework/modules + " - GCC-7.3.0-2.30", + " - FFTW-3.3.7-gompi-2018a", + " - foss-2018a", ]) test_es_path = os.path.join(self.test_prefix, 'test.yml') write_file(test_es_path, test_es_txt) @@ -143,10 +155,24 @@ def test_easystack_restore_env_after_each_build(self): '--easystack', test_es_path ] - stdout = self.eb_main(args, do_build=True, raise_error=True) + stdout = self.eb_main(args, do_build=True, raise_error=True, reset_env=False, redo_init_config=False) regex = re.compile(r"WARNING Loaded modules detected: \[.*gompi/2018.*\]\n") self.assertFalse(regex.search(stdout), "Pattern '%s' should not be found in: %s" % (regex.pattern, stdout)) + # temporary directory after run should be exactly 2 levels deeper than original one: + # - 1 level added by setting up configuration in EasyBuild main function + # - 1 extra level added by first re-configuration for easystack item + # (because $TMPDIR set by configuration done in main function is retained) + tmpdir_tempfile = tempfile.gettempdir() + tmpdir_env = os.getenv('TMPDIR') + tmpdir_tempfile_len = len(tmpdir_env.split(os.path.sep)) + tmpdir_env_len = len(tmpdir_env.split(os.path.sep)) + + self.assertEqual(tmpdir_tempfile_len, orig_tmpdir_tempfile_len + 2) + self.assertEqual(tmpdir_env_len, orig_tmpdir_env_len + 2) + self.assertTrue(tmpdir_tempfile.startswith(orig_tmpdir_tempfile)) + self.assertTrue(tmpdir_env.startswith(orig_tmpdir_env)) + def test_missing_easyconfigs_key(self): """Test that EasyStack file that doesn't contain an EasyConfigs key will fail with sane error message""" topdir = os.path.dirname(os.path.abspath(__file__)) From 7a1a56d0a7f94af13ed019c95115d7e046e51a9f Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 27 Sep 2023 15:42:01 +0200 Subject: [PATCH 18/44] fix mistake in test_easystack_restore_env_after_each_build when determine length of tmpdir path in tempfile Co-authored-by: ocaisa --- test/framework/easystack.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/easystack.py b/test/framework/easystack.py index 6951857129..91efcc94b3 100644 --- a/test/framework/easystack.py +++ b/test/framework/easystack.py @@ -165,7 +165,7 @@ def test_easystack_restore_env_after_each_build(self): # (because $TMPDIR set by configuration done in main function is retained) tmpdir_tempfile = tempfile.gettempdir() tmpdir_env = os.getenv('TMPDIR') - tmpdir_tempfile_len = len(tmpdir_env.split(os.path.sep)) + tmpdir_tempfile_len = len(tmpdir_tempfile.split(os.path.sep)) tmpdir_env_len = len(tmpdir_env.split(os.path.sep)) self.assertEqual(tmpdir_tempfile_len, orig_tmpdir_tempfile_len + 2) From e4b2ac353364b02152cc614ee2089bf8eaf026a5 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 27 Sep 2023 15:52:03 +0200 Subject: [PATCH 19/44] minor tweaks to CI workflow for running EasyBuild framework unit test suite --- .github/workflows/unit_tests.yml | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index d2f8ec0ae9..5b09fc654b 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -103,16 +103,11 @@ jobs: # and are only run after the PR gets merged GITHUB_TOKEN: ${{secrets.CI_UNIT_TESTS_GITHUB_TOKEN}} run: | - # don't install GitHub token when testing with Lmod 7.x or non-Lmod module tools, - # and only when testing with Lua as module syntax, - # to avoid hitting GitHub rate limit; + # don't install GitHub token when testing with Lmod 7.x or non-Lmod module tools, to avoid hitting GitHub rate limit; # tests that require a GitHub token are skipped automatically when no GitHub token is available if [[ ! "${{matrix.modules_tool}}" =~ 'Lmod-7' ]] && [[ ! "${{matrix.modules_tool}}" =~ 'modules-' ]]; then if [ ! -z $GITHUB_TOKEN ]; then - if [ "x${{matrix.python}}" == 'x2.6' ]; - then SET_KEYRING="keyring.set_keyring(keyring.backends.file.PlaintextKeyring())"; - else SET_KEYRING="import keyrings.alt.file; keyring.set_keyring(keyrings.alt.file.PlaintextKeyring())"; - fi; + SET_KEYRING="import keyrings.alt.file; keyring.set_keyring(keyrings.alt.file.PlaintextKeyring())"; python -c "import keyring; $SET_KEYRING; keyring.set_password('github_token', 'easybuild_test', '$GITHUB_TOKEN')"; fi echo "GitHub token installed!" @@ -174,17 +169,18 @@ jobs: else export EASYBUILD_MODULES_TOOL=Lmod fi - export TEST_EASYBUILD_MODULES_TOOL=$EASYBUILD_MODULES_TOOL + export TEST_EASYBUILD_MODULES_TOOL=${EASYBUILD_MODULES_TOOL} # Run tests with LUA and Tcl module syntax (where supported) for module_syntax in Lua Tcl; do # Only Lmod supports Lua - if [[ "$module_syntax" == "Lua" ]] && [[ "$EASYBUILD_MODULES_TOOL" != "Lmod" ]]; then + if [[ "${module_syntax}" == "Lua" ]] && [[ "${EASYBUILD_MODULES_TOOL}" != "Lmod" ]]; then + echo "Not testing with '${module_syntax}' as module syntax with '${EASYBUILD_MODULES_TOOL}' as modules tool" continue fi printf '\n\n=====================> Using $module_syntax module syntax <=====================\n\n' - export EASYBUILD_MODULE_SYNTAX="$module_syntax" - export TEST_EASYBUILD_MODULE_SYNTAX="$EASYBUILD_MODULE_SYNTAX" + export EASYBUILD_MODULE_SYNTAX="${module_syntax}" + export TEST_EASYBUILD_MODULE_SYNTAX="${EASYBUILD_MODULE_SYNTAX}" eb --show-config # gather some useful info on test system @@ -198,7 +194,7 @@ jobs: python -O -m test.framework.suite 2>&1 | tee test_framework_suite.log # try and make sure output of running tests is clean (no printed messages/warnings) IGNORE_PATTERNS="no GitHub token available|skipping SvnRepository test|requires Lmod as modules tool|stty: 'standard input': Inappropriate ioctl for device|CryptographyDeprecationWarning: Python 3.[56]|from cryptography.* import |CryptographyDeprecationWarning: Python 2|Blowfish|GC3Pie not available, skipping test" - # '|| true' is needed to avoid that Travis stops the job on non-zero exit of grep (i.e. when there are no matches) + # '|| true' is needed to avoid that GitHub Actions stops the job on non-zero exit of grep (i.e. when there are no matches) PRINTED_MSG=$(egrep -v "${IGNORE_PATTERNS}" test_framework_suite.log | grep '\.\n*[A-Za-z]' || true) test "x$PRINTED_MSG" = "x" || (echo "ERROR: Found printed messages in output of test suite" && echo "${PRINTED_MSG}" && exit 1) done From 23778be7d7b0f4e73a069d076fd89c8fa0dd4156 Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Thu, 12 Oct 2023 15:19:48 +0100 Subject: [PATCH 20/44] fix parallellisation typo --- easybuild/toolchains/compiler/gcc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/toolchains/compiler/gcc.py b/easybuild/toolchains/compiler/gcc.py index 7a2c3770e3..3fba8d3954 100644 --- a/easybuild/toolchains/compiler/gcc.py +++ b/easybuild/toolchains/compiler/gcc.py @@ -50,7 +50,7 @@ class Gcc(Compiler): COMPILER_FAMILY = TC_CONSTANT_GCC COMPILER_UNIQUE_OPTS = { - 'loop': (False, "Automatic loop parallellisation"), + 'loop': (False, "Automatic loop parallelisation"), 'f2c': (False, "Generate code compatible with f2c and f77"), 'lto': (False, "Enable Link Time Optimization"), } From 13370c1c7d6387747e7bcbeeb886acb54e18f428 Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Thu, 12 Oct 2023 15:38:35 +0100 Subject: [PATCH 21/44] parallelism --- easybuild/tools/options.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index a55cea4bbf..3cc0d5b7fb 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -473,7 +473,7 @@ def override_options(self): 'output-style': ("Control output style; auto implies using Rich if available to produce rich output, " "with fallback to basic colored output", 'choice', 'store', OUTPUT_STYLE_AUTO, OUTPUT_STYLES), - 'parallel': ("Specify (maximum) level of parallellism used during build procedure", + 'parallel': ("Specify (maximum) level of parallelism used during build procedure", 'int', 'store', None), 'parallel-extensions-install': ("Install list of extensions in parallel (if supported)", None, 'store_true', False), From c2a8c55bd7ad2aca0aba8c847734dd588e8e3540 Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Thu, 12 Oct 2023 15:38:54 +0100 Subject: [PATCH 22/44] parallelism --- easybuild/tools/systemtools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index b81e018bba..78ba312203 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -1212,7 +1212,7 @@ def get_default_parallelism(): raise EasyBuildError("Specified level of parallelism '%s' is not an integer value: %s", par, err) if maxpar is not None and maxpar < par: - _log.info("Limiting parallellism from %s to %s", par, maxpar) + _log.info("Limiting parallelism from %s to %s", par, maxpar) par = maxpar return par From 5370a6697b58403a0979a4740c1f83e5dacd27e6 Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Thu, 12 Oct 2023 15:39:28 +0100 Subject: [PATCH 23/44] parallelism --- test/framework/systemtools.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/framework/systemtools.py b/test/framework/systemtools.py index 9186b750ec..1707d6ec54 100644 --- a/test/framework/systemtools.py +++ b/test/framework/systemtools.py @@ -835,9 +835,9 @@ def test_system_info(self): def test_det_parallelism_native(self): """Test det_parallelism function (native calls).""" self.assertTrue(det_parallelism() > 0) - # specified parallellism + # specified parallelism self.assertEqual(det_parallelism(par=5), 5) - # max parallellism caps + # max parallelism caps self.assertEqual(det_parallelism(maxpar=1), 1) self.assertEqual(det_parallelism(16, 1), 1) self.assertEqual(det_parallelism(par=5, maxpar=2), 2) From dcf9bb3785f34d8781379875fb2e5b0334e4c29a Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Thu, 12 Oct 2023 15:39:53 +0100 Subject: [PATCH 24/44] parallelism --- test/framework/easyblock.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 819d74fc7a..5e2407575d 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -2080,7 +2080,7 @@ def test_extensions_sanity_check(self): eb.run_all_steps(True) def test_parallel(self): - """Test defining of parallellism.""" + """Test defining of parallelism.""" topdir = os.path.abspath(os.path.dirname(__file__)) toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') toytxt = read_file(toy_ec) @@ -2097,7 +2097,7 @@ def test_parallel(self): os.close(handle) write_file(toy_ec3, toytxt + "\nparallel = False") - # default: parallellism is derived from # available cores + ulimit + # default: parallelism is derived from # available cores + ulimit test_eb = EasyBlock(EasyConfig(toy_ec)) test_eb.check_readiness_step() self.assertTrue(isinstance(test_eb.cfg['parallel'], int) and test_eb.cfg['parallel'] > 0) From f3eafe67bafadd80c53ef9433d5e37c7288070d8 Mon Sep 17 00:00:00 2001 From: sam Date: Tue, 17 Oct 2023 08:57:08 +0200 Subject: [PATCH 25/44] minor help fix --- easybuild/tools/options.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index a55cea4bbf..61d75cb656 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -276,8 +276,8 @@ def basic_options(self): 'only-blocks': ("Only build listed blocks", 'strlist', 'extend', None, 'b', {'metavar': 'BLOCKS'}), 'rebuild': ("Rebuild software, even if module already exists (don't skip OS dependencies checks)", None, 'store_true', False), - 'robot': ("Enable dependency resolution, using easyconfigs in specified paths", - 'pathlist', 'store_or_None', [], 'r', {'metavar': 'PATH[%sPATH]' % os.pathsep}), + 'robot': ("Enable dependency resolution, optionally consider additional paths to search for easyconfigs", + 'pathlist', 'store_or_None', [], 'r', {'metavar': '[PATH[%sPATH]]' % os.pathsep}), 'robot-paths': ("Additional paths to consider by robot for easyconfigs (--robot paths get priority)", 'pathlist', 'add_flex', self.default_robot_paths, {'metavar': 'PATH[%sPATH]' % os.pathsep}), 'search-paths': ("Additional locations to consider in --search (next to --robot and --robot-paths paths)", From 9d43ede93713450d155c1be774cc940f76e0ca0b Mon Sep 17 00:00:00 2001 From: Xin An Date: Tue, 24 Oct 2023 16:17:19 +0200 Subject: [PATCH 26/44] add sysroot template value --- easybuild/framework/easyconfig/templates.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/easybuild/framework/easyconfig/templates.py b/easybuild/framework/easyconfig/templates.py index 465028c3ca..fa8c8861ea 100644 --- a/easybuild/framework/easyconfig/templates.py +++ b/easybuild/framework/easyconfig/templates.py @@ -90,6 +90,8 @@ # template values which are only generated dynamically TEMPLATE_NAMES_DYNAMIC = [ ('arch', "System architecture (e.g. x86_64, aarch64, ppc64le, ...)"), + ('sysroot', "Location root directory of system, prefix for standard paths like /usr/lib and /usr/include" + "as specify by the --sysroot configuration option"), ('mpi_cmd_prefix', "Prefix command for running MPI programs (with default number of ranks)"), ('cuda_compute_capabilities', "Comma-separated list of CUDA compute capabilities, as specified via " "--cuda-compute-capabilities configuration option or via cuda_compute_capabilities easyconfig parameter"), @@ -201,6 +203,13 @@ def template_constant_dict(config, ignore=None, skip_lower=None, toolchain=None) # set 'arch' for system architecture based on 'machine' (4th) element of platform.uname() return value template_values['arch'] = platform.uname()[4] + # set 'sysroot' based on sysroot build option + sysroot = build_option('sysroot') + if sysroot == None: + template_values['sysroot'] = "" + else: + template_values['sysroot'] = sysroot + # step 1: add TEMPLATE_NAMES_EASYCONFIG for name in TEMPLATE_NAMES_EASYCONFIG: if name in ignore: From dfe0c8ed679a163f454f93bde967cbb1031a659a Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 24 Oct 2023 17:14:35 +0200 Subject: [PATCH 27/44] Added unit tests: one for empty sysroot, one for non-empty sysroot --- test/framework/easyconfig.py | 70 ++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 14ed69084b..bd02de939d 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -1346,6 +1346,76 @@ def test_start_dir_template(self): self.assertIn('start_dir in extension configure is %s &&' % ext_start_dir, logtxt) self.assertIn('start_dir in extension build is %s &&' % ext_start_dir, logtxt) + def test_sysroot_template_empty(self): + """Test the %(sysroot)s template whenever --sysroot is unset (i.e. None)""" + + self.contents = textwrap.dedent(""" + name = 'toy' + version = '0.0' + + homepage = 'https://easybuilders.github.io/easybuild' + description = 'Toy C program, 100% toy.' + + toolchain = SYSTEM + + sources = [SOURCE_TAR_GZ] + + preconfigopts = 'echo sysroot in configure is %(sysroot)s && ' + prebuildopts = 'echo sysroot in build is %(sysroot)s && ' + preinstallopts = 'echo sysroot in install is %(sysroot)s && ' + + moduleclass = 'tools' + """) + self.prep() + ec = EasyConfig(self.eb_file) + from easybuild.easyblocks.toy import EB_toy + eb = EB_toy(ec) + # Check behaviour when sysroot is not set (i.e. None) + eb.cfg['sysroot'] = None # Should we define this explicitely? Or rely on this to be the default? + eb.cfg['stop'] = 'extensions' + with self.mocked_stdout_stderr(): + eb.run_all_steps(False) + logtxt = read_file(eb.logfile) + sysroot = "" + self.assertIn('sysroot in configure is %s/ &&' % sysroot, logtxt) + self.assertIn('sysroot in build is %s/ &&' % sysroot, logtxt) + self.assertIn('sysroot in install is %s/ &&' % sysroot, logtxt) + + def test_sysroot_template_non_empty(self): + """Test the %(sysroot)s template whenever --sysroot is unset (i.e. None)""" + + self.contents = textwrap.dedent(""" + name = 'toy' + version = '0.0' + + homepage = 'https://easybuilders.github.io/easybuild' + description = 'Toy C program, 100% toy.' + + toolchain = SYSTEM + + sources = [SOURCE_TAR_GZ] + + preconfigopts = 'echo sysroot in configure is %(sysroot)s && ' + prebuildopts = 'echo sysroot in build is %(sysroot)s && ' + preinstallopts = 'echo sysroot in install is %(sysroot)s && ' + + moduleclass = 'tools' + """) + self.prep() + ec = EasyConfig(self.eb_file) + from easybuild.easyblocks.toy import EB_toy + eb = EB_toy(ec) + # Check behaviour when sysroot is not set (i.e. None) + eb.cfg['sysroot'] = '/tmp' # This should be a path that exists, otherwise EasyBuild complains + eb.cfg['stop'] = 'extensions' + with self.mocked_stdout_stderr(): + eb.run_all_steps(False) + logtxt = read_file(eb.logfile) + sysroot = "" + self.assertIn('sysroot in configure is %s/ &&' % sysroot, logtxt) + self.assertIn('sysroot in build is %s/ &&' % sysroot, logtxt) + self.assertIn('sysroot in install is %s/ &&' % sysroot, logtxt) + def test_constant_doc(self): """test constant documentation""" doc = avail_easyconfig_constants() From 6d6571a902c6f004bdc7bd840b696e028f179ecf Mon Sep 17 00:00:00 2001 From: Xin An <34663977+xinan1911@users.noreply.github.com> Date: Tue, 24 Oct 2023 17:28:41 +0200 Subject: [PATCH 28/44] Update templates.py for the style check --- easybuild/framework/easyconfig/templates.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyconfig/templates.py b/easybuild/framework/easyconfig/templates.py index fa8c8861ea..9949955854 100644 --- a/easybuild/framework/easyconfig/templates.py +++ b/easybuild/framework/easyconfig/templates.py @@ -205,7 +205,7 @@ def template_constant_dict(config, ignore=None, skip_lower=None, toolchain=None) # set 'sysroot' based on sysroot build option sysroot = build_option('sysroot') - if sysroot == None: + if sysroot is None: template_values['sysroot'] = "" else: template_values['sysroot'] = sysroot From 2e6bd7a3c4662456dc050916ee4ee06b0cc9b82e Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen <33718780+casparvl@users.noreply.github.com> Date: Wed, 25 Oct 2023 11:01:53 +0200 Subject: [PATCH 29/44] Update easybuild/framework/easyconfig/templates.py Co-authored-by: Kenneth Hoste --- easybuild/framework/easyconfig/templates.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/easybuild/framework/easyconfig/templates.py b/easybuild/framework/easyconfig/templates.py index 9949955854..07d838eee9 100644 --- a/easybuild/framework/easyconfig/templates.py +++ b/easybuild/framework/easyconfig/templates.py @@ -203,12 +203,9 @@ def template_constant_dict(config, ignore=None, skip_lower=None, toolchain=None) # set 'arch' for system architecture based on 'machine' (4th) element of platform.uname() return value template_values['arch'] = platform.uname()[4] - # set 'sysroot' based on sysroot build option - sysroot = build_option('sysroot') - if sysroot is None: - template_values['sysroot'] = "" - else: - template_values['sysroot'] = sysroot + # set 'sysroot' template based on 'sysroot' configuration option, using empty string as fallback + value + template_values['sysroot'] = build_option('sysroot') or '' # step 1: add TEMPLATE_NAMES_EASYCONFIG for name in TEMPLATE_NAMES_EASYCONFIG: From d52c7792ff881e9f43b33804e2f3400f847b9703 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 25 Oct 2023 11:11:12 +0200 Subject: [PATCH 30/44] Removed stray 'value' in code --- easybuild/framework/easyconfig/templates.py | 1 - 1 file changed, 1 deletion(-) diff --git a/easybuild/framework/easyconfig/templates.py b/easybuild/framework/easyconfig/templates.py index 07d838eee9..aed8db6af4 100644 --- a/easybuild/framework/easyconfig/templates.py +++ b/easybuild/framework/easyconfig/templates.py @@ -204,7 +204,6 @@ def template_constant_dict(config, ignore=None, skip_lower=None, toolchain=None) template_values['arch'] = platform.uname()[4] # set 'sysroot' template based on 'sysroot' configuration option, using empty string as fallback - value template_values['sysroot'] = build_option('sysroot') or '' # step 1: add TEMPLATE_NAMES_EASYCONFIG From 510cb33b3a7e5b05524db47a081b409069099b89 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 25 Oct 2023 11:20:49 +0200 Subject: [PATCH 31/44] add `EasyConfig.dependency_names` --- easybuild/framework/easyconfig/easyconfig.py | 9 +++++++++ test/framework/easyconfig.py | 19 +++++++++++++------ 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 55fe3c8578..ff4ed3562c 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -1139,6 +1139,15 @@ def dependencies(self, build_only=False): return retained_deps + def dependency_names(self, build_only=False): + """ + Return a set of names of all (direct) dependencies after filtering. + Iterable builddependencies are flattened when not iterating. + + :param build_only: only return build dependencies, discard others + """ + return {dep['name'] for dep in self.dependencies(build_only=build_only) if dep['name']} + def builddependencies(self): """ Return a flat list of the parsed build dependencies diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 14ed69084b..5fde75938d 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -301,7 +301,9 @@ def test_dependency(self): self.assertEqual(det_full_ec_version(first), '1.1-GCC-4.6.3') self.assertEqual(det_full_ec_version(second), '2.2-GCC-4.6.3') + self.assertEqual(eb.dependency_names(), {'first', 'second', 'foo', 'bar'}) # same tests for builddependencies + self.assertEqual(eb.dependency_names(build_only=True), {'first', 'second'}) first = eb.builddependencies()[0] second = eb.builddependencies()[1] @@ -354,6 +356,7 @@ def test_false_dep_version(self): self.assertEqual(len(deps), 2) self.assertEqual(deps[0]['name'], 'second_build') self.assertEqual(deps[1]['name'], 'first') + self.assertEqual(eb.dependency_names(), {'first', 'second_build'}) # more realistic example: only filter dep for POWER self.contents = '\n'.join([ @@ -377,12 +380,14 @@ def test_false_dep_version(self): deps = eb.dependencies() self.assertEqual(len(deps), 1) self.assertEqual(deps[0]['name'], 'not_on_power') + self.assertEqual(eb.dependency_names(), {'not_on_power'}) # only power, dependency gets filtered st.get_cpu_architecture = lambda: POWER eb = EasyConfig(self.eb_file) deps = eb.dependencies() self.assertEqual(deps, []) + self.assertEqual(eb.dependency_names(), {}) def test_extra_options(self): """ extra_options should allow other variables to be stored """ @@ -1608,18 +1613,15 @@ def test_filter_deps(self): test_ecs_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs') ec_file = os.path.join(test_ecs_dir, 'f', 'foss', 'foss-2018a.eb') ec = EasyConfig(ec_file) - deps = sorted([dep['name'] for dep in ec.dependencies()]) - self.assertEqual(deps, ['FFTW', 'GCC', 'OpenBLAS', 'OpenMPI', 'ScaLAPACK']) + self.assertEqual(ec.dependency_names(), {'FFTW', 'GCC', 'OpenBLAS', 'OpenMPI', 'ScaLAPACK'}) # test filtering multiple deps init_config(build_options={'filter_deps': ['FFTW', 'ScaLAPACK']}) - deps = sorted([dep['name'] for dep in ec.dependencies()]) - self.assertEqual(deps, ['GCC', 'OpenBLAS', 'OpenMPI']) + self.assertEqual(ec.dependency_names(), {'GCC', 'OpenBLAS', 'OpenMPI'}) # test filtering of non-existing dep init_config(build_options={'filter_deps': ['zlib']}) - deps = sorted([dep['name'] for dep in ec.dependencies()]) - self.assertEqual(deps, ['FFTW', 'GCC', 'OpenBLAS', 'OpenMPI', 'ScaLAPACK']) + self.assertEqual(ec.dependency_names(), {'FFTW', 'GCC', 'OpenBLAS', 'OpenMPI', 'ScaLAPACK'}) # test parsing of value passed to --filter-deps opts = init_config(args=[]) @@ -1653,6 +1655,7 @@ def test_filter_deps(self): init_config(build_options=build_options) ec = EasyConfig(ec_file, validate=False) self.assertEqual(ec.dependencies(), []) + self.assertEqual(ec.dependency_names(), {}) def test_replaced_easyconfig_parameters(self): """Test handling of replaced easyconfig parameters.""" @@ -1841,6 +1844,9 @@ def test_external_dependencies(self): } self.assertEqual(deps[7]['external_module_metadata'], cray_netcdf_metadata) + # External module names are omitted + self.assertEqual(ec.dependency_names(), {'intel'}) + # provide file with partial metadata for some external modules; # metadata obtained from probing modules should be added to it... metadata = os.path.join(self.test_prefix, 'external_modules_metadata.cfg') @@ -1869,6 +1875,7 @@ def test_external_dependencies(self): deps = ec.dependencies() self.assertEqual(len(deps), 8) + self.assertEqual(ec.dependency_names(), {'intel'}) for idx in [0, 1, 2, 6]: self.assertEqual(deps[idx]['external_module_metadata'], {}) From 097f3942abeca593dd791959b066e21c0c2f8c97 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 25 Oct 2023 11:35:04 +0200 Subject: [PATCH 32/44] Updated tests for syroot template --- test/framework/easyconfig.py | 88 ++++++++++-------------------------- 1 file changed, 23 insertions(+), 65 deletions(-) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index bd02de939d..aed49f291e 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -1346,75 +1346,33 @@ def test_start_dir_template(self): self.assertIn('start_dir in extension configure is %s &&' % ext_start_dir, logtxt) self.assertIn('start_dir in extension build is %s &&' % ext_start_dir, logtxt) - def test_sysroot_template_empty(self): - """Test the %(sysroot)s template whenever --sysroot is unset (i.e. None)""" + def test_sysroot_template(self): + """Test the %(sysroot)s template""" - self.contents = textwrap.dedent(""" - name = 'toy' - version = '0.0' - - homepage = 'https://easybuilders.github.io/easybuild' - description = 'Toy C program, 100% toy.' - - toolchain = SYSTEM - - sources = [SOURCE_TAR_GZ] - - preconfigopts = 'echo sysroot in configure is %(sysroot)s && ' - prebuildopts = 'echo sysroot in build is %(sysroot)s && ' - preinstallopts = 'echo sysroot in install is %(sysroot)s && ' - - moduleclass = 'tools' - """) - self.prep() - ec = EasyConfig(self.eb_file) - from easybuild.easyblocks.toy import EB_toy - eb = EB_toy(ec) - # Check behaviour when sysroot is not set (i.e. None) - eb.cfg['sysroot'] = None # Should we define this explicitely? Or rely on this to be the default? - eb.cfg['stop'] = 'extensions' - with self.mocked_stdout_stderr(): - eb.run_all_steps(False) - logtxt = read_file(eb.logfile) - sysroot = "" - self.assertIn('sysroot in configure is %s/ &&' % sysroot, logtxt) - self.assertIn('sysroot in build is %s/ &&' % sysroot, logtxt) - self.assertIn('sysroot in install is %s/ &&' % sysroot, logtxt) - - def test_sysroot_template_non_empty(self): - """Test the %(sysroot)s template whenever --sysroot is unset (i.e. None)""" - - self.contents = textwrap.dedent(""" - name = 'toy' - version = '0.0' - - homepage = 'https://easybuilders.github.io/easybuild' - description = 'Toy C program, 100% toy.' - - toolchain = SYSTEM + test_easyconfigs = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs') + toy_ec = os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0.eb') + + test_ec = os.path.join(self.test_prefix, 'test.eb') + test_ec_txt = read_file(toy_ec) + test_ec_txt += '\nconfigopts = "--some-opt=%(sysroot)s/"' + test_ec_txt += '\nbuildopts = "--some-opt=%(sysroot)s/"' + test_ec_txt += '\ninstallopts = "--some-opt=%(sysroot)s/"' + write_file(test_ec, test_ec_txt) - sources = [SOURCE_TAR_GZ] + # Validate the value of the sysroot template if sysroot is unset (i.e. the build option is None) + ec = EasyConfig(test_ec) + self.assertEqual(ec['configopts'], "--some-opt=/") + self.assertEqual(ec['buildopts'], "--some-opt=/") + self.assertEqual(ec['installopts'], "--some-opt=/") - preconfigopts = 'echo sysroot in configure is %(sysroot)s && ' - prebuildopts = 'echo sysroot in build is %(sysroot)s && ' - preinstallopts = 'echo sysroot in install is %(sysroot)s && ' + # Validate the value of the sysroot template if sysroot is unset (i.e. the build option is None) + # As a test, we'll set the sysroot to self.test_prefix, as it has to be a directory that is guaranteed to exist + update_build_option('sysroot', self.test_prefix) - moduleclass = 'tools' - """) - self.prep() - ec = EasyConfig(self.eb_file) - from easybuild.easyblocks.toy import EB_toy - eb = EB_toy(ec) - # Check behaviour when sysroot is not set (i.e. None) - eb.cfg['sysroot'] = '/tmp' # This should be a path that exists, otherwise EasyBuild complains - eb.cfg['stop'] = 'extensions' - with self.mocked_stdout_stderr(): - eb.run_all_steps(False) - logtxt = read_file(eb.logfile) - sysroot = "" - self.assertIn('sysroot in configure is %s/ &&' % sysroot, logtxt) - self.assertIn('sysroot in build is %s/ &&' % sysroot, logtxt) - self.assertIn('sysroot in install is %s/ &&' % sysroot, logtxt) + ec = EasyConfig(test_ec) + self.assertEqual(ec['configopts'], "--some-opt=%s/" % self.test_prefix) + self.assertEqual(ec['buildopts'], "--some-opt=%s/" % self.test_prefix) + self.assertEqual(ec['installopts'], "--some-opt=%s/" % self.test_prefix) def test_constant_doc(self): """test constant documentation""" From 007667751d99b849c735013b13e35253a022b86c Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 25 Oct 2023 11:36:14 +0200 Subject: [PATCH 33/44] Make hound happy --- test/framework/easyconfig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index aed49f291e..209b92e479 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -1351,7 +1351,7 @@ def test_sysroot_template(self): test_easyconfigs = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs') toy_ec = os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0.eb') - + test_ec = os.path.join(self.test_prefix, 'test.eb') test_ec_txt = read_file(toy_ec) test_ec_txt += '\nconfigopts = "--some-opt=%(sysroot)s/"' From 8c9b09a994fc60cfd5a4ce9bac0f4bfdf627ff16 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 25 Oct 2023 12:31:34 +0200 Subject: [PATCH 34/44] Explicitely use `set()` instead of `{}` Some Python version may interpret the latter as an (empty) dict instead of a set --- test/framework/easyconfig.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 5fde75938d..fe921aa52e 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -387,7 +387,7 @@ def test_false_dep_version(self): eb = EasyConfig(self.eb_file) deps = eb.dependencies() self.assertEqual(deps, []) - self.assertEqual(eb.dependency_names(), {}) + self.assertEqual(eb.dependency_names(), set()) def test_extra_options(self): """ extra_options should allow other variables to be stored """ @@ -1655,7 +1655,7 @@ def test_filter_deps(self): init_config(build_options=build_options) ec = EasyConfig(ec_file, validate=False) self.assertEqual(ec.dependencies(), []) - self.assertEqual(ec.dependency_names(), {}) + self.assertEqual(ec.dependency_names(), set()) def test_replaced_easyconfig_parameters(self): """Test handling of replaced easyconfig parameters.""" From ea000b29e3f6d69ea61964521bc5499596933759 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 25 Oct 2023 14:02:17 +0200 Subject: [PATCH 35/44] Restore original sysroot at the end of test --- test/framework/easyconfig.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 209b92e479..a410e202e4 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -1367,6 +1367,7 @@ def test_sysroot_template(self): # Validate the value of the sysroot template if sysroot is unset (i.e. the build option is None) # As a test, we'll set the sysroot to self.test_prefix, as it has to be a directory that is guaranteed to exist + old_sysroot = build_option('sysroot') update_build_option('sysroot', self.test_prefix) ec = EasyConfig(test_ec) @@ -1374,6 +1375,9 @@ def test_sysroot_template(self): self.assertEqual(ec['buildopts'], "--some-opt=%s/" % self.test_prefix) self.assertEqual(ec['installopts'], "--some-opt=%s/" % self.test_prefix) + # Restore original value for sysroot build option + update_build_option('sysroot', old_sysroot) + def test_constant_doc(self): """test constant documentation""" doc = avail_easyconfig_constants() From e6bc40315b48378a7db4c036d837f7033ec75b8f Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 25 Oct 2023 14:48:24 +0200 Subject: [PATCH 36/44] Environment reset is not needed at the end of the test, each one starts with clean env. The real problem was that the template_constant_dict has changed, so the expected output is now different - and should include sysroot and its value --- test/framework/easyconfig.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index a410e202e4..5ecf493b2f 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -1367,7 +1367,6 @@ def test_sysroot_template(self): # Validate the value of the sysroot template if sysroot is unset (i.e. the build option is None) # As a test, we'll set the sysroot to self.test_prefix, as it has to be a directory that is guaranteed to exist - old_sysroot = build_option('sysroot') update_build_option('sysroot', self.test_prefix) ec = EasyConfig(test_ec) @@ -1375,9 +1374,6 @@ def test_sysroot_template(self): self.assertEqual(ec['buildopts'], "--some-opt=%s/" % self.test_prefix) self.assertEqual(ec['installopts'], "--some-opt=%s/" % self.test_prefix) - # Restore original value for sysroot build option - update_build_option('sysroot', old_sysroot) - def test_constant_doc(self): """test constant documentation""" doc = avail_easyconfig_constants() @@ -3266,6 +3262,7 @@ def test_template_constant_dict(self): 'nameletter': 'g', 'nameletterlower': 'g', 'parallel': None, + 'sysroot': '', 'toolchain_name': 'foss', 'toolchain_version': '2018a', 'version': '1.5', From f2bae7f811ae6c12bbdae922caf0f68c00858db8 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 25 Oct 2023 15:09:20 +0200 Subject: [PATCH 37/44] One more change in expected output. Probably need one more after this, lets try... --- test/framework/easyconfig.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 5ecf493b2f..742bfe50f9 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -3344,6 +3344,7 @@ def test_template_constant_dict(self): 'pyminver': '7', 'pyshortver': '3.7', 'pyver': '3.7.2', + 'sysroot': '', 'version': '0.01', 'version_major': '0', 'version_major_minor': '0.01', From a4105265c1632f0cef69f26b5c8df464bae5ed38 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 25 Oct 2023 15:59:28 +0200 Subject: [PATCH 38/44] Fix hopefully last reference in test_template_constant_dict to account for addition of sysroot template --- test/framework/easyconfig.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 742bfe50f9..ddc0f78b57 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -3409,6 +3409,7 @@ def test_template_constant_dict(self): 'namelower': 'foo', 'nameletter': 'f', 'nameletterlower': 'f', + 'sysroot': '', 'version': '1.2.3', 'version_major': '1', 'version_major_minor': '1.2', From 9c4c794bbebb5d66099b1ace51177985e37c00c6 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 26 Oct 2023 16:18:29 +0200 Subject: [PATCH 39/44] fix findPythonDeps when called with an absolute path to an EC `eb --missing` returns a list with filenames. So an absolute path passed to `--ec` isn't matched which leads to an error stating you should install the EasyConfig that you are trying to find the dependencies of. Fix by getting the filename/basename first. --- easybuild/scripts/findPythonDeps.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/easybuild/scripts/findPythonDeps.py b/easybuild/scripts/findPythonDeps.py index 663926a9d6..ef284a7e21 100755 --- a/easybuild/scripts/findPythonDeps.py +++ b/easybuild/scripts/findPythonDeps.py @@ -171,8 +171,9 @@ def print_deps(package, verbose): capture_stderr=False, action_desc='Get missing dependencies' ) + excluded_dep = '(%s)' % os.path.basename(args.ec) missing_deps = [dep for dep in missing_dep_out.split('\n') - if dep.startswith('*') and '(%s)' % args.ec not in dep + if dep.startswith('*') and excluded_dep not in dep ] if missing_deps: print('You need to install all modules on which %s depends first!' % args.ec) From dfe010283b132690fb2051180dfb8ecfe272b267 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Fri, 27 Oct 2023 09:14:59 +0200 Subject: [PATCH 40/44] Show stderr output on failing command even when not requested to be captured This helps diagnosing failures for e.g. `eb --missing` (called with `capture_stderr=False`) but which may print errors (e.g. failed parsing of some ECs) to stderr --- easybuild/scripts/findPythonDeps.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/easybuild/scripts/findPythonDeps.py b/easybuild/scripts/findPythonDeps.py index ef284a7e21..4f4200de09 100755 --- a/easybuild/scripts/findPythonDeps.py +++ b/easybuild/scripts/findPythonDeps.py @@ -55,9 +55,11 @@ def run_cmd(arguments, action_desc, capture_stderr=True, **kwargs): extra_args['universal_newlines'] = True stderr = subprocess.STDOUT if capture_stderr else subprocess.PIPE p = subprocess.Popen(arguments, stdout=subprocess.PIPE, stderr=stderr, **extra_args) - out, _ = p.communicate() + out, err = p.communicate() if p.returncode != 0: - raise RuntimeError('Failed to %s: %s' % (action_desc, out)) + if err: + err = "\nSTDERR:\n" + err + raise RuntimeError('Failed to %s: %s%s' % (action_desc, out, err)) return out From cf0bd4f2e68c2b5521dfb894c4e96dc820a179b9 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Fri, 27 Oct 2023 10:17:02 +0200 Subject: [PATCH 41/44] fix findPythonDeps when using a relative path as the `--ec` argument --- easybuild/scripts/findPythonDeps.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/easybuild/scripts/findPythonDeps.py b/easybuild/scripts/findPythonDeps.py index 4f4200de09..d6e496a048 100755 --- a/easybuild/scripts/findPythonDeps.py +++ b/easybuild/scripts/findPythonDeps.py @@ -182,12 +182,14 @@ def print_deps(package, verbose): print('\n\t'.join(['Missing:'] + missing_deps)) sys.exit(1) + # If the --ec argument is a (relative) existing path make it absolute so we can find it after the chdir + ec_arg = os.path.abspath(args.ec) if os.path.exists(args.ec) else args.ec with temporary_directory() as tmp_dir: old_dir = os.getcwd() os.chdir(tmp_dir) if args.verbose: print('Running EasyBuild to get build environment') - run_cmd(['eb', args.ec, '--dump-env', '--force'], action_desc='Dump build environment') + run_cmd(['eb', ec_arg, '--dump-env', '--force'], action_desc='Dump build environment') os.chdir(old_dir) cmd = "source %s/*.env && python %s '%s'" % (tmp_dir, sys.argv[0], args.package) From a842a38a7cfd44119e31ef4144efdaac1dd4cd15 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 27 Oct 2023 10:28:30 +0200 Subject: [PATCH 42/44] fix broken test for reasons_for_closing, which fails because commit status of easyconfigs PR is no longer available --- test/framework/github.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/framework/github.py b/test/framework/github.py index c9bfbaedd3..f011c89b4f 100644 --- a/test/framework/github.py +++ b/test/framework/github.py @@ -345,7 +345,6 @@ def test_github_reasons_for_closing(self): self.assertIsInstance(res, list) self.assertEqual(stderr.strip(), "WARNING: Using easyconfigs from closed PR #16080") patterns = [ - "Status of last commit is SUCCESS", "Last comment on", "No activity since", "* c-ares-1.18.1", From 642fe35a84a0b6c47a10b3c0d1ef48b4ff08701f Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Sat, 28 Oct 2023 16:26:58 +0800 Subject: [PATCH 43/44] prepare release notes for EasyBuild v4.8.2 + bump version to 4.8.2 --- RELEASE_NOTES | 22 ++++++++++++++++++++++ easybuild/tools/version.py | 2 +- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index 5bd07d4306..8365871584 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -4,6 +4,28 @@ For more detailed information, please see the git log. These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html. +v4.8.2 (29 October 2023) +------------------------ + +update/bugfix release + +- various enhancements, including: + - add support for `%(sysroot)s` template value (#4359) + - add `dependency_names` method to `EasyConfig` class to get set of names of (direct) dependencies (#4360) +- various bug fixes, including: + - add CI workflow to run unit tests with Python 2 (again) (#4333) + - fix typo in help message for --silence-hook-trigger (#4343) + - include major version (`*majver`) templates in auto-generated documentation (#4347) + - reset `tempfile.tempdir` to `None` to avoid that tmpdir path gets progressively deeper with each easystack item (#4350) + - fix parallellisation typo (#4352) + - minor help fix (#4355) + - fix `findPythonDeps.py` script when called with an absolute path to an easyconfig (#4365) + - fix broken test for `reasons_for_closing`, which fails because commit status of easyconfigs PR is no longer available (#4366) +- other changes: + - reduce number of CI jobs by testing for Lua and Tcl module syntax in a single CI job (#4192) + - clean up stray regex.search in `test_checksum_step` (#4345) + + v4.8.1 (11 September 2023) -------------------------- diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py index e34584df36..8582267629 100644 --- a/easybuild/tools/version.py +++ b/easybuild/tools/version.py @@ -45,7 +45,7 @@ # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like # UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0' # This causes problems further up the dependency chain... -VERSION = LooseVersion('4.8.2.dev0') +VERSION = LooseVersion('4.8.2') UNKNOWN = 'UNKNOWN' From 49deccdceddec80b6b782a6b99af19c3d98c78ea Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 28 Oct 2023 16:48:38 +0200 Subject: [PATCH 44/44] tweak release notes for EasyBuild v4.8.2 --- RELEASE_NOTES | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index 8365871584..a43ce28535 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -14,16 +14,13 @@ update/bugfix release - add `dependency_names` method to `EasyConfig` class to get set of names of (direct) dependencies (#4360) - various bug fixes, including: - add CI workflow to run unit tests with Python 2 (again) (#4333) - - fix typo in help message for --silence-hook-trigger (#4343) + - fix typo in help message for `--silence-hook-trigger` (#4343) - include major version (`*majver`) templates in auto-generated documentation (#4347) - reset `tempfile.tempdir` to `None` to avoid that tmpdir path gets progressively deeper with each easystack item (#4350) - - fix parallellisation typo (#4352) - - minor help fix (#4355) - - fix `findPythonDeps.py` script when called with an absolute path to an easyconfig (#4365) + - fix `findPythonDeps.py` script when called with an (absolute or relative) path to an easyconfig instead of a filename (#4365) - fix broken test for `reasons_for_closing`, which fails because commit status of easyconfigs PR is no longer available (#4366) - other changes: - reduce number of CI jobs by testing for Lua and Tcl module syntax in a single CI job (#4192) - - clean up stray regex.search in `test_checksum_step` (#4345) v4.8.1 (11 September 2023)