add Generation module naming scheme #3967
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions | |
| name: EasyBuild framework unit tests | |
| on: [push, pull_request] | |
| jobs: | |
| setup: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| lmod7: Lmod-7.8.22 | |
| lmod8: Lmod-8.4.27 | |
| modulesTcl: modules-tcl-1.147 | |
| modules3: modules-3.2.10 | |
| modules4: modules-4.1.4 | |
| steps: | |
| - run: "true" | |
| build: | |
| needs: setup | |
| runs-on: ubuntu-18.04 | |
| strategy: | |
| matrix: | |
| python: [2.7, 3.6] | |
| modules_tool: | |
| # use variables defined by 'setup' job above, see also | |
| # https://docs.github.com/en/actions/reference/context-and-expression-syntax-for-github-actions#needs-context | |
| - ${{needs.setup.outputs.lmod7}} | |
| - ${{needs.setup.outputs.lmod8}} | |
| - ${{needs.setup.outputs.modulesTcl}} | |
| - ${{needs.setup.outputs.modules3}} | |
| - ${{needs.setup.outputs.modules4}} | |
| module_syntax: [Lua, Tcl] | |
| lc_all: [""] | |
| # don't test with Lua module syntax (only supported in Lmod) | |
| exclude: | |
| - modules_tool: ${{needs.setup.outputs.modulesTcl}} | |
| module_syntax: Lua | |
| - modules_tool: ${{needs.setup.outputs.modules3}} | |
| module_syntax: Lua | |
| - modules_tool: ${{needs.setup.outputs.modules4}} | |
| module_syntax: Lua | |
| include: | |
| # Test different Python 3 versions with Lmod 8.x (with both Lua and Tcl module syntax) | |
| - python: 3.5 | |
| modules_tool: ${{needs.setup.outputs.lmod8}} | |
| module_syntax: Lua | |
| - python: 3.5 | |
| modules_tool: ${{needs.setup.outputs.lmod8}} | |
| module_syntax: Tcl | |
| - python: 3.7 | |
| modules_tool: ${{needs.setup.outputs.lmod8}} | |
| module_syntax: Lua | |
| - python: 3.7 | |
| modules_tool: ${{needs.setup.outputs.lmod8}} | |
| module_syntax: Tcl | |
| - python: 3.8 | |
| modules_tool: ${{needs.setup.outputs.lmod8}} | |
| module_syntax: Lua | |
| - python: 3.8 | |
| modules_tool: ${{needs.setup.outputs.lmod8}} | |
| module_syntax: Tcl | |
| - python: 3.9 | |
| modules_tool: ${{needs.setup.outputs.lmod8}} | |
| module_syntax: Lua | |
| - python: 3.9 | |
| modules_tool: ${{needs.setup.outputs.lmod8}} | |
| module_syntax: Tcl | |
| - python: '3.10' | |
| modules_tool: ${{needs.setup.outputs.lmod8}} | |
| module_syntax: Lua | |
| - python: '3.10' | |
| modules_tool: ${{needs.setup.outputs.lmod8}} | |
| module_syntax: Tcl | |
| # There may be encoding errors in Python 3 which are hidden when an UTF-8 encoding is set | |
| # Hence run the tests (again) with LC_ALL=C and Python 3.6 (or any < 3.7) | |
| - python: 3.6 | |
| modules_tool: ${{needs.setup.outputs.lmod8}} | |
| module_syntax: Lua | |
| lc_all: C | |
| fail-fast: false | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: set up Python | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: ${{matrix.python}} | |
| architecture: x64 | |
| - name: install OS & Python packages | |
| run: | | |
| # disable apt-get update, we don't really need it, | |
| # and it does more harm than good (it's fairly expensive, and it results in flaky test runs) | |
| # sudo apt-get update | |
| # for modules tool | |
| sudo apt-get install lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev | |
| # fix for lua-posix packaging issue, see https://bugs.launchpad.net/ubuntu/+source/lua-posix/+bug/1752082 | |
| # needed for Ubuntu 18.04, but not for Ubuntu 20.04, so skipping symlinking if posix.so already exists | |
| if [ ! -e /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so ] ; then | |
| sudo ln -s /usr/lib/x86_64-linux-gnu/lua/5.2/posix_c.so /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so | |
| fi | |
| # for GitPython, python-hglib | |
| sudo apt-get install git mercurial | |
| # dep for GC3Pie | |
| sudo apt-get install time | |
| # Python packages | |
| pip --version | |
| pip install --upgrade pip | |
| pip --version | |
| pip install -r requirements.txt | |
| # git config is required to make actual git commits (cfr. tests for GitRepository) | |
| git config --global user.name "Travis CI" | |
| git config --global user.email "[email protected]" | |
| git config --get-regexp 'user.*' | |
| - name: install GitHub token (if available) | |
| env: | |
| # see https://github.com/<username>/easybuild-framework/settings/secrets | |
| GITHUB_TOKEN: ${{secrets.TEST_GITHUB_TOKEN}} | |
| run: | | |
| # don't install GitHub token when testing with Lmod 7.x or non-Lmod module tools, | |
| # and only when testing with Lua as module syntax, | |
| # to avoid hitting GitHub rate limit; | |
| # tests that require a GitHub token are skipped automatically when no GitHub token is available | |
| if [[ ! "${{matrix.modules_tool}}" =~ 'Lmod-7' ]] && [[ ! "${{matrix.modules_tool}}" =~ 'modules-' ]] && [[ "${{matrix.modules_syntax}}" == 'Lua' ]]; then | |
| if [ ! -z $GITHUB_TOKEN ]; then | |
| if [ "x${{matrix.python}}" == 'x2.6' ]; | |
| then SET_KEYRING="keyring.set_keyring(keyring.backends.file.PlaintextKeyring())"; | |
| else SET_KEYRING="import keyrings.alt.file; keyring.set_keyring(keyrings.alt.file.PlaintextKeyring())"; | |
| fi; | |
| python -c "import keyring; $SET_KEYRING; keyring.set_password('github_token', 'easybuild_test', '$GITHUB_TOKEN')"; | |
| fi | |
| echo "GitHub token installed!" | |
| else | |
| echo "Installation of GitHub token skipped!" | |
| fi | |
| - name: install modules tool | |
| run: | | |
| # avoid downloading modules tool sources into easybuild-framework dir | |
| cd $HOME | |
| export INSTALL_DEP=$GITHUB_WORKSPACE/easybuild/scripts/install_eb_dep.sh | |
| # install Lmod | |
| source $INSTALL_DEP ${{matrix.modules_tool}} $HOME | |
| # changes in environment are not passed to other steps, so need to create files... | |
| echo $MOD_INIT > mod_init | |
| echo $PATH > path | |
| if [ ! -z $MODULESHOME ]; then echo $MODULESHOME > moduleshome; fi | |
| - name: check sources | |
| run: | | |
| # make sure there are no (top-level) "import setuptools" or "import pkg_resources" statements, | |
| # since EasyBuild should not have a runtime requirement on setuptools | |
| SETUPTOOLS_IMPORTS=$(egrep -RI '^(from|import)[ ]*pkg_resources|^(from|import)[ ]*setuptools' * || true) | |
| test "x$SETUPTOOLS_IMPORTS" = "x" || (echo "Found setuptools and/or pkg_resources imports in easybuild/:\n${SETUPTOOLS_IMPORTS}" && exit 1) | |
| - name: install sources | |
| run: | | |
| # install from source distribution tarball, to test release as published on PyPI | |
| python setup.py sdist | |
| ls dist | |
| export PREFIX=/tmp/$USER/$GITHUB_SHA | |
| pip install --prefix $PREFIX dist/easybuild-framework*tar.gz | |
| - name: run test suite | |
| env: | |
| EB_VERBOSE: 1 | |
| EASYBUILD_MODULE_SYNTAX: ${{matrix.module_syntax}} | |
| TEST_EASYBUILD_MODULE_SYNTAX: ${{matrix.module_syntax}} | |
| LC_ALL: ${{matrix.lc_all}} | |
| run: | | |
| # run tests *outside* of checked out easybuild-framework directory, | |
| # to ensure we're testing installed version (see previous step) | |
| cd $HOME | |
| # initialize environment for modules tool | |
| if [ -f $HOME/moduleshome ]; then export MODULESHOME=$(cat $HOME/moduleshome); fi | |
| source $(cat $HOME/mod_init); type module | |
| # make sure 'eb' is available via $PATH, and that $PYTHONPATH is set (some tests expect that); | |
| # also pick up changes to $PATH set by sourcing $MOD_INIT | |
| export PREFIX=/tmp/$USER/$GITHUB_SHA | |
| export PATH=$PREFIX/bin:$(cat $HOME/path) | |
| export PYTHONPATH=$PREFIX/lib/python${{matrix.python}}/site-packages:$PYTHONPATH | |
| eb --version | |
| # tell EasyBuild which modules tool is available | |
| if [[ ${{matrix.modules_tool}} =~ ^modules-tcl- ]]; then | |
| export EASYBUILD_MODULES_TOOL=EnvironmentModulesTcl | |
| elif [[ ${{matrix.modules_tool}} =~ ^modules-3 ]]; then | |
| export EASYBUILD_MODULES_TOOL=EnvironmentModulesC | |
| elif [[ ${{matrix.modules_tool}} =~ ^modules-4 ]]; then | |
| export EASYBUILD_MODULES_TOOL=EnvironmentModules | |
| else | |
| export EASYBUILD_MODULES_TOOL=Lmod | |
| fi | |
| export TEST_EASYBUILD_MODULES_TOOL=$EASYBUILD_MODULES_TOOL | |
| eb --show-config | |
| # gather some useful info on test system | |
| eb --show-system-info | |
| # check GitHub configuration | |
| eb --check-github --github-user=easybuild_test | |
| # create file owned by root but writable by anyone (used by test_copy_file) | |
| sudo touch /tmp/file_to_overwrite_for_easybuild_test_copy_file.txt | |
| sudo chmod o+w /tmp/file_to_overwrite_for_easybuild_test_copy_file.txt | |
| # run test suite | |
| python -O -m test.framework.suite 2>&1 | tee test_framework_suite.log | |
| # try and make sure output of running tests is clean (no printed messages/warnings) | |
| IGNORE_PATTERNS="no GitHub token available|skipping SvnRepository test|requires Lmod as modules tool|stty: 'standard input': Inappropriate ioctl for device|CryptographyDeprecationWarning: Python 3.[56]|from cryptography.*default_backend|CryptographyDeprecationWarning: Python 2|from cryptography.utils import int_from_bytes|Blowfish" | |
| # '|| true' is needed to avoid that Travis stops the job on non-zero exit of grep (i.e. when there are no matches) | |
| PRINTED_MSG=$(egrep -v "${IGNORE_PATTERNS}" test_framework_suite.log | grep '\.\n*[A-Za-z]' || true) | |
| test "x$PRINTED_MSG" = "x" || (echo "ERROR: Found printed messages in output of test suite\n${PRINTED_MSG}" && exit 1) |